标签：cgroups inf cap running map gadget pua wrong searching
部署的docker容器运行一端时间以后，由于需要修改配置，需要重启容器，但是重启的时候，报如下错误：
Error response from daemon: Cannot restart container rsnmp_v4: OCI runtime create failed: container_linux.go:349: starting container process caused "process_linux.go:297: applying cgroup configuration for process caused \"mountpoint for devices not found\"": unknown
上诉错误提示cgroup 挂载设备找不到
1、使用官方的脚本检查当前的配置是否ok
https://github.com/moby/moby/blob/master/contrib/check-config.sh
脚本的内容如下：
cat check-config.sh
#!/usr/bin/env bash
set -e
EXITCODE=0
# bits of this were adapted from lxc-checkconfig
# see also https://github.com/lxc/lxc/blob/lxc-1.0.2/src/lxc/lxc-checkconfig.in
possibleConfigs=(
‘/proc/config.gz‘
"/boot/config-$(uname -r)"
"/usr/src/linux-$(uname -r)/.config"
‘/usr/src/linux/.config‘
)
if [ $# -gt 0 ]; then
CONFIG="$1"
else
: "${CONFIG:="${possibleConfigs[0]}"}"
fi
if ! command -v zgrep &> /dev/null; then
zgrep() {
zcat "$2" | grep "$1"
}
fi
kernelVersion="$(uname -r)"
kernelMajor="${kernelVersion%%.*}"
kernelMinor="${kernelVersion#$kernelMajor.}"
kernelMinor="${kernelMinor%%.*}"
is_set() {
zgrep "CONFIG_$1=[y|m]" "$CONFIG" > /dev/null
}
is_set_in_kernel() {
zgrep "CONFIG_$1=y" "$CONFIG" > /dev/null
}
is_set_as_module() {
zgrep "CONFIG_$1=m" "$CONFIG" > /dev/null
}
color() {
local codes=()
if [ "$1" = ‘bold‘ ]; then
codes=("${codes[@]}" ‘1‘)
shift
fi
if [ "$#" -gt 0 ]; then
local code=
case "$1" in
# see https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
black) code=30 ;;
red) code=31 ;;
green) code=32 ;;
yellow) code=33 ;;
blue) code=34 ;;
magenta) code=35 ;;
cyan) code=36 ;;
white) code=37 ;;
esac
if [ "$code" ]; then
codes=("${codes[@]}" "$code")
fi
fi
local IFS=‘;‘
echo -en ‘\033[‘"${codes[*]}"‘m‘
}
wrap_color() {
text="$1"
shift
color "$@"
echo -n "$text"
color reset
echo
}
wrap_good() {
echo "$(wrap_color "$1" white): $(wrap_color "$2" green)"
}
wrap_bad() {
echo "$(wrap_color "$1" bold): $(wrap_color "$2" bold red)"
}
wrap_warning() {
wrap_color >&2 "$*" red
}
check_flag() {
if is_set_in_kernel "$1"; then
wrap_good "CONFIG_$1" ‘enabled‘
elif is_set_as_module "$1"; then
wrap_good "CONFIG_$1" ‘enabled (as module)‘
else
wrap_bad "CONFIG_$1" ‘missing‘
EXITCODE=1
fi
}
check_flags() {
for flag in "$@"; do
echo -n "- "
check_flag "$flag"
done
}
check_command() {
if command -v "$1" > /dev/null 2>&1; then
wrap_good "$1 command" ‘available‘
else
wrap_bad "$1 command" ‘missing‘
EXITCODE=1
fi
}
check_device() {
if [ -c "$1" ]; then
wrap_good "$1" ‘present‘
else
wrap_bad "$1" ‘missing‘
EXITCODE=1
fi
}
check_distro_userns() {
source /etc/os-release 2> /dev/null || /bin/true
if [[ "${ID}" =~ ^(centos|rhel)$ && "${VERSION_ID}" =~ ^7 ]]; then
# this is a CentOS7 or RHEL7 system
grep -q "user_namespace.enable=1" /proc/cmdline || {
# no user namespace support enabled
wrap_bad " (RHEL7/CentOS7" "User namespaces disabled; add ‘user_namespace.enable=1‘ to boot command line)"
EXITCODE=1
}
fi
}
if [ ! -e "$CONFIG" ]; then
wrap_warning "warning: $CONFIG does not exist, searching other paths for kernel config ..."
for tryConfig in "${possibleConfigs[@]}"; do
if [ -e "$tryConfig" ]; then
CONFIG="$tryConfig"
break
fi
done
if [ ! -e "$CONFIG" ]; then
wrap_warning "error: cannot find kernel config"
wrap_warning " try running this script again, specifying the kernel config:"
wrap_warning " CONFIG=/path/to/kernel/.config $0 or $0 /path/to/kernel/.config"
exit 1
fi
fi
wrap_color "info: reading kernel config from $CONFIG ..." white
echo
echo ‘Generally Necessary:‘
echo -n ‘- ‘
cgroupSubsystemDir="$(awk ‘/[, ](cpu|cpuacct|cpuset|devices|freezer|memory)[, ]/ && $3 == "cgroup" { print $2 }‘ /proc/mounts | head -n1)"
cgroupDir="$(dirname "$cgroupSubsystemDir")"
if [ -d "$cgroupDir/cpu" ] || [ -d "$cgroupDir/cpuacct" ] || [ -d "$cgroupDir/cpuset" ] || [ -d "$cgroupDir/devices" ] || [ -d "$cgroupDir/freezer" ] || [ -d "$cgroupDir/memory" ]; then
echo "$(wrap_good ‘cgroup hierarchy‘ ‘properly mounted‘) [$cgroupDir]"
else
if [ "$cgroupSubsystemDir" ]; then
echo "$(wrap_bad ‘cgroup hierarchy‘ ‘single mountpoint!‘) [$cgroupSubsystemDir]"
else
wrap_bad ‘cgroup hierarchy‘ ‘nonexistent??‘
fi
EXITCODE=1
echo " $(wrap_color ‘(see https://github.com/tianon/cgroupfs-mount)‘ yellow)"
fi
if [ "$(cat /sys/module/apparmor/parameters/enabled 2> /dev/null)" = ‘Y‘ ]; then
echo -n ‘- ‘
if command -v apparmor_parser &> /dev/null; then
wrap_good ‘apparmor‘ ‘enabled and tools installed‘
else
wrap_bad ‘apparmor‘ ‘enabled, but apparmor_parser missing‘
echo -n ‘ ‘
if command -v apt-get &> /dev/null; then
wrap_color ‘(use "apt-get install apparmor" to fix this)‘
elif command -v yum &> /dev/null; then
wrap_color ‘(your best bet is "yum install apparmor-parser")‘
else
wrap_color ‘(look for an "apparmor" package for your distribution)‘
fi
EXITCODE=1
fi
fi
flags=(
NAMESPACES {NET,PID,IPC,UTS}_NS
CGROUPS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG
KEYS
VETH BRIDGE BRIDGE_NETFILTER
NF_NAT_IPV4 IP_NF_FILTER IP_NF_TARGET_MASQUERADE
NETFILTER_XT_MATCH_{ADDRTYPE,CONNTRACK,IPVS}
IP_NF_NAT NF_NAT NF_NAT_NEEDED
# required for bind-mounting /dev/mqueue into containers
POSIX_MQUEUE
)
check_flags "${flags[@]}"
if [ "$kernelMajor" -lt 4 ] || ([ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -lt 8 ]); then
check_flags DEVPTS_MULTIPLE_INSTANCES
fi
echo
echo ‘Optional Features:‘
{
check_flags USER_NS
check_distro_userns
}
{
check_flags SECCOMP
}
{
check_flags CGROUP_PIDS
}
{
CODE=${EXITCODE}
check_flags MEMCG_SWAP MEMCG_SWAP_ENABLED
if [ -e /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes ]; then
echo " $(wrap_color ‘(cgroup swap accounting is currently enabled)‘ bold black)"
EXITCODE=${CODE}
elif is_set MEMCG_SWAP && ! is_set MEMCG_SWAP_ENABLED; then
echo " $(wrap_color ‘(cgroup swap accounting is currently not enabled, you can enable it by setting boot option "swapaccount=1")‘ bold black)"
fi
}
{
if is_set LEGACY_VSYSCALL_NATIVE; then
echo -n "- "
wrap_bad "CONFIG_LEGACY_VSYSCALL_NATIVE" ‘enabled‘
echo " $(wrap_color ‘(dangerous, provides an ASLR-bypassing target with usable ROP gadgets.)‘ bold black)"
elif is_set LEGACY_VSYSCALL_EMULATE; then
echo -n "- "
wrap_good "CONFIG_LEGACY_VSYSCALL_EMULATE" ‘enabled‘
elif is_set LEGACY_VSYSCALL_NONE; then
echo -n "- "
wrap_bad "CONFIG_LEGACY_VSYSCALL_NONE" ‘enabled‘
echo " $(wrap_color ‘(containers using eglibc <= 2.13 will not work. Switch to‘ bold black)"
echo " $(wrap_color ‘ "CONFIG_VSYSCALL_[NATIVE|EMULATE]" or use "vsyscall=[native|emulate]"‘ bold black)"
echo " $(wrap_color ‘ on kernel command line. Note that this will disable ASLR for the,‘ bold black)"
echo " $(wrap_color ‘ VDSO which may assist in exploiting security vulnerabilities.)‘ bold black)"
# else Older kernels (prior to 3dc33bd30f3e, released in v4.40-rc1) do
# not have these LEGACY_VSYSCALL options and are effectively
# LEGACY_VSYSCALL_EMULATE. Even older kernels are presumably
# effectively LEGACY_VSYSCALL_NATIVE.
fi
}
if [ "$kernelMajor" -lt 4 ] || ([ "$kernelMajor" -eq 4 ] && [ "$kernelMinor" -le 5 ]); then
check_flags MEMCG_KMEM
fi
if [ "$kernelMajor" -lt 3 ] || ([ "$kernelMajor" -eq 3 ] && [ "$kernelMinor" -le 18 ]); then
check_flags RESOURCE_COUNTERS
fi
if [ "$kernelMajor" -lt 3 ] || ([ "$kernelMajor" -eq 3 ] && [ "$kernelMinor" -le 13 ]); then
netprio=NETPRIO_CGROUP
else
netprio=CGROUP_NET_PRIO
fi
flags=(
BLK_CGROUP BLK_DEV_THROTTLING IOSCHED_CFQ CFQ_GROUP_IOSCHED
CGROUP_PERF
CGROUP_HUGETLB
NET_CLS_CGROUP $netprio
CFS_BANDWIDTH FAIR_GROUP_SCHED RT_GROUP_SCHED
IP_NF_TARGET_REDIRECT
IP_VS
IP_VS_NFCT
IP_VS_PROTO_TCP
IP_VS_PROTO_UDP
IP_VS_RR
)
check_flags "${flags[@]}"
if ! is_set EXT4_USE_FOR_EXT2; then
check_flags EXT3_FS EXT3_FS_XATTR EXT3_FS_POSIX_ACL EXT3_FS_SECURITY
if ! is_set EXT3_FS || ! is_set EXT3_FS_XATTR || ! is_set EXT3_FS_POSIX_ACL || ! is_set EXT3_FS_SECURITY; then
echo " $(wrap_color ‘(enable these ext3 configs if you are using ext3 as backing filesystem)‘ bold black)"
fi
fi
check_flags EXT4_FS EXT4_FS_POSIX_ACL EXT4_FS_SECURITY
if ! is_set EXT4_FS || ! is_set EXT4_FS_POSIX_ACL || ! is_set EXT4_FS_SECURITY; then
if is_set EXT4_USE_FOR_EXT2; then
echo " $(wrap_color ‘enable these ext4 configs if you are using ext3 or ext4 as backing filesystem‘ bold black)"
else
echo " $(wrap_color ‘enable these ext4 configs if you are using ext4 as backing filesystem‘ bold black)"
fi
fi
echo ‘- Network Drivers:‘
echo " - \"$(wrap_color ‘overlay‘ blue)\":"
check_flags VXLAN BRIDGE_VLAN_FILTERING | sed ‘s/^/ /‘
echo ‘ Optional (for encrypted networks):‘
check_flags CRYPTO CRYPTO_AEAD CRYPTO_GCM CRYPTO_SEQIV CRYPTO_GHASH XFRM XFRM_USER XFRM_ALGO INET_ESP INET_XFRM_MODE_TRANSPORT | sed ‘s/^/ /‘
echo " - \"$(wrap_color ‘ipvlan‘ blue)\":"
check_flags IPVLAN | sed ‘s/^/ /‘
echo " - \"$(wrap_color ‘macvlan‘ blue)\":"
check_flags MACVLAN DUMMY | sed ‘s/^/ /‘
echo " - \"$(wrap_color ‘ftp,tftp client in container‘ blue)\":"
check_flags NF_NAT_FTP NF_CONNTRACK_FTP NF_NAT_TFTP NF_CONNTRACK_TFTP | sed ‘s/^/ /‘
# only fail if no storage drivers available
CODE=${EXITCODE}
EXITCODE=0
STORAGE=1
echo ‘- Storage Drivers:‘
echo " - \"$(wrap_color ‘aufs‘ blue)\":"
check_flags AUFS_FS | sed ‘s/^/ /‘
if ! is_set AUFS_FS && grep -q aufs /proc/filesystems; then
echo " $(wrap_color ‘(note that some kernels include AUFS patches but not the AUFS_FS flag)‘ bold black)"
fi
[ "$EXITCODE" = 0 ] && STORAGE=0
EXITCODE=0
echo " - \"$(wrap_color ‘btrfs‘ blue)\":"
check_flags BTRFS_FS | sed ‘s/^/ /‘
check_flags BTRFS_FS_POSIX_ACL | sed ‘s/^/ /‘
[ "$EXITCODE" = 0 ] && STORAGE=0
EXITCODE=0
echo " - \"$(wrap_color ‘devicemapper‘ blue)\":"
check_flags BLK_DEV_DM DM_THIN_PROVISIONING | sed ‘s/^/ /‘
[ "$EXITCODE" = 0 ] && STORAGE=0
EXITCODE=0
echo " - \"$(wrap_color ‘overlay‘ blue)\":"
check_flags OVERLAY_FS | sed ‘s/^/ /‘
[ "$EXITCODE" = 0 ] && STORAGE=0
EXITCODE=0
echo " - \"$(wrap_color ‘zfs‘ blue)\":"
echo -n " - "
check_device /dev/zfs
echo -n " - "
check_command zfs
echo -n " - "
check_command zpool
[ "$EXITCODE" = 0 ] && STORAGE=0
EXITCODE=0
EXITCODE=$CODE
[ "$STORAGE" = 1 ] && EXITCODE=1
echo
check_limit_over() {
if [ "$(cat "$1")" -le "$2" ]; then
wrap_bad "- $1" "$(cat "$1")"
wrap_color " This should be set to at least $2, for example set: sysctl -w kernel/keys/root_maxkeys=1000000" bold black
EXITCODE=1
else
wrap_good "- $1" "$(cat "$1")"
fi
}
echo ‘Limits:‘
check_limit_over /proc/sys/kernel/keys/root_maxkeys 10000
echo
exit $EXITCODE
执行脚本检查：
发现cgroup不存在
出现挂载/sys/fs/cgroup
sudo mount -o remount,ro /sys/fs/cgroup
再次检查，cgroup正常
3、重新启动容器
报如下错误
ESTABLISHEDError response from daemon: Cannot restart container rsnmp_v4: cgroups: cannot find cgroup mount destination: unknown
这个错说白了还是由于cgroup挂载的问题，默认的挂载为
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
1> 有可能上述的这个本身就没有挂载，这个时候可以使用如下进行操作：
sudo mkdir /sys/fs/cgroup/systemd
sudo mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd
或者使用docker官方提供的挂载脚本
https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
具体的代码如下：
#!/bin/sh
# Copyright 2011 Canonical, Inc
# 2014 Tianon Gravi
# Author: Serge Hallyn <serge.hallyn@canonical.com>
# Tianon Gravi <tianon@debian.org>
set -e
# for simplicity this script provides no flexibility
# if cgroup is mounted by fstab, don‘t run
# don‘t get too smart - bail on any uncommented entry with ‘cgroup‘ in it
if grep -v ‘^#‘ /etc/fstab | grep -q cgroup; then
echo ‘cgroups mounted from fstab, not mounting /sys/fs/cgroup‘
exit 0
fi
# kernel provides cgroups?
if [ ! -e /proc/cgroups ]; then
exit 0
fi
# if we don‘t even have the directory we need, something else must be wrong
if [ ! -d /sys/fs/cgroup ]; then
exit 0
fi
# mount /sys/fs/cgroup if not already done
if ! mountpoint -q /sys/fs/cgroup; then
mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
fi
cd /sys/fs/cgroup
# get/mount list of enabled cgroup controllers
for sys in $(awk ‘!/^#/ { if ($4 == 1) print $1 }‘ /proc/cgroups); do
mkdir -p $sys
if ! mountpoint -q $sys; then
if ! mount -n -t cgroup -o $sys cgroup $sys; then
rmdir $sys || true
fi
fi
done
# example /proc/cgroups:
# #subsys_name hierarchy num_cgroups enabled
# cpuset 2 3 1
# cpu 3 3 1
# cpuacct 4 3 1
# memory 5 3 0
# devices 6 3 1
# freezer 7 3 1
# blkio 8 3 1
# enable cgroups memory hierarchy, like systemd does (and lxc/docker desires)
# https://github.com/systemd/systemd/blob/v245/src/core/cgroup.c#L2983
# https://bugs.debian.org/940713
if [ -e /sys/fs/cgroup/memory/memory.use_hierarchy ]; then
echo 1 > /sys/fs/cgroup/memory/memory.use_hierarchy
fi
exit 0
2> 如果已经被挂载，目前还没有好的办法，只能重启服务器了。
借鉴：
https://forums.cnrancher.com/article/38
https://blog.csdn.net/ren7wei/article/details/80855779
http://blog.winkidney.com/index.php/archives/10/
标签：cgroups inf cap running map gadget pua wrong searching
原文地址：https://www.cnblogs.com/yjt1993/p/13223897.html
