Related
The program is executed on an embedded Linux (Kernel version: 3.4.0). Before calling execve(), setgid() and setuid() are also called to switch to another user (tstuser). The user exists, the necessary entries are found in /etc/[passwd && group && shadow]. The command su tstuser (by root) also simply reports "Killed". Resource limits already checked with getrlimit() previously, all seems to be OK. No error log (not a single line) found in dmesg && /var/log/messages.
What can be the cause? How can I get the kernel to be more verbose? Any idea or further places to investigate?
Here is the complete code:
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <unistd.h>
#include <pwd.h>
int main( int argc, char *argv[] ) {
char *newargv[] = { NULL, NULL };
char *newenviron[] = { NULL };
if ( argc != 2 ) {
fprintf( stderr, "Usage: %s <file-to-exec>\n", argv[0] );
exit( EXIT_FAILURE );
}
newargv[0] = argv[1];
struct passwd * pw;
if ( ( pw = getpwnam( "tstuser" ) ) == NULL ) {
fprintf( stderr, "ERROR: the specified user '%s' does not exist\n", "tstuser" );
exit( EXIT_FAILURE );
}
else {
if ( setgid( pw->pw_gid ) != 0 ) {
fprintf( stderr, "ERROR: setgid() to %d failed\n", pw->pw_gid );
exit( EXIT_FAILURE );
}
else if ( setuid( pw->pw_uid ) != 0 ) {
fprintf( stderr, "ERROR: setuid() to %d failed\n", pw->pw_uid );
exit( EXIT_FAILURE );
}
}
execve( argv[1], newargv, newenviron );
perror( "execve" );
exit( EXIT_FAILURE );
}
The complete strace output:
~# strace ./execve /bin/ls
execve("./execve", ["./execve", "/bin/ls"], [/* 13 vars */]) = 0
brk(0) = 0x145d000
uname({sys="Linux", node="sagVED_A", ...}) = 0
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=4666, ...}) = 0
mmap2(NULL, 4666, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb6fb5000
close(3) = 0
open("/lib/tls/v5l/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v5l", 0xbee6b468) = -1 ENOENT (No such file or directory)
open("/lib/tls/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls", 0xbee6b468) = -1 ENOENT (No such file or directory)
open("/lib/v5l/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v5l", 0xbee6b468) = -1 ENOENT (No such file or directory)
open("/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\354\225\262F4\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1245200, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6fb4000
mmap2(0x46b10000, 1284496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x46b10000
mprotect(0x46c3c000, 32768, PROT_NONE) = 0
mmap2(0x46c44000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12c) = 0x46c44000
mmap2(0x46c48000, 6544, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x46c48000
close(3) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6fb3000
set_tls(0xb6fb34c0, 0xb6fb3b98, 0x46b07058, 0xb6fb34c0, 0x46b07058) = 0
mprotect(0x46c44000, 8192, PROT_READ) = 0
mprotect(0x46b06000, 4096, PROT_READ) = 0
munmap(0xb6fb5000, 4666) = 0
brk(0) = 0x145d000
brk(0x147e000) = 0x147e000
socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_FILE, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_FILE, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
fcntl64(3, F_GETFD) = 0x1 (flags FD_CLOEXEC)
fstat64(3, {st_mode=S_IFREG|0644, st_size=465, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6fb6000
read(3, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 465
read(3, "", 4096) = 0
close(3) = 0
munmap(0xb6fb6000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=4666, ...}) = 0
mmap2(NULL, 4666, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb6fb5000
close(3) = 0
open("/lib/libnss_compat.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\340\r\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=30524, ...}) = 0
mmap2(NULL, 62060, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6fa3000
mprotect(0xb6fa9000, 32768, PROT_NONE) = 0
mmap2(0xb6fb1000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb6fb1000
close(3) = 0
open("/lib/libnsl.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0p/\361F4\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=73264, ...}) = 0
mmap2(0x46f10000, 112464, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x46f10000
mprotect(0x46f21000, 28672, PROT_NONE) = 0
mmap2(0x46f28000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x10) = 0x46f28000
mmap2(0x46f2a000, 5968, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x46f2a000
close(3) = 0
mprotect(0x46f28000, 4096, PROT_READ) = 0
mprotect(0xb6fb1000, 4096, PROT_READ) = 0
munmap(0xb6fb5000, 4666) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=4666, ...}) = 0
mmap2(NULL, 4666, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb6fb5000
close(3) = 0
open("/lib/libnss_nis.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
open("/usr/lib/tls/v5l/libnss_nis.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/tls/v5l", 0xbee6b340) = -1 ENOENT (No such file or directory)
open("/usr/lib/tls/libnss_nis.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/tls", 0xbee6b340) = -1 ENOENT (No such file or directory)
open("/usr/lib/v5l/libnss_nis.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/usr/lib/v5l", 0xbee6b340) = -1 ENOENT (No such file or directory)
open("/usr/lib/libnss_nis.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/usr/lib", {st_mode=S_IFDIR|0755, st_size=8456, ...}) = 0
munmap(0xb6fb5000, 4666) = 0
open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
fcntl64(3, F_GETFD) = 0x1 (flags FD_CLOEXEC)
_llseek(3, 0, [0], SEEK_CUR) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=848, ...}) = 0
mmap2(NULL, 848, PROT_READ, MAP_SHARED, 3, 0) = 0xb6fb6000
_llseek(3, 848, [848], SEEK_SET) = 0
munmap(0xb6fb6000, 848) = 0
close(3) = 0
setgid32(1001) = 0
setuid32(1001) = 0
execve("/bin/ls", ["/bin/ls"], [/* 0 vars */] <unfinished ...>
+++ killed by SIGKILL +++
Killed
The problem was: the value of the kernel config paramter CONFIG_DEFAULT_MMAP_MIN_ADDR was set to 65536.
When page size is 64k, 0x8000(32k) and CONFIG_DEFAULT_MMAP_MIN_ADDR are both at the 0th page, which is causing non-root user cannot run the binary built by the toolchain, e.g. busybox (source: http://mkl-note.blogspot.hu/2011/06/configdefaultmmapminaddr.html).
In cap_file_mmap() (see https://elixir.bootlin.com/linux/v3.4/source/security/commoncap.c#L960) cap_capable() rejected the memory mapping requested by load_elf_binary() at https://elixir.bootlin.com/linux/v3.4/source/fs/binfmt_elf.c#L809 (through a chain of other function calls).
Setting CONFIG_DEFAULT_MMAP_MIN_ADDR to 32768 or smaller solved the issue.
I am currently implementing an elf loader to emulate binaries with the unicorn engine. To avoid implementing my own dynamic linker, I load the ld-linux-x86-64.so.2 (into the unicorn engine) and execute it with the path to my dynamic linked executable. Now here comes my problem:
I load the dynamic linker binary (/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2) into the memory and launch it with the path to my executable (basically printf("Hello World")). After the loader loads the executable and the libc the following gets printed to the stdout:
./executable: ./executable: no version information available (required by ./executable)
./executable: relocation error: ./executable: symbol __libc_start_main, version GLIBC_2.2.5 not defined in file libc.so.6 with link time reference
I copied the binaries (ld and libc) from the system I compiled the hello_world program. It works when I execute the binary there:
$ /lib/x86_64-linux-gnu/ld-2.21.so ./executable
Hello World
Any idea why this should not work? Any idea where I should look for my error? (Note that I am using my custom loader)
Here is the 'strace' of the loader running in the unicorn-engine that is hopefully correctly executed (I need to simulate the kernel).
sys_brk(0x0)
sys_open(./executable)
sys_read(3, 0x7fff6c844868, 832)
sys_fstat(3, 0x7fff6c844718)
sys_getcwd(0x8888888800225738, 128)
sys_mmap2(0x400000, 4096, 5, 2066, 3, 0)
sys_mmap2(0x600000, 8192, 3, 2066, 3, 0)
sys_close(3)
sys_uname(0x7fff6c844a22)
sys_access(/etc/ld.so.nohwcap, 0)
sys_mmap2(0x0, 8192, 3, 34, -1, 0)
sys_access(/etc/ld.so.preload, 4)
sys_open(/etc/ld.so.cache)
sys_open(/lib/x86_64-linux-gnu/tls/x86_64/libc.so.6)
sys_stat(/lib/x86_64-linux-gnu/tls/x86_64, 140735013995640)
sys_open(/lib/x86_64-linux-gnu/tls/libc.so.6)
sys_stat(/lib/x86_64-linux-gnu/tls, 140735013995640)
sys_open(/lib/x86_64-linux-gnu/x86_64/libc.so.6)
sys_stat(/lib/x86_64-linux-gnu/x86_64, 140735013995640)
sys_open(/lib/x86_64-linux-gnu/libc.so.6)
sys_read(4, 0x7fff6c8445c8, 832)
sys_fstat(4, 0x7fff6c844478)
sys_close(4)
sys_writev(2, 7fff6c844538, 6)
=> ./executable: ./executable: no version information available (required by ./executable)
sys_mmap2(0x0, 4096, 3, 34, -1, 0)
sys_arch_prctl(4098, 0x6555555500001fc0)
sys_mprotect(0x600000, 4096, 1)
sys_writev(2, 7fff6c844220, 10)
=> ./executable: relocation error: ./executable: symbol __libc_start_main, version GLIBC_2.2.5 not defined in file libc.so.6 with link time reference
sys_exit(127)
And the strace on the machine I compiled the binaries:
$ strace /lib/x86_64-linux-gnu/ld-2.21.so ./dynamic_x86_64_print_args
execve("/lib/x86_64-linux-gnu/ld-2.21.so", ["/lib/x86_64-linux-gnu/ld-2.21.so", "./dynamic_x86_64_print_args"], [/* 23 vars */]) = 0
brk(0) = 0x5636dd9f3000
open("./dynamic_x86_64_print_args", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\2\0>\0\1\0\0\0#\4#\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=8528, ...}) = 0
getcwd("/vagrant/tdb/static", 128) = 20
mmap(0x400000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0x400000
mmap(0x600000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0x600000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc3a45e1000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
// ...
// SNIP 100 tries to load the libc.so.6 that is not there
// ...
open("/home/vagrant/villoc/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/home/vagrant/villoc", {st_mode=S_IFDIR|0775, st_size=4096, ...}) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=55962, ...}) = 0
mmap(NULL, 55962, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc3a45d3000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\v\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1869392, ...}) = 0
mmap(NULL, 3972864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc3a4209000
mprotect(0x7fc3a43c9000, 2097152, PROT_NONE) = 0
mmap(0x7fc3a45c9000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7fc3a45c9000
mmap(0x7fc3a45cf000, 16128, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc3a45cf000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc3a4208000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc3a4207000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc3a4206000
arch_prctl(ARCH_SET_FS, 0x7fc3a4207700) = 0
mprotect(0x7fc3a45c9000, 16384, PROT_READ) = 0
mprotect(0x600000, 4096, PROT_READ) = 0
mprotect(0x5636dba3b000, 4096, PROT_READ) = 0
munmap(0x7fc3a45d3000, 55962) = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc3a45e0000
write(1, "arg 0 ./dynamic_x86_64_print_arg"..., 34arg 0 ./dynamic_x86_64_print_args
) = 34
exit_group(0) = ?
+++ exited with 0 +++
Looking at your syscalls...
sys_mmap2(0x400000, 4096, 5, 2066, 3, 0)
sys_mmap2(0x600000, 8192, 3, 2066, 3, 0)
You appear to have MAP_FIXED bit set.
MAP_FIXED is a dangerous tool - it will quite happily overwrite whatever else is at that address. Chances are, your loader is using those addresses. I know that /lib64/ld-linux-x86-64.so.2 uses MAP_FIXED - but...
> readelf --segments /lib64/ld-linux-x86-64.so.2
Elf file type is DYN (Shared object file)
Entry point 0x1140
There are 7 program headers, starting at offset 64
Program Headers:
Type Offset VirtAddr PhysAddr
FileSiz MemSiz Flags Align
LOAD 0x0000000000000000 0x0000000000000000 0x0000000000000000
0x0000000000021960 0x0000000000021960 R E 200000
LOAD 0x0000000000021b40 0x0000000000221b40 0x0000000000221b40
0x0000000000001438 0x0000000000001610 RW 200000
DYNAMIC 0x0000000000021e00 0x0000000000221e00 0x0000000000221e00
0x0000000000000190 0x0000000000000190 RW 8
NOTE 0x00000000000001c8 0x00000000000001c8 0x00000000000001c8
0x0000000000000024 0x0000000000000024 R 4
GNU_EH_FRAME 0x000000000001eda4 0x000000000001eda4 0x000000000001eda4
0x0000000000000674 0x0000000000000674 R 4
GNU_STACK 0x0000000000000000 0x0000000000000000 0x0000000000000000
0x0000000000000000 0x0000000000000000 RW 10
GNU_RELRO 0x0000000000021b40 0x0000000000221b40 0x0000000000221b40
0x00000000000004c0 0x00000000000004c0 R 1
Section to Segment mapping:
Segment Sections...
00 .note.gnu.build-id .hash .gnu.hash .dynsym .dynstr .gnu.version .gnu.version_d .rela.dyn .rela.plt .plt .text .rodata .stapsdt.base .eh_frame_hdr .eh_frame
01 .data.rel.ro .dynamic .got .data .bss
02 .dynamic
03 .note.gnu.build-id
04 .eh_frame_hdr
05
06 .data.rel.ro .dynamic .got
...it doesn't appear to be using the addresses it is using MAP_FIXED for. It's a clever beast.
> strace /lib64/ld-linux-x86-64.so.2 `which readelf` |& grep mmap
mmap(0x400000, 503808, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0x400000
mmap(0x67a000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7a000) = 0x67a000
mmap(0x67d000, 10248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x67d000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0af3a38000
mmap(NULL, 313165, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0af39eb000
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0af3614000
mmap(0x7f0af3816000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f0af3816000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0af39ea000
mmap(NULL, 3985920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0af3246000
mmap(0x7f0af3609000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c3000) = 0x7f0af3609000
mmap(0x7f0af360f000, 16896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0af360f000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0af39e8000
mmap(NULL, 106172832, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0aecd04000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0af3a37000
If I use MAP_FIXED in a Bad Way™ on my own system, I get a somewhat similar message:
./myloader: symbol lookup error: ./myloader: undefined symbol:
But given we're just smashing memory up here, any number of things could happen.
I am trying load /etc/cgconfig.conf using cgconfigparser to configure cgroup properties of jstorm.
My OS is ubuntu 14.04, username is 'whaty' and I have installed libcgroup1, cgroup-bin, cgroup-lite. There is no cgconfig.conf in /etc. so I create it manually.
Here is my cgconfig.conf
mount {
cpu = /sys/fs/cgroup/cpu;
cpuset = /sys/fs/cgroup/cpuset;
}
group jstorm {
perm {
task {
uid = 1000;
gid = 1000;
}
admin {
uid = 1000;
gid = 1000;
}
}
cpuset {
cpuset.cpus = 0;
cpuset.mems = 0;
}
}
I try to load this config using:
sudo cgconfigparser -l /etc/cgconfig.conf
The result shows:
cgconfigparser; error loading /etc/cgconfig.conf: Cgroup mounting failed
Here is my /proc/cgroups:
#subsys_name hierarchy num_cgroups enabled
cpuset 12 5 1
cpu 3 5 1
cpuacct 0 1 1
memory 0 1 1
devices 0 1 1
freezer 0 1 1
blkio 0 1 1
perf_event 0 1 1
hugetlb 0 1 1
Here is my mount of cgroup:
whaty#whaty:/sys/fs/cgroup$ ls
blkio cpu cpuacct cpuset devices freezer hugetlb memory perf_event systemd
Here is the strace of cgconfigparsser:
execve("/usr/sbin/cgconfigparser", ["cgconfigparser", "-l", "/etc/cgconfig.conf"], [/* 24 vars */]) = 0
brk(0) = 0x1421000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6499e62000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=98553, ...}) = 0
mmap(NULL, 98553, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f6499e49000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libcgroup.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2608\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=76872, ...}) = 0
mmap(NULL, 4640896, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f64997d4000
mprotect(0x7f64997e6000, 2093056, PROT_NONE) = 0
mmap(0x7f64999e5000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11000) = 0x7f64999e5000
mmap(0x7f64999e7000, 2465920, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f64999e7000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\37\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1840928, ...}) = 0
mmap(NULL, 3949248, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f649940f000
mprotect(0x7f64995ca000, 2093056, PROT_NONE) = 0
mmap(0x7f64997c9000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ba000) = 0x7f64997c9000
mmap(0x7f64997cf000, 17088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f64997cf000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0po\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141574, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6499e48000
mmap(NULL, 2217264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f64991f1000
mprotect(0x7f649920a000, 2093056, PROT_NONE) = 0
mmap(0x7f6499409000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x7f6499409000
mmap(0x7f649940b000, 13616, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f649940b000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6499e47000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6499e45000
arch_prctl(ARCH_SET_FS, 0x7f6499e45b80) = 0
mprotect(0x7f64997c9000, 16384, PROT_READ) = 0
mprotect(0x7f6499409000, 4096, PROT_READ) = 0
mprotect(0x7f64999e5000, 4096, PROT_READ) = 0
mprotect(0x602000, 4096, PROT_READ) = 0
mprotect(0x7f6499e64000, 4096, PROT_READ) = 0
munmap(0x7f6499e49000, 98553) = 0
set_tid_address(0x7f6499e45e50) = 14211
set_robust_list(0x7f6499e45e60, 24) = 0
futex(0x7ffde9414310, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7f6499e45b80) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7f64991f79f0, [], SA_RESTORER|SA_SIGINFO, 0x7f6499201340}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f64991f7a80, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f6499201340}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(0) = 0x1421000
brk(0x1442000) = 0x1442000
open("/etc/cgconfig.conf", O_RDONLY|O_CLOEXEC) = 3
mmap(NULL, 319488, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6499df7000
ioctl(3, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, 0x7ffde9413840) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(3, {st_mode=S_IFREG|0644, st_size=411, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f6499e61000
read(3, "mount {\n cpu = /sys/fs/cgr"..., 8192) = 411
read(3, "", 4096) = 0
read(3, "", 8192) = 0
ioctl(3, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, 0x7ffde9413840) = -1 ENOTTY (Inappropriate ioctl for device)
close(3) = 0
munmap(0x7f6499e61000, 4096) = 0
stat("/sys/fs/cgroup/cpu", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
mount("cgroup", "/sys/fs/cgroup/cpu", "cgroup", 0, "cpu") = -1 EBUSY (Device or resource busy)
munmap(0x7f6499df7000, 319488) = 0
write(2, "cgconfigparser; error loading /e"..., 73cgconfigparser; error loading /etc/cgconfig.conf: Cgroup mounting failed
) = 73
exit_group(50021) = ?
+++ exited with 101 +++
It seems that certain resource is occupied and busy. What's wrong with my configuration?
I am trying to detach an Amazon EC2 Volume, but It get stuck some where, here is the strace output, I tried fuser -km /mnt/web but none of the option is working.
I also tried ls /mnt/, and deleted the entry from /etc/fstab but it still won't work.
Strace Output
user#server:~$ sudo strace umount -l /mnt/web
execve("/bin/umount", ["umount", "-l", "/mnt/web"], [/* 14 vars */]) = 0
brk(0) = 0xa11000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc411613000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=30545, ...}) = 0
mmap(NULL, 30545, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc41160b000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320^\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=155384, ...}) = 0
mmap(NULL, 2250504, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc4111cd000
mprotect(0x7fc4111ef000, 2097152, PROT_NONE) = 0
mmap(0x7fc4113ef000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7fc4113ef000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libmount.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260Z\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=154648, ...}) = 0
mmap(NULL, 2249736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc410fa7000
mprotect(0x7fc410fcc000, 2093056, PROT_NONE) = 0
mmap(0x7fc4111cb000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24000) = 0x7fc4111cb000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\37\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1845024, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc41160a000
mmap(NULL, 3953344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc410be1000
mprotect(0x7fc410d9c000, 2097152, PROT_NONE) = 0
mmap(0x7fc410f9c000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bb000) = 0x7fc410f9c000
mmap(0x7fc410fa2000, 17088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc410fa2000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\26\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18936, ...}) = 0
mmap(NULL, 2113968, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc4109dc000
mprotect(0x7fc4109e0000, 2093056, PROT_NONE) = 0
mmap(0x7fc410bdf000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fc410bdf000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0[\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=134296, ...}) = 0
mmap(NULL, 2238192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc4107b9000
mprotect(0x7fc4107d9000, 2093056, PROT_NONE) = 0
mmap(0x7fc4109d8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7fc4109d8000
mmap(0x7fc4109da000, 5872, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc4109da000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpcre.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\27\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=252032, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc411609000
mmap(NULL, 2347200, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc41057b000
mprotect(0x7fc4105b8000, 2093056, PROT_NONE) = 0
mmap(0x7fc4107b7000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3c000) = 0x7fc4107b7000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14664, ...}) = 0
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc410377000
mprotect(0x7fc41037a000, 2093056, PROT_NONE) = 0
mmap(0x7fc410579000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fc410579000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc411608000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc411606000
arch_prctl(ARCH_SET_FS, 0x7fc411606880) = 0
mprotect(0x7fc410f9c000, 16384, PROT_READ) = 0
mprotect(0x7fc410579000, 4096, PROT_READ) = 0
mprotect(0x7fc4107b7000, 4096, PROT_READ) = 0
mprotect(0x7fc4109d8000, 4096, PROT_READ) = 0
mprotect(0x7fc410bdf000, 4096, PROT_READ) = 0
mprotect(0x7fc4113ef000, 12288, PROT_READ) = 0
mprotect(0x7fc4111cb000, 4096, PROT_READ) = 0
mprotect(0x60f000, 4096, PROT_READ) = 0
mprotect(0x7fc411615000, 4096, PROT_READ) = 0
munmap(0x7fc41160b000, 30545) = 0
statfs("/sys/fs/selinux", 0x7fff570957f0) = -1 ENOENT (No such file or directory)
statfs("/selinux", 0x7fff570957f0) = -1 ENOENT (No such file or directory)
brk(0) = 0xa11000
brk(0xa32000) = 0xa32000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc411612000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tr"..., 1024) = 374
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7fc411612000, 4096) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1607664, ...}) = 0
mmap(NULL, 1607664, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc41147d000
close(3) = 0
umask(022) = 022
getuid() = 0
geteuid() = 0
readlink("/mnt", 0x7fff57093710, 4096) = -1 EINVAL (Invalid argument)
readlink("/mnt/web",
In one of my system, I observed below output.
root#base0-0-0-1-0-11-1:/root> uname -a
Linux base0-0-0-1-0-11-1 2.6.34.12-kairos-ksc-dec02-1800 #2 SMP Mon Dec 2 17:48:50 IST 2013 x86_64 x86_64 x86_64 GNU/Linux
root#base0-0-0-1-0-11-1:/root> getconf LONG_BIT
32
root#base0-0-0-1-0-11-1:/root>
The output from uname says the system is 64 bit while getconf says its 32 bit. Is there any reason for this mismatch?
I confirmed in my other systems where no such mismatch was found.
Please suggest. Thanks.
You can confirm which files getconf is picking up by running it through strace.
$ strace getconf LONG_BIT
Example
From my 64-bit Fedora 19 installation.
$ strace getconf LONG_BIT
execve("/usr/bin/getconf", ["getconf", "LONG_BIT"], [/* 92 vars */]) = 0
brk(0) = 0x1114000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8b865b9000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=190823, ...}) = 0
mmap(NULL, 190823, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8b8658a000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\34\242\213?\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2108632, ...}) = 0
mmap(0x3f8ba00000, 3932768, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3f8ba00000
mprotect(0x3f8bbb6000, 2097152, PROT_NONE) = 0
mmap(0x3f8bdb6000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x3f8bdb6000
mmap(0x3f8bdbc000, 16992, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3f8bdbc000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8b86589000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8b86587000
arch_prctl(ARCH_SET_FS, 0x7f8b86587740) = 0
mprotect(0x604000, 4096, PROT_READ) = 0
mprotect(0x3f8bdb6000, 16384, PROT_READ) = 0
mprotect(0x3f8b820000, 4096, PROT_READ) = 0
munmap(0x7f8b8658a000, 190823) = 0
brk(0) = 0x1114000
brk(0x1135000) = 0x1135000
brk(0) = 0x1135000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106055264, ...}) = 0
mmap(NULL, 106055264, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8b80062000
close(3) = 0
readlink("/usr/libexec/getconf/default", "POSIX_V6_LP64_OFF64", 21) = 19
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 4), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8b865b8000
write(1, "64\n", 364
) = 3
exit_group(0) = ?
+++ exited with 0 +++