I am trying to understand the basic of linux process memory layout and i got this program:
#include <stdio.h> // standard io
#include <stdlib.h> // C standard library
#include <pthread.h> // threading
#include <unistd.h> // unix standard library
#include <sys/types.h> // system types for linux
// getchar basically is like "read"
// it prompts the user for input
// in this case, the input is thrown away
// which makes similar to a "pause" continuation primitive
// but a pause that is resolved through user input, which we promptly throw away!
void * thread_func (void * arg) {
printf("Before malloc in thread 1\n");
getchar();
char * addr = (char *) malloc(1000);
printf("After malloc and before free in thread 1\n");
getchar();
free(addr);
printf("After free in thread 1\n");
getchar();
}
int main () {
char * addr;
printf("Welcome to per thread arena example::%d\n", getpid());
printf("Before malloc in the main thread\n");
getchar();
addr = (char *) malloc(1000);
printf("After malloc and before free in main thread\n");
getchar();
free(addr);
printf("After free in main thread\n");
getchar();
// pointer to the thread 1
pthread_t thread_1;
// pthread_* functions return 0 upon succeeding, and other numbers upon failing
int pthread_status;
pthread_status = pthread_create(&thread_1, NULL, thread_func, NULL);
if (pthread_status != 0) {
printf("Thread creation error\n");
return -1;
}
// returned status code from thread_1
void * thread_1_status;
pthread_status = pthread_join(thread_1, &thread_1_status);
if (pthread_status != 0) {
printf("Thread join error\n");
return -1;
}
return 0;
}
When I started the program, the content in /proc/<pid>/maps is:
00400000-00401000 r-xp 00000000 08:01 1323314 /home/oscp/xg/c/memory_layout/a.out
00600000-00601000 r--p 00000000 08:01 1323314 /home/oscp/xg/c/memory_layout/a.out
00601000-00602000 rw-p 00001000 08:01 1323314 /home/oscp/xg/c/memory_layout/a.out
7fcc372d7000-7fcc37491000 r-xp 00000000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37491000-7fcc37691000 ---p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37691000-7fcc37695000 r--p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37695000-7fcc37697000 rw-p 001be000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37697000-7fcc3769c000 rw-p 00000000 00:00 0
7fcc3769c000-7fcc376b5000 r-xp 00000000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc376b5000-7fcc378b4000 ---p 00019000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc378b4000-7fcc378b5000 r--p 00018000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc378b5000-7fcc378b6000 rw-p 00019000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc378b6000-7fcc378ba000 rw-p 00000000 00:00 0
7fcc378ba000-7fcc378dd000 r-xp 00000000 08:01 1053733 /lib/x86_64-linux-gnu/ld-2.19.so
7fcc37abe000-7fcc37ac1000 rw-p 00000000 00:00 0
7fcc37ad8000-7fcc37adc000 rw-p 00000000 00:00 0
7fcc37adc000-7fcc37add000 r--p 00022000 08:01 1053733 /lib/x86_64-linux-gnu/ld-2.19.so
7fcc37add000-7fcc37ade000 rw-p 00023000 08:01 1053733 /lib/x86_64-linux-gnu/ld-2.19.so
7fcc37ade000-7fcc37adf000 rw-p 00000000 00:00 0
7ffdc1cff000-7ffdc1d20000 rw-p 00000000 00:00 0 [stack]
7ffdc1dd8000-7ffdc1ddb000 r--p 00000000 00:00 0 [vvar]
7ffdc1ddb000-7ffdc1ddd000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
What are the purposes of these memory regions?
7fcc37491000-7fcc37691000 ---p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
...
7fcc37abe000-7fcc37ac1000 rw-p 00000000 00:00 0
7fcc37ad8000-7fcc37adc000 rw-p 00000000 00:00 0
Then I press enter a few times after running the program. After it prints "Before malloc in thread 1". The memory layout looks like below:
00400000-00401000 r-xp 00000000 08:01 1323314 /home/oscp/xg/c/memory_layout/a.out
00600000-00601000 r--p 00000000 08:01 1323314 /home/oscp/xg/c/memory_layout/a.out
00601000-00602000 rw-p 00001000 08:01 1323314 /home/oscp/xg/c/memory_layout/a.out
00632000-00653000 rw-p 00000000 00:00 0 [heap]
7fcc36ad6000-7fcc36ad7000 ---p 00000000 00:00 0
7fcc36ad7000-7fcc372d7000 rw-p 00000000 00:00 0
7fcc372d7000-7fcc37491000 r-xp 00000000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37491000-7fcc37691000 ---p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37691000-7fcc37695000 r--p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37695000-7fcc37697000 rw-p 001be000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37697000-7fcc3769c000 rw-p 00000000 00:00 0
7fcc3769c000-7fcc376b5000 r-xp 00000000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc376b5000-7fcc378b4000 ---p 00019000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc378b4000-7fcc378b5000 r--p 00018000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc378b5000-7fcc378b6000 rw-p 00019000 08:01 1053877 /lib/x86_64-linux-gnu/libpthread-2.19.so
7fcc378b6000-7fcc378ba000 rw-p 00000000 00:00 0
7fcc378ba000-7fcc378dd000 r-xp 00000000 08:01 1053733 /lib/x86_64-linux-gnu/ld-2.19.so
7fcc37abe000-7fcc37ac1000 rw-p 00000000 00:00 0
7fcc37ad8000-7fcc37adc000 rw-p 00000000 00:00 0
7fcc37adc000-7fcc37add000 r--p 00022000 08:01 1053733 /lib/x86_64-linux-gnu/ld-2.19.so
7fcc37add000-7fcc37ade000 rw-p 00023000 08:01 1053733 /lib/x86_64-linux-gnu/ld-2.19.so
7fcc37ade000-7fcc37adf000 rw-p 00000000 00:00 0
7ffdc1cff000-7ffdc1d20000 rw-p 00000000 00:00 0 [stack]
7ffdc1dd8000-7ffdc1ddb000 r--p 00000000 00:00 0 [vvar]
7ffdc1ddb000-7ffdc1ddd000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
What are the purposes of these two regions?
7fcc36ad6000-7fcc36ad7000 ---p 00000000 00:00 0
7fcc36ad7000-7fcc372d7000 rw-p 00000000 00:00 0
After it prints "After malloc and before free in thread 1", it creates another two regions below:
7fcc30000000-7fcc30021000 rw-p 00000000 00:00 0
7fcc30021000-7fcc34000000 ---p 00000000 00:00 0
What are the purposes of these two regions?
Your question covers many completely different things, so the answer will be long.
The first question is the meaning of
7fcc37491000-7fcc37691000 ---p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
in
7fcc372d7000-7fcc37491000 r-xp 00000000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37491000-7fcc37691000 ---p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37691000-7fcc37695000 r--p 001ba000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
7fcc37695000-7fcc37697000 rw-p 001be000 08:01 1053757 /lib/x86_64-linux-gnu/libc-2.19.so
This inaccessible memory region is a gap between adjacent ELF segments of the library (which is supposed to occupy the contiguous chunk of memory). The ---p protection mode forbids using this gap for occasional memory allocation. If you strace(1) the process when it loads the library, you may see something like this:
mmap(NULL, 1848896, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3</usr/lib/libc-2.28.so>, 0) = 0x7f9673d8f000
mprotect(0x7f9673db1000, 1671168, PROT_NONE) = 0
mmap(0x7f9673db1000, 1355776, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3</usr/lib/libc-2.28.so>, 0x22000) = 0x7f9673db1000
mmap(0x7f9673efc000, 311296, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3</usr/lib/libc-2.28.so>, 0x16d000) = 0x7f9673efc000
mmap(0x7f9673f49000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3</usr/lib/libc-2.28.so>, 0x1b9000) = 0x7f9673f49000
The first mmap() maps the first ELF segment into the memory but reserves the space for whole library. This is done to allow the kernel to choose the location for the library on its discretion. To protect any possible gaps between segments mprotect(..., PROT_NONE) is called; then all remaining segments are mapped into the memory using mmap() - this also changes the protection mode of the appropriate memory pages from ---p to whatever mode the segment requires. You may have some fun by taking a look at how it actually works. If you want to verify how this ---p gap is formed during the loading, you may also use readelf(1) with library's binary and do some hexadecimal math with segments' locations and alignments, collating the results with the output of strace.
The second question are the following anonymous mappings:
7fcc36ad6000-7fcc36ad7000 ---p 00000000 00:00 0
7fcc36ad7000-7fcc372d7000 rw-p 00000000 00:00 0
This looks like a thread stack for thread 1. The second mapping is the stack itself (372d7000 - 36ad7000 = 800000 = 8 MiB, which is a default stack size limit in many distros, which, in turn, is a default stack size for pthread), and the first one is a stack guard page. This page with mode ---p protects the stack from overflowing, and triggers the segfault when the overflow happens (because of a write to this write-protected page).
Note: in older Linux kernels, the thread stacks were annotated with [stack:TID] names in maps file, but this feature was removed, so I cannot guarantee that this mapping is really a thread stack (though it looks like). However, you may use strace to find the exact thread's stack location from child_stack argument of clone() syscall and compare with this mapping.
Going on. The third question is
7fcc30000000-7fcc30021000 rw-p 00000000 00:00 0
7fcc30021000-7fcc34000000 ---p 00000000 00:00 0
Well, this is what malloc() in thread 1 did to allocate the memory you have requested. In short, the whole region 7fcc30000000-7fcc34000000 is a heap, from which allocations are done. The rw-p interval 7fcc30000000-7fcc30021000, allocated from this heap, will grow as you will request more and more memory with malloc(). When this heap will deplete, new one will be requested using mmap().
As you probably noticed, I don't have an explanation for the following mappings in your question:
7fcc37abe000-7fcc37ac1000 rw-p 00000000 00:00 0
7fcc37ad8000-7fcc37adc000 rw-p 00000000 00:00 0
I can't recognize those guys quickly and not sure that these are ordinary allocations. Probably this needs separate investigation, as this topic is already too long.
I Need to create process state replication between two processes.
I am using a simple bash script that count to infinity.
I am running it on server 1 and server 2 and making the process on server 2 always paused and i need to copy the state (memory) from the first process to the second one (the replica).
I am very familar with procfs and I know that the memory pages contains the state in memory page as follows:
root#ubuntu:/proc/41932# cat maps
7f7254d85000-7f7254f40000 r-xp 00000000 00:27 30 /lib/x86_64-linux-gnu/libc-2.19.so
7f7254f40000-7f725513f000 ---p 001bb000 00:27 30 /lib/x86_64-linux-gnu/libc-2.19.so
7f725513f000-7f7255143000 r--p 001ba000 00:27 30 /lib/x86_64-linux-gnu/libc-2.19.so
7f7255143000-7f7255145000 rw-p 001be000 00:27 30 /lib/x86_64-linux-gnu/libc-2.19.so
7f7255145000-7f725514a000 rw-p 00000000 00:00 0
7f725514a000-7f725516d000 r-xp 00000000 00:27 27 /lib/x86_64-linux-gnu/ld-2.19.so
7f7255364000-7f7255367000 rw-p 00000000 00:00 0
7f725536a000-7f725536c000 rw-p 00000000 00:00 0
7f725536c000-7f725536d000 r--p 00022000 00:27 27 /lib/x86_64-linux-gnu/ld-2.19.so
7f725536d000-7f725536e000 rw-p 00023000 00:27 27 /lib/x86_64-linux-gnu/ld-2.19.so
7f725536e000-7f725536f000 rw-p 00000000 00:00 0
7f725536f000-7f725538b000 r-xp 00000000 00:27 22 /bin/dash
7f725558a000-7f725558c000 r--p 0001b000 00:27 22 /bin/dash
7f725558c000-7f725558d000 rw-p 0001d000 00:27 22 /bin/dash
7f725558d000-7f725558f000 rw-p 00000000 00:00 0
7f7256799000-7f72567ba000 rw-p 00000000 00:00 0 [heap]
7fff06be2000-7fff06c03000 rw-p 00000000 00:00 0 [stack]
7fff06cb8000-7fff06cba000 r-xp 00000000 00:00 0 [vdso]
7fff06cba000-7fff06cbc000 r--p 00000000 00:00 0 [vvar]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
root#ubuntu:/proc/41932# ls map_files/
7f7254d85000-7f7254f40000 7f725513f000-7f7255143000 7f725514a000-7f725516d000 7f725536d000-7f725536e000 7f725558a000-7f725558c000
7f7254f40000-7f725513f000 7f7255143000-7f7255145000 7f725536c000-7f725536d000 7f725536f000-7f725538b000 7f725558c000-7f725558d000
Which files i need to copy knowing that both start from the same script and how to copy this state?
an example for /proc/pid/maps
0022a000-00245000 r-xp 00000000 ca:01 11633540 /lib/ld-2.5.so
00245000-00246000 r--p 0001a000 ca:01 11633540 /lib/ld-2.5.so
00246000-00247000 rw-p 0001b000 ca:01 11633540 /lib/ld-2.5.so
00249000-003a3000 r-xp 00000000 ca:01 11633640 /lib/i686/nosegneg/libc-2.5.so
003a3000-003a5000 r--p 0015a000 ca:01 11633640 /lib/i686/nosegneg/libc-2.5.so
003a5000-003a6000 rw-p 0015c000 ca:01 11633640 /lib/i686/nosegneg/libc-2.5.so
003a6000-003a9000 rw-p 003a6000 00:00 0
00ada000-00adb000 r-xp 00ada000 00:00 0 [vdso]
08048000-08049000 r-xp 00000000 00:16 4735574 /home/yimingwa/test/Ctest/link_test/SectionMapping.elf
08049000-0804a000 rw-p 00000000 00:16 4735574 /home/yimingwa/test/Ctest/link_test/SectionMapping.elf
b7fcf000-b7fd0000 rw-p b7fcf000 00:00 0
b7fe1000-b7fe2000 rw-p b7fe1000 00:00 0
bfe82000-bfe98000 rw-p bffe8000 00:00 0 [stack]
the 4th column means “If the region was mapped from a file, this is the major and minor device number (in hex) where the file lives”
In the above, ca:01 I can find through /proc/devices /dev
Question is that what does "00:16" 00 means which major device?
Well I know that there is a heap that is common for all threads which grows upwards and we have stacks for each of the threads which grow downwards (Is it really so, the stack growing downwards or is it just a simplistic view?). I even read somewhere that stack can be a part of the heap space.
And how are these stacks placed? One above the other? What happens if the stack at the top overflows and attempts to write the stack below it? Is it really like this?
A detailed view please.
I just tested it with a short Python "program" in the interactive interpreter:
import threading
import time
def d(): time.sleep(120)
t = [threading.Thread(target=d) for _ in range(250)]
for i in t: i.start()
Then I pressed ^Z and looked at the appropriate /proc/.../maps file for this process.
It showed me
00048000-00049000 ---p 00000000 00:00 0
00049000-00848000 rw-p 00000000 00:00 0 [stack:28625]
00848000-00849000 ---p 00000000 00:00 0
00849000-01048000 rw-p 00000000 00:00 0 [stack:28624]
01048000-01049000 ---p 00000000 00:00 0
01049000-01848000 rw-p 00000000 00:00 0 [stack:28623]
01848000-01849000 ---p 00000000 00:00 0
01849000-02048000 rw-p 00000000 00:00 0 [stack:28622]
...
47700000-47701000 ---p 00000000 00:00 0
47701000-47f00000 rw-p 00000000 00:00 0 [stack:28483]
47f00000-47f01000 ---p 00000000 00:00 0
47f01000-48700000 rw-p 00000000 00:00 0 [stack:28482]
...
bd777000-bd778000 ---p 00000000 00:00 0
bd778000-bdf77000 rw-p 00000000 00:00 0 [stack:28638]
bdf77000-bdf78000 ---p 00000000 00:00 0
bdf78000-be777000 rw-p 00000000 00:00 0 [stack:28639]
be777000-be778000 ---p 00000000 00:00 0
be778000-bef77000 rw-p 00000000 00:00 0 [stack:28640]
bef77000-bef78000 ---p 00000000 00:00 0
bef78000-bf777000 rw-p 00000000 00:00 0 [stack:28641]
bf85c000-bf87d000 rw-p 00000000 00:00 0 [stack]
which shows what I already suspected: the stacks are allocated with a relative distance which is (hopefully) large enough.
The stacks have a relative distance of 8 MiB (this is the default value; it is possible to set it otherwise), and one page at the top is protected in order to detect a stack overflow.
The one at the bottom is the "main" stack; it can - in this example - grow until the next one is reached.
I have a strange ELF binary. I can run this binary in 32bit linux.
But if I open this binary with IDA disassembler, IDA says "invalid entry point".
Result of readelf is as below:
root#meltdown-VirtualBox:/home/meltdown# readelf -S -l SimpleVM
There are no sections in this file.
Elf file type is EXEC (Executable file)
Entry point 0xc023dc
There are 2 program headers, starting at offset 52
Program Headers:
Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align
LOAD 0x000000 0x00c01000 0x00c01000 0x013c7 0x013c7 RWE 0x1000
LOAD 0x00019c 0x0804b19c 0x0804b19c 0x00000 0x00000 RW 0x1000
There is no section. I thought this binary is packed. But, last virtual address of first LOAD segment is 0xc023c7. And virtual address of entry point is 0xc023dc which is out of range...
Can someone tell me whats going on?
Thank you in advance.
/proc/PID/maps is as follows (two processes are created...)
root#meltdown-VirtualBox:/proc/3510# cat maps
00110000-00111000 rwxp 00000000 00:00 0
006c0000-006c1000 r-xp 00000000 00:00 0 [vdso]
007d2000-007d4000 rwxp 00000000 00:00 0
00c01000-00c02000 rwxp 00000000 08:01 3801242 /home/meltdown/SimpleVM
00ca4000-00e43000 r-xp 00000000 08:01 17171359 /lib/i386-linux-gnu/libc-2.15.so
00e43000-00e45000 r-xp 0019f000 08:01 17171359 /lib/i386-linux-gnu/libc-2.15.so
00e45000-00e46000 rwxp 001a1000 08:01 17171359 /lib/i386-linux-gnu/libc-2.15.so
00e46000-00e49000 rwxp 00000000 00:00 0
08048000-0804b000 r-xp 00000000 00:00 0
0804b000-0804c000 rwxp 00000000 00:00 0
b77a7000-b77c7000 r-xp 00000000 08:01 17171339 /lib/i386-linux-gnu/ld-2.15.so
b77c7000-b77c8000 r-xp 0001f000 08:01 17171339 /lib/i386-linux-gnu/ld-2.15.so
b77c8000-b77c9000 rwxp 00020000 08:01 17171339 /lib/i386-linux-gnu/ld-2.15.so
bfa90000-bfab1000 rwxp 00000000 00:00 0 [stack]
root#meltdown-VirtualBox:/proc/3511# cat maps
00110000-00111000 rwxp 00000000 00:00 0
006c0000-006c1000 r-xp 00000000 00:00 0 [vdso]
007d2000-007d4000 rwxp 00000000 00:00 0
00c01000-00c02000 rwxp 00000000 08:01 3801242 /home/meltdown/SimpleVM
00ca4000-00e43000 r-xp 00000000 08:01 17171359 /lib/i386-linux-gnu/libc-2.15.so
00e43000-00e45000 r-xp 0019f000 08:01 17171359 /lib/i386-linux-gnu/libc-2.15.so
00e45000-00e46000 rwxp 001a1000 08:01 17171359 /lib/i386-linux-gnu/libc-2.15.so
00e46000-00e49000 rwxp 00000000 00:00 0
08048000-0804b000 r-xp 00000000 00:00 0
0804b000-0804c000 rwxp 00000000 00:00 0
b77a7000-b77c7000 r-xp 00000000 08:01 17171339 /lib/i386-linux-gnu/ld-2.15.so
b77c7000-b77c8000 r-xp 0001f000 08:01 17171339 /lib/i386-linux-gnu/ld-2.15.so
b77c8000-b77c9000 rwxp 00020000 08:01 17171339 /lib/i386-linux-gnu/ld-2.15.so
bfa90000-bfab1000 rwxp 00000000 00:00 0 [stack]
It's probably because of the granularity of mapping length. The length of the mapping is going to be rounded up to be a multiple of the page size. On my system the page size is 4k so the mapping would be rounded up to 4k and encompass the entry point. Even with a page size of 1k the length would round up to 0x1400, enough to include the entry point. If the file is long enough then the extra bytes would probably come from the file instead of the page initialization.