observe physical address content - linux

I compile and run the pagemap_dump.c from here :
/proc/[pid]/pagemaps and /proc/[pid]/maps | linux
and another application attach a exist shared memory and every time I run pagemap_dump , the shared memory's physical address is really the same (virtual memory address is different each time I observe) :
./pagemap_dump.exe 2135 <== 2135 is pid
./pagemap_dump.exe 5864 <== 5864 is pid
./pagemap_dump.exe 5110 <== 5110 is pid
the output :
7fabb37e1000 54edde 0 1 0 1 /dev/shm/memtest.shared
7f0040fa1000 54edde 0 1 0 1 /dev/shm/memtest.shared
7ffbd2869000 54edde 0 1 0 1 /dev/shm/memtest.shared
these 3 application attach memtest.shared and their physical address are all 0x54edde in my linux ,
then I try to access the contents from /dev/mem in this
physical address :
How to access physical addresses from user space in Linux?
int main(int argc, char** argv)
{
if (argc < 3) {
printf("Usage: %s <phys_addr> <offset>\n", argv[0]);
return 0;
}
off_t offset = strtoul(argv[1], NULL, 0);
size_t len = strtoul(argv[2], NULL, 0);
// Truncate offset to a multiple of the page size, or mmap will fail.
size_t pagesize = sysconf(_SC_PAGE_SIZE);
off_t page_base = (offset / pagesize) * pagesize;
off_t page_offset = offset - page_base;
int fd = open("/dev/mem", O_RDONLY|O_SYNC);
unsigned char *mem = (unsigned char *) mmap(NULL, page_offset + len, PROT_READ , MAP_SHARED, fd, page_base);
if (mem == MAP_FAILED) {
perror("Can't map memory");
return -1;
}
size_t i;
for (i = 0; i < len; ++i)
printf("%02x ", (int)mem[page_offset + i]);
return 0;
} // main
chown root testz1.exe
chmod 4775 testz1.exe
./testz1.exe 5565918 16 <=== 0x54edde = 5565918
I got Can't map memory: Operation not permitted
./testz1.exe 54edde 16
I got some thing not like "hello world" what I strcpy to
/dev/shm/memtest.shared when I create it .
Since I seems to get the physical address of /dev/shm/memtest.shared , how can I get the right way to get the contents from /dev/mem of this physical address ?!
Edit :
After download devmem2.c and run :
./devmem2.exe 0x54edde w
get the following message :
/dev/mem opened.
Error at line 75, file devmem2.c (1) [Operation not permitted]
Look like 0x54edde is not a physical address as I expect , but Why it happened?!
pagemap_dump.c really show the physical address of /dev/shm/memtest.shared
for all processes which attach it the value 0x54edde , then Why
devmem2.c can not access it right ?!

Related

mmap() NVMe cmb resource file and memcpy() to it, with the unplugged SSD still could write

I mapped cmb resource file and wrote data to it. When I unplugged the SSD, I thought that a SIGBUS should be generated but the program was still running. So was this a normal behavior?
Linux kernel: v5.15.88
OS: Centos 7.4
CPU: Intel(R) Xeon(R) Silver 4114
My code like this:
size_t resource_size = 268435456;
size_t remain = resource_size;
size_t offset = 0;
size_t range = 128;
char tmp;
int fd = open("/sys/bus/pci/devices/0000:5e:00.0/resource4_wc", O_RDWR);
void * map_addr = mmap(NULL, resource_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
while (remain > 0) {
memcpy(map_addr + offset, buf + offset, range);
memcpy(&tmp, map_addr + offset, 0);
remain -= range;
offset += range;
if (remain == 0) {
remain = resource_size;
offset = 0;
}
}

Registering Mapped Linux Character Device Memory with cudaHostRegister Results in Invalid Argument

I'm trying to boost DMA<->CPU<->GPU data transfer by:
1. Mapping my (proprietary) device Linux Kernel allocated memory to user space
2. Registering the later (mapped memory) to Cuda with cudaHostRegister API function.
While mapping User Space allocated memory mapped to my device DMA and then registered to Cuda with cudaHostRegister works just fine, trying to register "kmalloced" memory results in "Invalid Argument" error returned by cudaHostRegister.
First I thought the problem was with alignment or my device driver complicated memory pool management, so I've written a simplest character device which implements .mmap() where kzalloced 10Kb buffer is remapped with remap_pfn_range and the problem still stands.
Unfortunately, I did not find any resembling questions over the Net, so I sincerely hope I'll find an answer here.
Some system info and Kernel driver <-> user space app code + runtime log info:
CUDA : 8.0
OS Dist : Ubuntu 14.04
Kernel : 3.16.0-31-generic
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 375.26 Driver Version: 375.26 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 GeForce GTX 770 Off | 0000:83:00.0 N/A | N/A |
| 26% 32C P8 N/A / N/A | 79MiB / 1997MiB | N/A Default |
+-------------------------------+----------------------+----------------------+
Character device mmap() code:
#define MEM_CHUNK_SIZE 4 * _K
#define MEM_POOL_SIZE 10 * _K
/**/
static int chdv_mmap(struct file *filp, struct vm_area_struct *vma)
{
unsigned int pages_per_buf = ( MEM_CHUNK_SIZE >> PAGE_SHIFT ) ;
unsigned long pfn, vsize;
/*make sure the buffer is allocated*/
if((NULL == g_membuff) &&
(NULL == (g_membuff = kzalloc(MEM_POOL_SIZE , GFP_KERNEL))))
{
kdbgprintln("Error: Not enough memory");
return -ENOMEM;
}
vsize = vma->vm_end - vma->vm_start ;
kdbgprintln("MEM_CHUNK_SIZE %u, pages_per_buf %u, vsize %lu vma->vm_pgoff %lu",
MEM_CHUNK_SIZE,
pages_per_buf,
vsize,
vma->vm_pgoff);
if(vsize > MEM_POOL_SIZE)
{
kdbgprintln("Error: vsize %lu > MEM_POOL_SIZE %u", vsize, MEM_POOL_SIZE);
return -EINVAL;
}
/* We allow only mapping of one whole buffer so offset must be multiple
* of pages_per_buf and size must be equal to dma_buf_size.
*/
if( vma->vm_pgoff % pages_per_buf )
{
kdbgprintln("Error:Mapping DMA buffers is allowed only from beginning");
return -EINVAL ;
}
vma->vm_flags = vma->vm_flags | (VM_DONTEXPAND | VM_LOCKED | VM_IO);
/*Get the PFN for remap*/
pfn = page_to_pfn(virt_to_page((unsignedcudaHostRegister char *)g_membuff));
kdbgprintln("PFN : %lu", pfn);
if(remap_pfn_range(vma, vma->vm_start, pfn, vsize, vma->vm_page_prot))
{
kdbgprintln("Error:Failed to remap memory");
return -EINVAL;
}
/*Sealing data header & footer*/
*((unsigned long *)g_membuff) = 0xCDFFFFFFFFFFFFAB;
*((unsigned long *)g_membuff + 1) = 0xAB000000000000EF;
*(unsigned long *)((unsigned char *)g_membuff + vsize - sizeof(unsigned long)) = 0xEF0000000C0000AA;
kdbgprintln("Mapped 'kalloc' buffer" \
"\n\t\tFirst 8 bytes: %lX" \
"\n\t\tSecond 8 bytes: %lX" \
"\n\t\tLast 8 bytes: %lX",
*((unsigned long *)g_membuff),
*((unsigned long *)g_membuff + 1),
*(unsigned long *)((unsigned char *)g_membuff + vsize - sizeof(unsigned long)));
return 0;
}
Test Application code:
static unsigned long map_mem_size;
int main(int argc, char** argv)
{
int fd;
const char dev_name[] = "/dev/chardev";
void * address = NULL;
long page_off = 0;
cudaError_t cudarc;
switch(argc)
{
case 2:
page_off = atoi(argv[1]) * getpagesize();
break;
default:
page_off = 0;
break;
}
map_mem_size = 2 * getpagesize();
printf("Opening %s file\n", dev_name);
errno = 0;
if(0 > (fd = open(dev_name, O_RDWR) ))
{
printf("Error %d - %s\n", errno, strerror(errno));
}
else
{
printf("About to map %lu bytes of %s device memory\n", map_mem_size, dev_name);
errno = 0;
if(MAP_FAILED == (address = mmap(NULL, map_mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, page_off)))
{
printf("Error %d - %s\n", errno, strerror(errno));
}
else
{
printf("mapped %s driver 'kmalloc' memory" \
"\n\t\tFirst 8 bytes : %lX" \
"\n\t\tSecond 8 bytes: %lX" \
"\n\t\tLast 8 bytes: %lX\n",
dev_name,
*((unsigned long *)address),
*((unsigned long *)address + 1),
*(unsigned long *)((unsigned char *)address + map_mem_size - sizeof(unsigned long)));
if (cudaSuccess != (cudarc = cudaHostRegister(address, map_mem_size, cudaHostRegisterDefault)))
{
printf("Error: Failed cudaHostRegister: %s, address %p\n", cudaGetErrorString(cudarc), address);
}
}
}
/*Release resources block*/
return EXIT_SUCCESS;
}
Run time debug information:
User space:
./chrdev_test
Opening /dev/chardev file
About to map 8192 bytes of /dev/chardev device memory
mapped /dev/chardev driver 'kmalloc' memory
First 8 bytes : CDFFFFFFFFFFFFAB
Second 8 bytes: AB000000000000EF
Last 8 bytes: EF0000000C0000AA
Error: Failed cudaHostRegister: invalid argument
Unmapping /dev/chardev file
Closing /dev/chardev file
Kernel space (tail -f /var/log/syslog):
[ 4814.119537] [chardev] chardev.c, chdv_mmap, line 292:MEM_CHUNK_SIZE 4096, pages_per_buf 1, vsize 8192 vma->vm_pgoff 0
[ 4814.119538] [chardev] chardev.c, chdv_mmap, line 311:PFN : 16306184
[ 4814.119543] [chardev] chardev.c, chdv_mmap, line 330:Mapped 'kzalloced' buffer
[ 4814.119543] First 8 bytes: CDFFFFFFFFFFFFAB
[ 4814.119543] Second 8 bytes: AB000000000000EF
[ 4814.119543] Last 8 bytes: EF0000000C0000AA
Thanks ahead.
Made it work!
The full answer may be found in:
https://devtalk.nvidia.com/default/topic/1014391/cuda-programming-and-performance/registering-mapped-linux-character-device-memory-with-cudahostregister-results-in-invalid-argument/?offset=3#5174771
There is a problem with memory chunks longer than 2 pages (> 8K)
working with Cuda...
Thanks,
Yoel.

How does limits on the shared memory work on Linux

I was looking into the Linux kernel limits on the shared memory
/proc/sys/kernel/shmall
specifies the maximum amount of pages that can be allocated. Considering this number as x and the page size as p. I assume that "x * p" bytes is the limit on the system wide shared memory.
Now I wrote a small program to create a shared memory segment and i attached to that shared memory segment twice as below
shm_id = shmget(IPC_PRIVATE, 4*sizeof(int), IPC_CREAT | 0666);
if (shm_id < 0) {
printf("shmget error\n");
exit(1);
}
printf("\n The shared memory created is %d",shm_id);
ptr = shmat(shm_id,NULL,0);
ptr_info = shmat(shm_id,NULL,0);
In the above program ptr and ptr_info were different. So the shared memory is mapped to 2 virtual addresses in my process address space.
When I do an ipcs it looks like this
...
0x00000000 1638416 sun 666 16000000 2
...
Now coming to the shmall limit x * p noted above in my question. Is this limit applicable on the sum of all the virtual memory allocated for every shared memory segment? or does this limit apply on the physical memory?
Physical memory is only one here (shared memory) and from the program above when I do 2 shmat's there is twice the amount of memory allocated in my process address space. So this limit will hit soon if do continuous shmat's on a single shared memory segment?
The limit only applies to physical memory, that is the real shared memory allocated for all segments, because shmat() just maps that allocated segment into process address space.
You can trace it in the kernel, there is only one place where this limit is checked — in the newseg() function that allocates new segments (ns->shm_ctlall comparison). shmat() implementation is busy with a lot of stuff, but doesn't care at all about shmall limit, so you can map one segment as many times as you want to (well, address space is also limited, but in practice you rarely care about this limit).
You can also try some test from userspace with a simple program like this one:
#define _GNU_SOURCE
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <unistd.h>
unsigned long int get_shmall() {
FILE *f = NULL;
char buf[512];
unsigned long int value = 0;
if ((f = fopen("/proc/sys/kernel/shmall", "r")) != NULL) {
if (fgets(buf, sizeof(buf), f) != NULL)
value = strtoul(buf, NULL, 10); // no proper checks
fclose(f); // no return value check
}
return value;
}
int set_shmall(unsigned long int value) {
FILE *f = NULL;
char buf[512];
int retval = 0;
if ((f = fopen("/proc/sys/kernel/shmall", "w")) != NULL) {
if (snprintf(buf, sizeof(buf), "%lu\n", value) >= sizeof(buf) ||
fwrite(buf, 1, strlen(buf), f) != strlen(buf))
retval = -1;
fclose(f); // fingers crossed
} else
retval = -1;
return retval;
}
int main()
{
int shm_id1 = -1, shm_id2 = -1;
unsigned long int shmall = 0, shmused, newshmall;
void *ptr1, *ptr2;
struct shm_info shminf;
if ((shmall = get_shmall()) == 0) {
printf("can't get shmall\n");
goto out;
}
printf("original shmall: %lu pages\n", shmall);
if (shmctl(0, SHM_INFO, (struct shmid_ds *)&shminf) < 0) {
printf("can't get SHM_INFO\n");
goto out;
}
shmused = shminf.shm_tot * getpagesize();
printf("shmused: %lu pages (%lu bytes)\n", shminf.shm_tot, shmused);
newshmall = shminf.shm_tot + 1;
if (set_shmall(newshmall) != 0) {
printf("can't set shmall\n");
goto out;
}
if (get_shmall() != newshmall) {
printf("something went wrong with shmall setting\n");
goto out;
}
printf("new shmall: %lu pages (%lu bytes)\n", newshmall, newshmall * getpagesize());
printf("shmget() for %u bytes: ", (unsigned int) getpagesize());
shm_id1 = shmget(IPC_PRIVATE, (size_t)getpagesize(), IPC_CREAT | 0666);
if (shm_id1 < 0) {
printf("failed: %s\n", strerror(errno));
goto out;
}
printf("ok\nshmat 1: ");
ptr1 = shmat(shm_id1, NULL, 0);
if (ptr1 == 0) {
printf("failed\n");
goto out;
}
printf("ok\nshmat 2: ");
ptr2 = shmat(shm_id1, NULL, 0);
if (ptr2 == 0) {
printf("failed\n");
goto out;
}
printf("ok\n");
if (ptr1 == ptr2) {
printf("ptr1 and ptr2 are the same with shm_id1\n");
goto out;
}
printf("shmget() for %u bytes: ", (unsigned int) getpagesize());
shm_id2 = shmget(IPC_PRIVATE, (size_t)getpagesize(), IPC_CREAT | 0666);
if (shm_id2 < 0)
printf("failed: %s\n", strerror(errno));
else
printf("ok, although it's wrong\n");
out:
if (shmall != 0 && set_shmall(shmall) != 0)
printf("failed to restrore shmall\n");
if (shm_id1 >= 0 && shmctl(shm_id1, IPC_RMID, NULL) < 0)
printf("failed to remove shm_id1\n");
if (shm_id2 >= 0 && shmctl(shm_id2, IPC_RMID, NULL) < 0)
printf("failed to remove shm_id2\n");
return 0;
}
What is does is it sets the shmall limit just one page above what is currently used by the system, then tries to get page-sized new segment and map it twice (all successfully), then tries to get one more page-sized segment and fails to do that (execute the program as superuser because it writes to /proc/sys/kernel/shmall):
$ sudo ./a.out
original shmall: 18446744073708503040 pages
shmused: 21053 pages (86233088 bytes)
new shmall: 21054 pages (86237184 bytes)
shmget() for 4096 bytes: ok
shmat 1: ok
shmat 2: ok
shmget() for 4096 bytes: failed: No space left on device
I did not find any Physical memory allocation at do_shmat function (linux/ipc/shm.c)
https://github.com/torvalds/linux/blob/5469dc270cd44c451590d40c031e6a71c1f637e8/ipc/shm.c
so shmat consumes only vm (your process address space),
the main function of shmat is mmap

mmap /dev/fb0 fails with "Invalid argument"

I have an embedded system and want to use /dev/fb0 directly. As a first test, I use some code based on example-code found everywhere in the net and SO. Opening succeeds, also fstat and similar. But mmap fails with EINVAL.
Source:
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <linux/fb.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
int main() {
int fbfd = 0;
struct fb_var_screeninfo vinfo;
struct fb_fix_screeninfo finfo;
long int screensize = 0;
char *fbp = 0;
int x = 0, y = 0;
long int location = 0;
// Open the file for reading and writing
fbfd = open("/dev/fb0", O_RDWR);
if (fbfd == -1) {
perror("Error: cannot open framebuffer device");
exit(1);
}
printf("The framebuffer device was opened successfully.\n");
struct stat stat;
fstat(fbfd, &stat);
printf("/dev/mem -> size: %u blksize: %u blkcnt: %u\n",
stat.st_size, stat.st_blksize, stat.st_blocks);
// Get fixed screen information
if (ioctl(fbfd, FBIOGET_FSCREENINFO, &finfo) == -1) {
perror("Error reading fixed information");
exit(2);
}
// Get variable screen information
if (ioctl(fbfd, FBIOGET_VSCREENINFO, &vinfo) == -1) {
perror("Error reading variable information");
exit(3);
}
printf("%dx%d, %dbpp\n", vinfo.xres, vinfo.yres, vinfo.bits_per_pixel);
// Figure out the size of the screen in bytes
screensize = vinfo.xres * vinfo.yres * vinfo.bits_per_pixel / 8;
const int PADDING = 4096;
int mmapsize = (screensize + PADDING - 1) & ~(PADDING-1);
// Map the device to memory
fbp = (char *)mmap(0, mmapsize, PROT_READ | PROT_WRITE, MAP_SHARED, fbfd, 0);
if ((int)fbp == -1) {
perror("Error: failed to map framebuffer device to memory");
exit(4);
}
printf("The framebuffer device was mapped to memory successfully.\n");
munmap(fbp, screensize);
close(fbfd);
return 0;
}
Output:
The framebuffer device was opened successfully.
/dev/mem -> size: 0 blksize: 4096 blkcnt: 0
640x480, 4bpp
Error: failed to map framebuffer device to memory: Invalid argument
strace:
...
open("/dev/fb0", O_RDWR) = 3
write(1, "The framebuffer device was opene"..., 48The framebuffer device was opened successfully.
) = 48
fstat64(3, {st_mode=S_IFCHR|0640, st_rdev=makedev(29, 0), ...}) = 0
write(1, "/dev/mem -> size: 0 blksize: 409"..., 44/dev/mem -> size: 0 blksize: 4096 blkcnt: 0
) = 44
ioctl(3, FBIOGET_FSCREENINFO or FBIOPUT_CONTRAST, 0xbfca6564) = 0
ioctl(3, FBIOGET_VSCREENINFO, 0xbfca6600) = 0
write(1, "640x480, 4bpp\n", 14640x480, 4bpp
) = 14
old_mmap(NULL, 155648, PROT_READ|PROT_WRITE, MAP_SHARED, 3, 0) = -1 EINVAL (Invalid argument)
write(2, "Error: failed to map framebuffer"..., 49Error: failed to map framebuffer device to memory) = 49
write(2, ": ", 2: ) = 2
write(2, "Invalid argument", 16Invalid argument) = 16
write(2, "\n", 1
) = 1
The boot-screen with console and tux is visible. And cat /dev/urandom > /dev/fb0 fills the screen with noise. The pagesize is 4096 on the system (`getconf PAGESIZE). So, 155648 (0x26000) is a multiple. Offset and pointer are both zero. Mapping and filemode are both RW .. what am I missing?
This is for an embedded device build with uClibc and busybox running a single application and I have to port it from an ancient kernel. There is code for linedrawing and such and no need for multiprocessing/ windowing .. please no hints to directfb ;).
The kernel driver that presents the framebuffer doesn't support the legacy direct mmap() of the framebuffer device; you need to use the newer DRM and KMS interface.

Capturing user-space assembly with ftrace and kprobes (by using virtual address translation)?

Apologies for the longish post, I'm having trouble formulating it in a shorter way. Also, maybe this is more appropriate for Unix & Linux Stack Exchange, but I'll try here at SO first, as there is an ftrace tag.
Anyways - I'd like to observe do machine instructions of a user program execute in the context of a full function_graph capture using ftrace. One problem is that I need this for an older kernel:
$ uname -a
Linux mypc 2.6.38-16-generic #67-Ubuntu SMP Thu Sep 6 18:00:43 UTC 2012 i686 i686 i386 GNU/Linux
... and in this edition, there is no UPROBES - which, as Uprobes in 3.5 [LWN.net] notes, should be able to do something like that. (As long as I don't have to patch the original kernel, I would be willing to try a kernel module built out of tree, as User-Space Probes (Uprobes) [chunghwan.com] seems to demonstrate; but as far as I can see from 0: Inode based uprobes [LWN.net], the 2.6 would probably need a full patch)
However, on this version, there is a /sys/kernel/debug/kprobes, and /sys/kernel/debug/tracing/kprobe_events; and Documentation/trace/kprobetrace.txt implies that a kprobe can be set directly on an address; even if I cannot find an example anywhere on how this is used.
In any case, I would still not be sure what addresses to use - as a small example, let's say I want to trace the start of the main function of the wtest.c program (included below). I can do this to compile and obtain an machine instruction assembly listing:
$ gcc -g -O0 wtest.c -o wtest
$ objdump -S wtest | less
...
08048474 <main>:
int main(void) {
8048474: 55 push %ebp
8048475: 89 e5 mov %esp,%ebp
8048477: 83 e4 f0 and $0xfffffff0,%esp
804847a: 83 ec 30 sub $0x30,%esp
804847d: 65 a1 14 00 00 00 mov %gs:0x14,%eax
8048483: 89 44 24 2c mov %eax,0x2c(%esp)
8048487: 31 c0 xor %eax,%eax
char filename[] = "/tmp/wtest.txt";
...
return 0;
804850a: b8 00 00 00 00 mov $0x0,%eax
}
...
I would set up ftrace logging via this script:
sudo bash -c '
KDBGPATH="/sys/kernel/debug/tracing"
echo function_graph > $KDBGPATH/current_tracer
echo funcgraph-abstime > $KDBGPATH/trace_options
echo funcgraph-proc > $KDBGPATH/trace_options
echo 0 > $KDBGPATH/tracing_on
echo > $KDBGPATH/trace
echo 1 > $KDBGPATH/tracing_on ; ./wtest ; echo 0 > $KDBGPATH/tracing_on
cat $KDBGPATH/trace > wtest.ftrace
'
You can see a portion of the (otherwise complex) resulting ftrace log in debugging - Observing a hard-disk write in kernel space (with drivers/modules) - Unix & Linux Stack Exchange (where I got the example from).
Basically, I'd want a printout in this ftrace log, when the first instructions of main - say, the instructions at 0x8048474, 0x8048475, 0x8048477, 0x804847a, 0x804847d, 0x8048483 and 0x8048487 - are executed by (any) CPU. The problem is, as far as I can understand from Anatomy of a Program in Memory : Gustavo Duarte, these addresses are the virtual addresses, as seen from the perspective of the process itself (and I gather, the same perspective is shown by /proc/PID/maps)... And apparently, for krpobe_event I'd need a physical address?
So, my idea would be: if I can find the physical addresses corresponding to the virtual addresses of the program disassembly (say by coding a kernel module, which would accept pid and address, and return the physical address via procfs), I could set up addresses as a sort of "tracepoints" via /sys/kernel/debug/tracing/kprobe_events in the above script - and hopefully get them in the ftrace log. Could this work, in principle?
One problem with this, I found on Linux(ubuntu), C language: Virtual to Physical Address Translation - Stack Overflow:
In user code, you can't know the physical address corresponding to a virtual address. This is information is simply not exported outside the kernel. It could even change at any time, especially if the kernel decides to swap out part of your process's memory.
...
Pass the virtual address to the kernel using systemcall/procfs and use vmalloc_to_pfn. Return the Physical address through procfs/registers.
However, vmalloc_to_pfn doesn't seem to be trivial either:
x86 64 - vmalloc_to_pfn returns 32 bit address on Linux 32 system. Why does it chop off higher bits of PAE physical address? - Stack Overflow
VA: 0xf8ab87fc PA using vmalloc_to_pfn: 0x36f7f7fc. But I'm actually expecting: 0x136f7f7fc.
...
The physical address falls between 4 to 5 GB. But I can't get the exact physical address, I only get the chopped off 32-bit address. Is there another way to get true physical address?
So, I'm not sure how reliably I could extract the physical addresses so they are traced by kprobes - especially since "it could even change at any time". But here, I would hope that since the program is small and trivial, there would be a reasonable chance that the program would not swap while being traced, allowing for a proper capture to be obtained. (So even if I have to run the debug script above multiple times, as long as I can hope to obtain a "proper" capture once out of 10 times (or even 100 times), I'd be OK with it.).
Note that I'd want an output through ftrace, so that the timestamps are expressed in the same domain (see Reliable Linux kernel timestamps (or adjustment thereof) with both usbmon and ftrace? - Stack Overflow for an illustration of a problem with timestamps). Thus, even if I could come up with, say, a gdb script, to run and trace the program from userspace (while simultaneously an ftrace capture is obtained) - I'd like to avoid that, as the overhead from gdb itself will show in the ftrace logs.
So, in summary:
Is the approach of obtaining (possibly through a separate kernel module) physical addresses from the virtual (from a disassembly of an executable) addresses - so they are used to trigger a kprobe_event logged by ftrace - worth pursuing? If so, are there any examples of kernel modules that can be used for this address translation purpose?
Could I otherwise use a kernel module to "register" a callback/handler function when a particular memory address is being executed? Then I could simply use a trace_printk in that function to have an ftrace log (or even without that, the handler function name itself should show in the ftrace log), and it doesn't seem there will be too much overhead with that...
Actually, in this 2007 posting, Jim Keniston - utrace-based uprobes: systemtap mailing list, there is a 11. Uprobes Example (added to Documentation/uprobes.txt), which seems to be exactly that - a kernel module registering a handler function. Unfortunately, it uses linux/uprobes.h; and I have only kprobes.h in my /usr/src/linux-headers-2.6.38-16/include/linux/. Also, on my system, even systemtap complains about CONFIG_UTRACE not being enabled (see this comment)... So if there's any other approach I could use to obtain a debug trace like I want, without having to recompile the kernel to get uprobes, it would be great to know...
wtest.c:
#include <stdio.h>
#include <fcntl.h> // O_CREAT, O_WRONLY, S_IRUSR
int main(void) {
char filename[] = "/tmp/wtest.txt";
char buffer[] = "abcd";
int fd;
mode_t perms = S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH;
fd = open(filename, O_RDWR|O_CREAT, perms);
write(fd,buffer,4);
close(fd);
return 0;
}
Obviously, this would be much easier with built-in uprobes on kernels 3.5+; but given that uprobes for my kernel 2.6.38 is a very deep-going patch (which I couldn't really isolate in a separate kernel module, so as to avoid patching the kernel), here is what I can note for a standalone module on 2.6.38. (Since I'm still unsure of many things, I would still like to see an answer that would corrects any misunderstandings in this post.)
I think I got somewhere, but not with kprobes. I'm not sure, but it seems I managed to get physical addresses right; however, kprobes documentation is specific that when using "#ADDR : fetch memory at ADDR (ADDR should be in kernel)"; and the physical addresses I get are below kernel boundary of 0xc0000000 (but then, 0xc0000000 is usually together with the virtual memory layout?).
So I used a hardware breakpoint instead - the module is below, however caveat emptor - it behaves randomly, and occasionally can cause a kernel oops!. By compiling the module, and running in bash:
$ sudo bash -c 'KDBGPATH="/sys/kernel/debug/tracing" ;
echo function_graph > $KDBGPATH/current_tracer ; echo funcgraph-abstime > $KDBGPATH/trace_options
echo funcgraph-proc > $KDBGPATH/trace_options ; echo 8192 > $KDBGPATH/buffer_size_kb ;
echo 0 > $KDBGPATH/tracing_on ; echo > $KDBGPATH/trace'
$ sudo insmod ./callmodule.ko && sleep 0.1 && sudo rmmod callmodule && \
tail -n25 /var/log/syslog | tee log.txt && \
sudo cat /sys/kernel/debug/tracing/trace >> log.txt
... I get a log. I want to trace the first two instructions of the main() of wtest, which for me are:
$ objdump -S wtest/wtest | grep -A3 'int main'
int main(void) {
8048474: 55 push %ebp
8048475: 89 e5 mov %esp,%ebp
8048477: 83 e4 f0 and $0xfffffff0,%esp
... at virtual addresses 0x08048474 and 0x08048475. In the syslog output, I could get, say:
...
[ 1106.383011] callmodule: parent task a: f40a9940 c: kworker/u:1 p: [14] s: stopped
[ 1106.383017] callmodule: - wtest [9404]
[ 1106.383023] callmodule: Trying to walk page table; addr task 0xEAE90CA0 ->mm ->start_code: 0x08048000 ->end_code: 0x080485F4
[ 1106.383029] callmodule: walk_ 0x8048000 callmodule: Valid pgd : Valid pud: Valid pmd: page frame struct is # f63e5d80; *virtual (page_address) # (null) (is_vmalloc_addr 0 virt_addr_valid 0 virt_to_phys 0x40000000) page_to_pfn 639ec page_to_phys 0x639ec000
[ 1106.383049] callmodule: walk_ 0x80483c0 callmodule: Valid pgd : Valid pud: Valid pmd: page frame struct is # f63e5d80; *virtual (page_address) # (null) (is_vmalloc_addr 0 virt_addr_valid 0 virt_to_phys 0x40000000) page_to_pfn 639ec page_to_phys 0x639ec000
[ 1106.383067] callmodule: walk_ 0x8048474 callmodule: Valid pgd : Valid pud: Valid pmd: page frame struct is # f63e5d80; *virtual (page_address) # (null) (is_vmalloc_addr 0 virt_addr_valid 0 virt_to_phys 0x40000000) page_to_pfn 639ec page_to_phys 0x639ec000
[ 1106.383083] callmodule: physaddr : (0x080483c0 ->) 0x639ec3c0 : (0x08048474 ->) 0x639ec474
[ 1106.383106] callmodule: 0x08048474 id [3]
[ 1106.383113] callmodule: 0x08048475 id [4]
[ 1106.383118] callmodule: (( 0x08048000 is_vmalloc_addr 0 virt_addr_valid 0 ))
[ 1106.383130] callmodule: cont pid task a: eae90ca0 c: wtest p: [9404] s: runnable
[ 1106.383147] initcall callmodule_init+0x0/0x1000 [callmodule] returned with preemption imbalance
[ 1106.518074] callmodule: < exit
... meaning that it mapped the virtual address 0x08048474 to physical address 0x639ec474. However, the physical is not used for hardware breakpoints - there we can supply a virtual address directly to register_user_hw_breakpoint; however, we also need to supply the task_struct of the process too. With that, I can get something like this in the ftrace output:
...
597.907256 | 1) wtest-5339 | | handle_mm_fault() {
...
597.907310 | 1) wtest-5339 | + 35.627 us | }
597.907311 | 1) wtest-5339 | + 46.245 us | }
597.907312 | 1) wtest-5339 | + 56.143 us | }
597.907313 | 1) wtest-5339 | 1.039 us | up_read();
597.907317 | 1) wtest-5339 | 1.285 us | native_get_debugreg();
597.907319 | 1) wtest-5339 | 1.075 us | native_set_debugreg();
597.907322 | 1) wtest-5339 | 1.129 us | native_get_debugreg();
597.907324 | 1) wtest-5339 | 1.189 us | native_set_debugreg();
597.907329 | 1) wtest-5339 | | () {
597.907333 | 1) wtest-5339 | | /* callmodule: hwbp hit: id [3] */
597.907334 | 1) wtest-5339 | 5.567 us | }
597.907336 | 1) wtest-5339 | 1.123 us | native_set_debugreg();
597.907339 | 1) wtest-5339 | 1.130 us | native_get_debugreg();
597.907341 | 1) wtest-5339 | 1.075 us | native_set_debugreg();
597.907343 | 1) wtest-5339 | 1.075 us | native_get_debugreg();
597.907345 | 1) wtest-5339 | 1.081 us | native_set_debugreg();
597.907348 | 1) wtest-5339 | | () {
597.907350 | 1) wtest-5339 | | /* callmodule: hwbp hit: id [4] */
597.907351 | 1) wtest-5339 | 3.033 us | }
597.907352 | 1) wtest-5339 | 1.105 us | native_set_debugreg();
597.907358 | 1) wtest-5339 | 1.315 us | down_read_trylock();
597.907360 | 1) wtest-5339 | 1.123 us | _cond_resched();
597.907362 | 1) wtest-5339 | 1.027 us | find_vma();
597.907364 | 1) wtest-5339 | | handle_mm_fault() {
...
... where the traces corresponding to the assembly are marked by breakpoint id. Thankfully, they are right after another, as expected; however, ftrace has also captured some debug commands in-between. In any case, this is what I wanted to see.
Here are some notes about the module:
Most of the module is from Execute/invoke user-space program, and get its pid, from a kernel module ; where a user process is started and pid obtained
Since we have to get to the task_struct to get to the pid; here I save both (which is kind of redundant)
Where functions symbols are not exported; if the symbol is in kallsyms, then I use a function pointer to the address; else other needed functions are copied from source
I didn't know how to start the user-space process stopped, so after spawning I issue a SIGSTOP (which on its own, seems kind of unreliable at that point), and set state to __TASK_STOPPED).
I may still get status "runnable" where I don't expect it sometimes - however, if the init exits early with an error, I've noticed wtest hanging in process list long after it would have terminated naturally, so I guess that works.
To get absolute/physical addresses, I used Walking page tables of a process in Linux to get to the page corresponding to a virtual address, and then digging through kernel sources I found page_to_phys() to get to the address (internally via page frame number); LDD3 ch.15 helps with understanding relationship between pfn and physical address.
Since here I expect to have physical address, I don't use PAGE_SHIFT, but calculate offsets directly from objdump's assembly output - I am not 100% sure this is correct, though.
Note, ( see also How to get a struct page from any address in the Linux kernel ), the module output says that the virtual address 0x08048000 is neither is_vmalloc_addr nor virt_addr_valid; I guess, this should tell me, one couldn't have used neither vmalloc_to_pfn() nor virt_to_page() to get to its physical address !?
Setting up kprobes for ftrace from kernel space is kinda tricky (needs functions copied)
Trying to set a kprobe on the physical addresses I get (e.g. 0x639ec474), always results with "Could not insert probe(-22)"
Just to see if the format is parsed, I'm trying with the kallsyms address of the tracing_on() function (0xc10bcf60) below; that seems to work - because it raises a fatal "BUG: scheduling while atomic" (apparently, we're not meant to set breakpoints in module_init?). Bug is fatal, because it makes the kprobes directory dissapear from the ftrace debug directory
Just creating the kprobe would not make it appear in the ftrace log - it also needs to be enabled; the necessary code for enabling is there - but I've never tried it, because of the previous bug
Finally, the breakpoint setting is from Watch a variable (memory address) change in Linux kernel, and print stack trace when it changes?
I've never seen an example for setting an executable hardware breakpoint; it kept failing for me, until through kernel source search, I found that for HW_BREAKPOINT_X, attr.bp_len need to be set to sizeof(long)
If I try to printk the attr variable(s) - from _init or from the handler - something gets seriously messed up, and whatever variable I try to print next, I get value 0x5 (or 0x48) for it (?!)
Since I'm trying to use a single handler function for both breakpoints, the only reliable piece of info that survives from _init to the handler, able to differentiate between the two, seems to be bp->id
These id's are autoassigned, and seems they are not re-claimed if you unregister the breakpoints (I do not unregister them to avoid extra ftrace printouts).
As far as the randomness goes, I think this is because the process is not started in a stopped state; and by the time it gets stopped, it ends up in a different state (or, quite possibly, I'm missing some locking somewhere). Anyways, you can also expect in syslog:
[ 1661.815114] callmodule: Trying to walk page table; addr task 0xEAF68CA0 ->mm ->start_code: 0x08048000 ->end_code: 0x080485F4
[ 1661.815319] callmodule: walk_ 0x8048000 callmodule: Valid pgd : Valid pud: Valid pmd: page frame struct is # f5772000; *virtual (page_address) # c0000000 (is_vmalloc_addr 0 virt_addr_valid 1 virt_to_phys 0x0) page_to_pfn 0 page_to_phys 0x0
[ 1661.815837] callmodule: walk_ 0x80483c0 callmodule: Valid pgd : Valid pud: Valid pmd: page frame struct is # f5772000; *virtual (page_address) # c0000000 (is_vmalloc_addr 0 virt_addr_valid 1 virt_to_phys 0x0) page_to_pfn 0 page_to_phys 0x0
[ 1661.816846] callmodule: walk_ 0x8048474 callmodule: Valid pgd : Valid pud: Valid pmd: page frame struct is # f5772000; *virtual (page_address) # c0000000 (is_vmalloc_addr 0 virt_addr_valid 1 virt_to_phys 0x0) page_to_pfn 0 page_to_phys 0x0
... that is, even with a proper task pointer (judging by start_code), only 0x0 is obtained as physical address. Sometimes you get the same outcome, but with start_code: 0x00000000 ->end_code: 0x00000000. And sometimes, a task_struct cannot be obtained, even if pid can:
[ 833.380417] callmodule:c: pid 7663
[ 833.380424] callmodule: everything all right; pid 7663 (7663)
[ 833.380430] callmodule: p is NULL - exiting
[ 833.516160] callmodule: < exit
Well, hopefully someone will comment and clarify some of the behavior of this module :)
Hope this helps someone,
Cheers!
Makefile:
EXTRA_CFLAGS=-g -O0
obj-m += callmodule.o
all:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
clean:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
callmodule.c:
#include <linux/module.h>
#include <linux/slab.h> //kzalloc
#include <linux/syscalls.h> // SIGCHLD, ... sys_wait4, ...
#include <linux/kallsyms.h> // kallsyms_lookup, print_symbol
#include <linux/highmem.h> // ‘kmap_atomic’ (via pte_offset_map)
#include <asm/io.h> // page_to_phys (arch/x86/include/asm/io.h)
struct subprocess_infoB; // forward declare
// global variable - to avoid intervening too much in the return of call_usermodehelperB:
static int callmodule_pid;
static struct subprocess_infoB* callmodule_infoB;
#define TRY_USE_KPROBES 0 // 1 // enable/disable kprobes usage code
#include <linux/kprobes.h> // enable_kprobe
// for hardware breakpoint:
#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
// define a modified struct (with extra fields) here:
struct subprocess_infoB {
struct work_struct work;
struct completion *complete;
char *path;
char **argv;
char **envp;
int wait; //enum umh_wait wait;
int retval;
int (*init)(struct subprocess_info *info);
void (*cleanup)(struct subprocess_info *info);
void *data;
pid_t pid;
struct task_struct *task;
unsigned long long last_page_physaddr;
};
struct subprocess_infoB *call_usermodehelper_setupB(char *path, char **argv,
char **envp, gfp_t gfp_mask);
static inline int
call_usermodehelper_fnsB(char *path, char **argv, char **envp,
int wait, //enum umh_wait wait,
int (*init)(struct subprocess_info *info),
void (*cleanup)(struct subprocess_info *), void *data)
{
struct subprocess_info *info;
struct subprocess_infoB *infoB;
gfp_t gfp_mask = (wait == UMH_NO_WAIT) ? GFP_ATOMIC : GFP_KERNEL;
int ret;
populate_rootfs_wait();
infoB = call_usermodehelper_setupB(path, argv, envp, gfp_mask);
printk(KBUILD_MODNAME ":a: pid %d\n", infoB->pid);
info = (struct subprocess_info *) infoB;
if (info == NULL)
return -ENOMEM;
call_usermodehelper_setfns(info, init, cleanup, data);
printk(KBUILD_MODNAME ":b: pid %d\n", infoB->pid);
// this must be called first, before infoB->pid is populated (by __call_usermodehelperB):
ret = call_usermodehelper_exec(info, wait);
// assign global pid (and infoB) here, so rest of the code has it:
callmodule_pid = infoB->pid;
callmodule_infoB = infoB;
printk(KBUILD_MODNAME ":c: pid %d\n", callmodule_pid);
return ret;
}
static inline int
call_usermodehelperB(char *path, char **argv, char **envp, int wait) //enum umh_wait wait)
{
return call_usermodehelper_fnsB(path, argv, envp, wait,
NULL, NULL, NULL);
}
static void __call_usermodehelperB(struct work_struct *work)
{
struct subprocess_infoB *sub_infoB =
container_of(work, struct subprocess_infoB, work);
int wait = sub_infoB->wait; // enum umh_wait wait = sub_info->wait;
pid_t pid;
struct subprocess_info *sub_info;
// hack - declare function pointers
int (*ptrwait_for_helper)(void *data);
int (*ptr____call_usermodehelper)(void *data);
// assign function pointers to verbatim addresses as obtained from /proc/kallsyms
int killret;
struct task_struct *spawned_task;
ptrwait_for_helper = (void *)0xc1065b60;
ptr____call_usermodehelper = (void *)0xc1065ed0;
sub_info = (struct subprocess_info *)sub_infoB;
if (wait == UMH_WAIT_PROC)
pid = kernel_thread((*ptrwait_for_helper), sub_info, //(wait_for_helper, sub_info,
CLONE_FS | CLONE_FILES | SIGCHLD);
else
pid = kernel_thread((*ptr____call_usermodehelper), sub_info, //(____call_usermodehelper, sub_info,
CLONE_VFORK | SIGCHLD);
spawned_task = pid_task(find_vpid(pid), PIDTYPE_PID);
// stop/suspend/pause task
killret = kill_pid(find_vpid(pid), SIGSTOP, 1);
if (spawned_task!=NULL) {
// does this stop the process really?
spawned_task->state = __TASK_STOPPED;
printk(KBUILD_MODNAME ": : exst %d exco %d exsi %d diex %d inex %d inio %d\n", spawned_task->exit_state, spawned_task->exit_code, spawned_task->exit_signal, spawned_task->did_exec, spawned_task->in_execve, spawned_task->in_iowait);
}
printk(KBUILD_MODNAME ": : (kr: %d)\n", killret);
printk(KBUILD_MODNAME ": : pid %d (%p) (%s)\n", pid, spawned_task,
(spawned_task!=NULL)?((spawned_task->state==-1)?"unrunnable":((spawned_task->state==0)?"runnable":"stopped")):"null" );
// grab and save the pid (and task_struct) here:
sub_infoB->pid = pid;
sub_infoB->task = spawned_task;
switch (wait) {
case UMH_NO_WAIT:
call_usermodehelper_freeinfo(sub_info);
break;
case UMH_WAIT_PROC:
if (pid > 0)
break;
/* FALLTHROUGH */
case UMH_WAIT_EXEC:
if (pid < 0)
sub_info->retval = pid;
complete(sub_info->complete);
}
}
struct subprocess_infoB *call_usermodehelper_setupB(char *path, char **argv,
char **envp, gfp_t gfp_mask)
{
struct subprocess_infoB *sub_infoB;
sub_infoB = kzalloc(sizeof(struct subprocess_infoB), gfp_mask);
if (!sub_infoB)
goto out;
INIT_WORK(&sub_infoB->work, __call_usermodehelperB);
sub_infoB->path = path;
sub_infoB->argv = argv;
sub_infoB->envp = envp;
out:
return sub_infoB;
}
#if TRY_USE_KPROBES
// copy from /kernel/trace/trace_probe.c (is unexported)
int traceprobe_command(const char *buf, int (*createfn)(int, char **))
{
char **argv;
int argc, ret;
argc = 0;
ret = 0;
argv = argv_split(GFP_KERNEL, buf, &argc);
if (!argv)
return -ENOMEM;
if (argc)
ret = createfn(argc, argv);
argv_free(argv);
return ret;
}
// copy from kernel/trace/trace_kprobe.c?v=2.6.38 (is unexported)
#define TP_FLAG_TRACE 1
#define TP_FLAG_PROFILE 2
typedef void (*fetch_func_t)(struct pt_regs *, void *, void *);
struct fetch_param {
fetch_func_t fn;
void *data;
};
typedef int (*print_type_func_t)(struct trace_seq *, const char *, void *, void *);
enum {
FETCH_MTD_reg = 0,
FETCH_MTD_stack,
FETCH_MTD_retval,
FETCH_MTD_memory,
FETCH_MTD_symbol,
FETCH_MTD_deref,
FETCH_MTD_END,
};
// Fetch type information table * /
struct fetch_type {
const char *name; /* Name of type */
size_t size; /* Byte size of type */
int is_signed; /* Signed flag */
print_type_func_t print; /* Print functions */
const char *fmt; /* Fromat string */
const char *fmttype; /* Name in format file */
// Fetch functions * /
fetch_func_t fetch[FETCH_MTD_END];
};
struct probe_arg {
struct fetch_param fetch;
struct fetch_param fetch_size;
unsigned int offset; /* Offset from argument entry */
const char *name; /* Name of this argument */
const char *comm; /* Command of this argument */
const struct fetch_type *type; /* Type of this argument */
};
struct trace_probe {
struct list_head list;
struct kretprobe rp; /* Use rp.kp for kprobe use */
unsigned long nhit;
unsigned int flags; /* For TP_FLAG_* */
const char *symbol; /* symbol name */
struct ftrace_event_class class;
struct ftrace_event_call call;
ssize_t size; /* trace entry size */
unsigned int nr_args;
struct probe_arg args[];
};
static int probe_is_return(struct trace_probe *tp)
{
return tp->rp.handler != NULL;
}
static int probe_event_enable(struct ftrace_event_call *call)
{
struct trace_probe *tp = (struct trace_probe *)call->data;
tp->flags |= TP_FLAG_TRACE;
if (probe_is_return(tp))
return enable_kretprobe(&tp->rp);
else
return enable_kprobe(&tp->rp.kp);
}
#define KPROBE_EVENT_SYSTEM "kprobes"
#endif // TRY_USE_KPROBES
// <<<<<<<<<<<<<<<<<<<<<<
static struct page *walk_page_table(unsigned long addr, struct task_struct *intask)
{
pgd_t *pgd;
pte_t *ptep, pte;
pud_t *pud;
pmd_t *pmd;
struct page *page = NULL;
struct mm_struct *mm = intask->mm;
callmodule_infoB->last_page_physaddr = 0ULL; // reset here, in case of early exit
printk(KBUILD_MODNAME ": walk_ 0x%lx ", addr);
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd) || pgd_bad(*pgd))
goto out;
printk(KBUILD_MODNAME ": Valid pgd ");
pud = pud_offset(pgd, addr);
if (pud_none(*pud) || pud_bad(*pud))
goto out;
printk( ": Valid pud");
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd) || pmd_bad(*pmd))
goto out;
printk( ": Valid pmd");
ptep = pte_offset_map(pmd, addr);
if (!ptep)
goto out;
pte = *ptep;
page = pte_page(pte);
if (page) {
callmodule_infoB->last_page_physaddr = (unsigned long long)page_to_phys(page);
printk( ": page frame struct is # %p; *virtual (page_address) # %p (is_vmalloc_addr %d virt_addr_valid %d virt_to_phys 0x%llx) page_to_pfn %lx page_to_phys 0x%llx", page, page_address(page), is_vmalloc_addr((void*)page_address(page)), virt_addr_valid(page_address(page)), (unsigned long long)virt_to_phys(page_address(page)), page_to_pfn(page), callmodule_infoB->last_page_physaddr);
}
//~ pte_unmap(ptep);
out:
printk("\n");
return page;
}
static void sample_hbp_handler(struct perf_event *bp,
struct perf_sample_data *data,
struct pt_regs *regs)
{
trace_printk(KBUILD_MODNAME ": hwbp hit: id [%llu]\n", bp->id );
//~ unregister_hw_breakpoint(bp);
}
// ----------------------
static int __init callmodule_init(void)
{
int ret = 0;
char userprog[] = "/path/to/wtest";
char *argv[] = {userprog, "2", NULL };
char *envp[] = {"HOME=/", "PATH=/sbin:/usr/sbin:/bin:/usr/bin", NULL };
struct task_struct *p;
struct task_struct *par;
struct task_struct *pc;
struct list_head *children_list_head;
struct list_head *cchildren_list_head;
char *state_str;
unsigned long offset, taddr;
int (*ptr_create_trace_probe)(int argc, char **argv);
struct trace_probe* (*ptr_find_probe_event)(const char *event, const char *group);
//int (*ptr_probe_event_enable)(struct ftrace_event_call *call); // not exported, copy
#if TRY_USE_KPROBES
char trcmd[256] = "";
struct trace_probe *tp;
#endif //TRY_USE_KPROBES
struct perf_event *sample_hbp, *sample_hbpb;
struct perf_event_attr attr, attrb;
printk(KBUILD_MODNAME ": > init %s\n", userprog);
ptr_create_trace_probe = (void *)0xc10d5120;
ptr_find_probe_event = (void *)0xc10d41e0;
print_symbol(KBUILD_MODNAME ": symbol # 0xc1065b60 is %s\n", 0xc1065b60); // shows wait_for_helper+0x0/0xb0
print_symbol(KBUILD_MODNAME ": symbol # 0xc1065ed0 is %s\n", 0xc1065ed0); // shows ____call_usermodehelper+0x0/0x90
print_symbol(KBUILD_MODNAME ": symbol # 0xc10d5120 is %s\n", 0xc10d5120); // shows create_trace_probe+0x0/0x590
ret = call_usermodehelperB(userprog, argv, envp, UMH_WAIT_EXEC);
if (ret != 0)
printk(KBUILD_MODNAME ": error in call to usermodehelper: %i\n", ret);
else
printk(KBUILD_MODNAME ": everything all right; pid %d (%d)\n", callmodule_pid, callmodule_infoB->pid);
tracing_on(); // earlier, so trace_printk of handler is caught!
// find the task:
rcu_read_lock();
p = pid_task(find_vpid(callmodule_pid), PIDTYPE_PID);
rcu_read_unlock();
if (p == NULL) {
printk(KBUILD_MODNAME ": p is NULL - exiting\n");
return 0;
}
state_str = (p->state==-1)?"unrunnable":((p->state==0)?"runnable":"stopped");
printk(KBUILD_MODNAME ": pid task a: %p c: %s p: [%d] s: %s\n",
p, p->comm, p->pid, state_str);
// find parent task:
par = p->parent;
if (par == NULL) {
printk(KBUILD_MODNAME ": par is NULL - exiting\n");
return 0;
}
state_str = (par->state==-1)?"unrunnable":((par->state==0)?"runnable":"stopped");
printk(KBUILD_MODNAME ": parent task a: %p c: %s p: [%d] s: %s\n",
par, par->comm, par->pid, state_str);
// iterate through parent's (and our task's) child processes:
rcu_read_lock(); // read_lock(&tasklist_lock);
list_for_each(children_list_head, &par->children){
p = list_entry(children_list_head, struct task_struct, sibling);
printk(KBUILD_MODNAME ": - %s [%d] \n", p->comm, p->pid);
if (p->pid == callmodule_pid) {
list_for_each(cchildren_list_head, &p->children){
pc = list_entry(cchildren_list_head, struct task_struct, sibling);
printk(KBUILD_MODNAME ": - - %s [%d] \n", pc->comm, pc->pid);
}
}
}
rcu_read_unlock(); //~ read_unlock(&tasklist_lock);
// NOTE: here p == callmodule_infoB->task !!
printk(KBUILD_MODNAME ": Trying to walk page table; addr task 0x%X ->mm ->start_code: 0x%08lX ->end_code: 0x%08lX \n", (unsigned int) callmodule_infoB->task, callmodule_infoB->task->mm->start_code, callmodule_infoB->task->mm->end_code);
walk_page_table(0x08048000, callmodule_infoB->task);
// 080483c0 is start of .text; 08048474 start of main; for objdump -S wtest
walk_page_table(0x080483c0, callmodule_infoB->task);
walk_page_table(0x08048474, callmodule_infoB->task);
if (callmodule_infoB->last_page_physaddr != 0ULL) {
printk(KBUILD_MODNAME ": physaddr ");
taddr = 0x080483c0; // .text
offset = taddr - callmodule_infoB->task->mm->start_code;
printk(": (0x%08lx ->) 0x%08llx ", taddr, callmodule_infoB->last_page_physaddr+offset);
taddr = 0x08048474; // main
offset = taddr - callmodule_infoB->task->mm->start_code;
printk(": (0x%08lx ->) 0x%08llx ", taddr, callmodule_infoB->last_page_physaddr+offset);
printk("\n");
#if TRY_USE_KPROBES // can't use this here (BUG: scheduling while atomic, if probe inserts)
//~ sprintf(trcmd, "p:myprobe 0x%08llx", callmodule_infoB->last_page_physaddr+offset);
// try symbol for c10bcf60 - tracing_on
sprintf(trcmd, "p:myprobe 0x%08llx", (unsigned long long)0xc10bcf60);
ret = traceprobe_command(trcmd, ptr_create_trace_probe); //create_trace_probe);
printk("%s -- ret: %d\n", trcmd, ret);
// try find probe and enable it (compiles, but untested):
tp = ptr_find_probe_event("myprobe", KPROBE_EVENT_SYSTEM);
if (tp != NULL) probe_event_enable(&tp->call);
#endif //TRY_USE_KPROBES
}
hw_breakpoint_init(&attr);
attr.bp_len = sizeof(long); //HW_BREAKPOINT_LEN_1;
attr.bp_type = HW_BREAKPOINT_X ;
attr.bp_addr = 0x08048474; // main
sample_hbp = register_user_hw_breakpoint(&attr, (perf_overflow_handler_t)sample_hbp_handler, p);
printk(KBUILD_MODNAME ": 0x08048474 id [%llu]\n", sample_hbp->id); //
if (IS_ERR((void __force *)sample_hbp)) {
int ret = PTR_ERR((void __force *)sample_hbp);
printk(KBUILD_MODNAME ": Breakpoint registration failed (%d)\n", ret);
//~ return ret;
}
hw_breakpoint_init(&attrb);
attrb.bp_len = sizeof(long);
attrb.bp_type = HW_BREAKPOINT_X ;
attrb.bp_addr = 0x08048475; // first instruction after main
sample_hbpb = register_user_hw_breakpoint(&attrb, (perf_overflow_handler_t)sample_hbp_handler, p);
printk(KBUILD_MODNAME ": 0x08048475 id [%llu]\n", sample_hbpb->id); //45
if (IS_ERR((void __force *)sample_hbpb)) {
int ret = PTR_ERR((void __force *)sample_hbpb);
printk(KBUILD_MODNAME ": Breakpoint registration failed (%d)\n", ret);
//~ return ret;
}
printk(KBUILD_MODNAME ": (( 0x08048000 is_vmalloc_addr %d virt_addr_valid %d ))\n", is_vmalloc_addr((void*)0x08048000), virt_addr_valid(0x08048000));
kill_pid(find_vpid(callmodule_pid), SIGCONT, 1); // resume/continue/restart task
state_str = (p->state==-1)?"unrunnable":((p->state==0)?"runnable":"stopped");
printk(KBUILD_MODNAME ": cont pid task a: %p c: %s p: [%d] s: %s\n",
p, p->comm, p->pid, state_str);
return 0;
}
static void __exit callmodule_exit(void)
{
tracing_off(); //corresponds to the user space /sys/kernel/debug/tracing/tracing_on file
printk(KBUILD_MODNAME ": < exit\n");
}
module_init(callmodule_init);
module_exit(callmodule_exit);
MODULE_LICENSE("GPL");

Resources