I'm currently making shared memory between userspace to kernel.
The following code is simple version of my mmap method.
for (j = 0; j < 3; j++){
RxFrame[j] = kmalloc(4096*6, GFP_KERNEL);
npages = (4096*6 - 1)/(PAGE_SIZE) + 1;
for(i = 0; i < npages*PAGE_SIZE; i += PAGE_SIZE){
printk("RxFrame[%d] PAGE_SIZE is %d\n", j, PAGE_SIZE);
SetPageReserved(virt_to_page(RxFrame[j] + i));
}
}
static int my_mmap(struct file *filp, struct vm_area_struct *vma){
unsigned long pfn = virt_to_phys(RxFrame[index]);
ret = remap_pfn_range(vma, vma->vm_start, pfn>>PAGE_SHIFT, len, vma->vm_page_prot);
if (ret < 0) {
pr_err("could not map the address area\n");
return -EIO;
}
return 0;
}
And following is my userspace application code which mmap used.
FRAME *RxFrame[3];
for(int i = 0; i < 3; i++){
if( (RxFrame[i] = mmap(0, 4096*6, PROT_WRITE | PROT_READ, MAP_SHARED, fd[i], 0)) == NULL ){
printf("mmap error\n");
return 0;
}
}
auto DoReceive = [&](const int index){
// read RxFrame[index][0].data
}
for(int i = 0; i < 3; i++){
thread[i] = std::thread(DoReceive , i);
usleep(1000);
}
My purpose was making three kernel buffers which dedicated data(different data will store in different kernel buffer) will store and use them with mmap in userspace with multi-thread.
when just one kernel buffer is used(data stored in just one kernel buffer), I can read RxFrame[index][0].data very well.
But when three kernel buffer is used(different data stored in separate kernel buffer), I can only read the first transmitted data.. for example if RxFrame[0]'s buffer received data, I can read RxFrame[0][0].data but can't read RxFrame[1][0].data and RxFrame[2][0].data..
Strange thing is that when received data is quite a lot at few moment, I can read RxFrame[1][0].data and RxFrame[2][0].data which looks like cached result!
I used volatile and other mmap option but I can't figure out why my mapped memory works like cached in multi-thread.
Any help will be thanks!
userspace-kernel shared memory with mmap works strange on multi-thread
I'm reading the source code in https://wayland-book.com/surfaces/shared-memory.html .
The author create a shared memory using shm_open(), and shm_unlink() it immediately, then ftruncate() the fd to a specific size, mmap() the fd and fill the region with pixels.
I'm so confused why the fd still available after shm_unlink().
according to the man page:
The operation of shm_unlink() is analogous to unlink(2): it removes a shared memory object name, and, once all processes have unmapped the object, de-allocates and destroys the contents of the associated memory region. After a successful shm_unlink(), attempts to shm_open() an object with the same name will fail (unless O_CREAT was specified, in which case a new, distinct object is created).
so shm_unlink() will cause the memory destroyed because there is no process mmap
the region. But how fd still avaliable?
here is the code:
static int
create_shm_file(void)
{
int retries = 100;
do {
char name[] = "/wl_shm-XXXXXX";
randname(name + sizeof(name) - 7);
--retries;
int fd = shm_open(name, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
if (fd >= 0) {
shm_unlink(name); // unlink immediately
return fd;
}
} while (retries > 0 && errno == EEXIST);
return -1;
}
static int
allocate_shm_file(size_t size)
{
int fd = create_shm_file();
if (fd < 0)
return -1;
int ret;
do {
ret = ftruncate(fd, size); //why the fd still available?
} while (ret < 0 && errno == EINTR);
if (ret < 0) {
close(fd);
return -1;
}
return fd;
}
//after above, there was mmap
uint32_t *data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
The code snippet works fine on my machine(Linux/x86-64)
int main()
{
char* addr;
int rc;
int fd;
const size_t PAGE_SIZE = 4096; // assuming the page size is 4096
char buf[PAGE_SIZE];
memset(buf, 'x', sizeof(buf));
// error checking is ignored, for demonstration purpose
fd = open("abc", O_RDWR | O_CREAT, S_IWUSR | S_IRUSR);
ftruncate(fd, 0);
write(fd, buf, 4090);
// the file size is less than one page, but we allocate 2 page address space
addr = mmap(NULL, PAGE_SIZE * 2, PROT_WRITE, MAP_SHARED, fd, 0);
// it would crash if we read/write from addr[4096]
// extend the size after mmap
ftruncate(fd, PAGE_SIZE * 2);
// now we can access(read/write) addr[4096]...addr[4096*2 -1]
munmap(addr, PAGE_SIZE * 2);
close(fd);
exit(EXIT_SUCCESS);
}
But POSIX says:
If the size of the mapped file changes after the call to mmap() as a result of some other operation on the mapped file, the effect of references to portions of the mapped region that correspond to added or removed portions of the file is unspecified.
So I guess this is not a portable way. But is it guaranteed to work on Linux?
I am trying to learn device drivers and I started with char device driver. I implemented a small program which is able to read/write from/to kernel buffer. Further, I tried to implement memory mapping and this is not working properly. When I am trying to read through a simple process by mapping my kernel module, it is giving me garbage value. Can any one help with this?
#include<linux/init.h>
#include<linux/module.h>
#include<linux/kernel.h> //printk()
#include<linux/errno.h>
#include<linux/types.h>
#include<linux/proc_fs.h>
#include<asm/uaccess.h> //copy_from,to_user
#include<linux/mm.h> //remap_pfn_range
//#include<linux/mman.h> //private_mapping_ok
#define BUFF_SIZE 128
#define DEV_NAME "MyDevice"
MODULE_LICENSE("GPL");
//Method declarations
int mod_open(struct inode *,struct file *);
int mod_release(struct inode *,struct file *);
ssize_t mod_read(struct file *,char *,size_t ,loff_t *);
ssize_t mod_write(struct file *,char *,size_t ,loff_t *);
int mod_mmap(struct file *, struct vm_area_struct *);
void mod_exit(void);
int mod_init(void);
//Structure that declares the usual file access functions
struct file_operations mod_fops = {
read: mod_read,
write: mod_write,
open: mod_open,
release: mod_release,
mmap: mod_mmap
};
static const struct vm_operations_struct mod_mem_ops = {
};
module_init(mod_init);
module_exit(mod_exit);
char *read_buf;
char *write_buf;
static int Major;
//static int Device_Open = 0;
int buffsize = 0;
int mod_init(void)
{
Major = register_chrdev(0,DEV_NAME,&mod_fops);
if(Major < 0)
{
printk(KERN_ALERT "Can not register %s. No major number alloted",DEV_NAME);
return Major;
}
//allocate memory to buffers
read_buf = kmalloc(BUFF_SIZE, GFP_KERNEL);
write_buf = kmalloc(BUFF_SIZE, GFP_KERNEL);
if(!read_buf || !write_buf)
{
mod_exit();
return -ENOMEM;
}
//reset buffers
memset(read_buf,0, BUFF_SIZE);
memset(write_buf,0, BUFF_SIZE);
printk(KERN_INFO "I was assigned major number %d. To talk to\n", Major);
printk(KERN_INFO "the driver, create a dev file with\n");
printk(KERN_INFO "'mknod /dev/%s c %d 0'.\n",DEV_NAME, Major);
printk(KERN_INFO "Try various minor numbers. Try to cat and echo to\n");
printk(KERN_INFO "the device file.\n");
printk(KERN_INFO "Remove the device file and module when done.\n");
return 0;
}
void mod_exit(void)
{
unregister_chrdev(Major,"memory");
if(read_buf) kfree(read_buf);
if(write_buf) kfree(write_buf);
printk(KERN_INFO "removing module\n");
}
int mod_mmap(struct file *filp, struct vm_area_struct *vma)
{
size_t size = vma->vm_end - vma->vm_start;
vma->vm_ops = &mod_mem_ops;
/* Remap-pfn-range will mark the range VM_IO */
if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, size, vma->vm_page_prot)) {
return -EAGAIN;
}
printk(KERN_INFO "VMA Open. Virt_addr: %lx, phy_addr: %lx\n",vma->vm_start, vma->vm_pgoff<<PAGE_SHIFT);
return 0;
}
ssize_t mod_read(struct file *filp, char *buf, size_t len, loff_t *f_pos)
{
ssize_t bytes;
if(buffsize < len)
bytes = buffsize;
else
bytes = len;
printk(KERN_INFO "Buffer size availabe: %d\n", buffsize);
printk(KERN_INFO "VMA Open. read buffer initial: %lx\n",read_buf);
if(bytes == 0)
return 0;
int retval = copy_to_user(buf,read_buf,bytes);
if(retval)
{
printk(KERN_INFO "copy_to_user fail");
return -EFAULT;
}
else
{
printk(KERN_INFO "copy_to_user succeeded\n");
buffsize -= bytes;
return bytes;
}
}
ssize_t mod_write( struct file *filp,char *buf, size_t len, loff_t *f_pos)
{
memset(read_buf,0,BUFF_SIZE);
memset(write_buf,0,BUFF_SIZE);
if(len > BUFF_SIZE)
{
printk(KERN_ALERT "Buffer not available. Writing only %d bytes.\n",BUFF_SIZE);
len = BUFF_SIZE;
}
printk(KERN_INFO "User space msg size: %d\n",len);
int retval = copy_from_user(read_buf,buf,len);
printk(KERN_INFO "read %d bytes as: %s\n", retval,read_buf);
// memcpy(write_buf,read_buf,len);
// printk(KERN_INFO "written: %s\n", write_buf);
buffsize = len;
return len;
}
int mod_open(struct inode *inode, struct file *filp){return 0;}
int mod_release(struct inode *inode, struct file *filp) {return 0;}
The program which is trying to access this device driver:
#include<stdio.h>
#include<sys/fcntl.h>
#include<stdlib.h>
#include<errno.h>
#include<string.h>
#include<sys/types.h>
#include<sys/mman.h>
int main(int argc,char *argv[])
{
int fd,n,len;
char *buff;
if(argc != 3)
{
printf("Too few arguments.\n");
exit(EXIT_FAILURE);
}
buff = (char *)malloc(128);
if(strcmp(argv[1],"read")==0)
{
if(-1 == (fd = open("/dev/MyDevice",O_RDONLY)))
{
printf("Device open fail. Error: %s",strerror(errno));
exit(EXIT_FAILURE);
}
memset(buff,0,128);
if(-1 == (buff = mmap(0,128,PROT_READ,MAP_SHARED | MAP_NORESERVE,fd,0)))
{
printf("Mapping failed. Error: %s\n", strerror(errno));
exit(EXIT_FAILURE);
}
/* if(-1 == (n = read(fd,buff,128)))
{
printf("Device read fail. Error: %s",strerror(errno));
exit(EXIT_FAILURE);
}
*/
printf("Read from device:\n%s\n",buff);
close(fd);
}
else if(strcmp(argv[1],"write")==0)
{
len = strlen(argv[2]);
if(-1 == (fd = open("/dev/MyDevice",O_WRONLY)))
{
printf("Device open fail. Error: %s",strerror(errno));
exit(EXIT_FAILURE);
}
if(-1 == (n = write(fd,argv[2],len)))
{
printf("Device write fail. Error: %s",strerror(errno));
exit(EXIT_FAILURE);
}
printf("Written %d bytes successfully.\n",n);
close(fd);
}
else
{
printf("Invalid argument..\n");
exit(EXIT_FAILURE);
}
return 0;
}
I got the error in my code. I was not mapping my buffer to vma->vm_pgoff. Just add following code before calling rmap_pfn_range, then this code will work fine
vma->vm_pgoff = virt_to_phys(read_buff)>>PAGE_SHIFT;
There are still several potential issues in your code although you found the root cause.
"vma->vm_pgoff = virt_to_phys(read_buff)>>PAGE_SHIFT"
It is not very good practice to program in this example, as basically you are overwriting a user file offset (in PAGE size unit). If your driver need to support mmap a memory offset, then obvious there is a issue. In this case, you can just pass virt_to_phys(read_buff)>>PAGE_SHIFT in place.
It is not recommended to use kmalloc to allocate the memory for remap purpose, as it is required to be page aligned, you can just use the kernel page APIs, like get_free_page to allocate the memory, further more, it is better to remap the memory in units of PAGE size, rather than 128 bytes here.
This is my first post so please let me know if there is any mistake from .
My aim is to get approx 150MBytes of data transfer from KERNEL to user space.
[This is because i am building an driver for DMA device on OMAP l138 to transfer and receive data between DMA DEVICE and FPGA]
Now in LINUX KERNEL i am allocating BUFFER of VARIABLE size using dma_alloc_coherent
Then the PHYSICAL address of this buffer i am passing to user space to be user as
OFFSET parameter to be used for mmap call from user space .
Then from data is copied and read back to and from from user space to kernel
This logic work fine till size of buffer is 4096. Above 4k the mmap fails and return "MAP_FAILED"
static int driver_mmap(struct file *f, struct vm_area_struct *vma)
{
u32bit ret;
u32bit size = (vma->vm_end)-(vma->vm_start);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (size > (NUM_PAGE*PAGE_SIZE)){
return(-1);
}
if ((ret = remap_pfn_range(vma,vma->vm_start,
(virt_to_phys((void *)krnl_area) >> PAGE_SHIFT),
size,vma->vm_page_prot)) < 0)
{
return ret;
}
printk("\nDVR:The MMAP returned %x to USER SAPCE \n",ret);
return 0;
}
//MMAP STEP 1
dmasrc_ptr = dma_alloc_coherent( NULL ,GLOBAL_BUFFER_SIZE , &dmasrc ,0);
if( !dmasrc_ptr ) {
printk(KERN_INFO "DMA_ALLOC_FAILED for the source buffer ...\n");
return -ENOMEM;
}else{
printk( "\n--->The address of SRC is %x..\n",dmasrc_ptr);
}
temp_source=dmasrc_ptr;
//MMAP STEP 2
// Round the allocated KERNEL MEMORY to the page bondary
krnl_area=(int *)((((unsigned long)dmasrc_ptr) + PAGE_SIZE - 1)&PAGE_MASK);
printk(KERN_CRIT "DVR:The KERNEL VIRTUAL ADDRS is %x..\n",krnl_area);
//MMAP STEP 3
// Marking the PAGES as RESERVED
for (i = 0; i < (NUM_PAGE * PAGE_SIZE); i+= PAGE_SIZE) {
SetPageReserved(virt_to_page(((unsigned long)krnl_area) + i));
//Application code part
while(1){
fflush(stdin);
fflush(stdout);
printf("\n\n\n----------------------------------------------------\n");
printf("USR:Please enter your requirement ");
printf("\n----------------------------------------------------\n");
printf("\t1----->GET_UPP_OFFSET\n");
printf("\t2----->UPP_MMAP_CALL\n");
printf("\t3----->IOCTL_UPP_WRITE\n");
printf("\t4----->IOCTL_UPP_READ\n");
printf("\n");
scanf("%d",&option);
printf("\nThe OPTION is %d..\n",option);
printf("\n");
switch(option){
case 1 :
{
offset=0;
ret = ioctl(dev_FD ,IOCTL_UPP_START, &info);
if (ret < 0) {
printf("dma buffer info ioctl failed\n");
}
offset = info.var;
printf("THE ADDRESS WE GOT IS %X..\n",offset);
}
break;
case 2 :
{
printf("THE OFFSET is %X..\n",offset);
mmap_Ptr= mmap(0,BUFFER_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, dev_FD, 0);
if (mmap_Ptr == MAP_FAILED){
printf("USR[UPP] :MMAP FAiled \n\n");
close(dev_FD);
exit(-1);
}
printf("THE MMAP address is %X..\n",mmap_Ptr);
}
break;
case 3:
{
struct upp_struct user_local_struct;
printf("\n***************************************************\n");
for (i = 0; i <(1024);i++) {
*(mmap_Ptr+i)=test_var;
printf("WR:%X ",*(mmap_Ptr+i));
//test_var++;
}
ioctl(dev_FD , IOCTL_UPP_WRITE ,&user_local_struct);
printf("\n***************************************************\n\n\n");
for(i=0;i<20402;i++){
//NOP
}
//test_var=0x00;
}
break;
case 4:
{
struct upp_struct user_local_struct;
ioctl(dev_FD , IOCTL_UPP_READ,&user_local_struct);
for(i=0;i<20402;i++){
//NOP
}
printf("\n***************************************************\n");
for (i = 0; i <(1024);i++) {
printf("RD:%X",*(mmap_Ptr+i));
}
printf("\n***************************************************\n\n\n");
}
break;
default:
{
printf("USR:You have entered an wrong option \n");
printf("\nUSR:CLosing the FILE ENTERIES ...\n");
munmap(mmap_Ptr,BUFFER_SIZE);
free(source_ptr);
free(dest_ptr);
close(dev_FD);
exit(0);
}
break;
} //END OF SWITCH LOOP
} //END OF WHILE LOOP
use get_free_pages to allocate multiple pages, or use vmalloc but you need call remap_pfn_range at every page basis as vmalloc-ed physical memory could be not physically continuous.