My platform:
Ubuntu 17.10 32-bit (Vbox VM)
Qt Creator 3.5.1 (opensource)
Qt 5.5.1 (GCC 4.9.1 20140922 (Red Hat 4.9.1-10), 32 bit
I am trying to invoke a multithreaded program (with arguments) using QProcess.start().
My program runs fine on terminal, i.e. periodically prints on stdout.
Using a TextEdit to log the stdout/stderr of the program I have connected QProcess readyReadStandardOutput/Error signals.
The stdout/stderr that comes from the main thread of the program is correctly shown on the TextEdit, the rest of the output (the one from all the other threads) is not shown.
EDIT
On the main thread an HTTP server is listening.
If a HTTP request is performed by browser at the url "127.0.0.1:32001" (port 32001 is hard coded in the QT code), when the HTTP request is received the program appends the HTTP packet and all the pending output from the other thread (moduleThread) to the TextEdit, so it could be a problem of flushing.
main.c
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <pthread.h>
#include <semaphore.h>
#include <sys/time.h>
#include <sys/signal.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "http_server.h"
static pthread_t moduleThr;
static pthread_attr_t moduleThread_attr;
static bool one_second_elapsed;
static sem_t oneSecondSem;
void *moduleThread(void *arg0)
{
bool one_second_elapsed_local;
while (1)
{
sem_wait(&oneSecondSem);
one_second_elapsed_local = one_second_elapsed;
one_second_elapsed = false;
sem_post(&oneSecondSem);
if (one_second_elapsed_local)
fprintf(stdout, "Hello world each second!\r\n");
usleep(50000);
}
}
static void oneSecElapsed(int signum)
{
sem_wait(&oneSecondSem);
one_second_elapsed = true;
sem_post(&oneSecondSem);
}
static void TIMER_1sec_init(void)
{
struct sigaction sa;
struct itimerval timer;
/* Install timer_handler as the signal handler for SIGALRM. */
memset (&sa, 0, sizeof (sa));
sa.sa_handler = &oneSecElapsed;
sigaction (SIGALRM, &sa, NULL);
/* Configure the timer to expire after 1 sec... */
timer.it_value.tv_sec = 1;
timer.it_value.tv_usec = 0;
/* ... and every 1 sec after that... */
timer.it_interval.tv_sec = 1;
timer.it_interval.tv_usec = 0;
/* Start a virtual timer. It counts down whenever this process is
executing. */
setitimer (ITIMER_REAL, &timer, NULL);
}
/*
* ======== main ========
*/
int main(int argc, char *argv[])
{
const char *tcpport;
if (argc > 2)
{
fprintf(stdout, "id = %s\r\n", argv[1]);
fprintf(stdout, "tcpport = %s\r\n", argv[2]);
tcpport = argv[2];
}
else
exit(-1);
sem_init(&oneSecondSem, 0, 1);
one_second_elapsed = false;
/* Create thread quadro manager */
pthread_attr_init(&moduleThread_attr);
pthread_attr_setstacksize(&moduleThread_attr, 2048);
pthread_create(&moduleThr, &moduleThread_attr, moduleThread, NULL);
TIMER_1sec_init();
HTTPSERVER_init(tcpport);
/* should never return */
return (0);
}
http_server.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h> /* memset() */
#include <stdbool.h>
#include <signal.h>
#include <sys/socket.h>
#include <sys/signal.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <netdb.h>
#include "module.h"
typedef struct { char *name, *value; } header_t;
static header_t reqhdr[17] = { {"\0", "\0"} };
//#define PORT "32001" /* Port to listen on */
#define BACKLOG 10 /* Passed to listen() */
static char *buf;
static char *method; // "GET" or "POST"
static char *uri; // "/index.html" things before '?'
static char *qs; // "a=1&b=2" things after '?'
static char *prot; // "HTTP/1.1"
static char *payload; // for POST
static int payload_size;
// get request header
char *request_header(const char* name)
{
header_t *h = reqhdr;
while(h->name) {
if (strcmp(h->name, name) == 0) return h->value;
h++;
}
return NULL;
}
void handle(int newsock)
{
//static int count = 0;
/* recv(), send(), close() */
int rcvd;
buf = malloc(65535);
rcvd=recv(newsock, buf, 65535, 0);
if (rcvd<0) // receive error
fprintf(stderr,("recv() error\n"));
else if (rcvd==0) // receive socket closed
fprintf(stderr,"Client disconnected unexpectedly.\n");
else // message received
{
buf[rcvd] = '\0';
//fputs(buf, stdout);
//fprintf(stdout, "count = %d\n", count);
method = strtok(buf, " \t\r\n");
uri = strtok(NULL, " \t");
prot = strtok(NULL, " \t\r\n");
fprintf(stderr, "\x1b[32m + [%s] %s\x1b[0m\n", method, uri);
if ((qs = strchr(uri, '?')))
{
*qs++ = '\0'; //split URI
} else {
qs = uri - 1; //use an empty string
}
header_t *h = reqhdr;
char *t = (char *)reqhdr;
char *t2;
while(h < reqhdr+16) {
char *k,*v;
k = strtok(NULL, "\r\n: \t");
if (!k)
break;
v = strtok(NULL, "\r\n");
while(*v && *v==' ')
v++;
h->name = k;
h->value = v;
h++;
fprintf(stderr, "[H] %s: %s\n", k, v);
t = v + 1 + strlen(v);
if (t[1] == '\r' && t[2] == '\n')
break;
}
t+=2;
t++; // now the *t shall be the beginning of user payload
t2 = request_header("content-length"); // and the related header if there is
payload = t;
payload_size = t2 ? atol(t2) : (rcvd-(t-buf));
fprintf(stdout, "-- payload len = %d >", payload_size);
fputs(payload, stdout);
fprintf(stdout, "<\r\n");
if (strcmp(method, "GET") == 0)
{
fprintf(stdout, "\nit's a GET!\r\n");
}
else if (strcmp(method, "POST") == 0)
{
fprintf(stdout, "\nit's a POST!\r\n");
}
sprintf(buf, "HTTP/1.1 200 OK\r\n\r\n");
send(newsock, buf, strlen(buf), 0);
close(newsock);
free(buf);
}
}
int HTTPSERVER_init(const char *tcpport)
{
int sock;
struct addrinfo hints, *res;
int reuseaddr = 1; /* True */
sigset_t sigset, oldset;
sigfillset(&sigset);
pthread_sigmask(SIG_BLOCK, &sigset, &oldset);
/* Get the address info */
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
if (getaddrinfo(NULL, tcpport, &hints, &res) != 0) {
perror("getaddrinfo");
return 1;
}
/* Create the socket */
sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sock == -1) {
perror("socket");
return 1;
}
/* Enable the socket to reuse the address */
if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &reuseaddr, sizeof(int)) == -1) {
perror("setsockopt");
return 1;
}
/* Bind to the address */
if (bind(sock, res->ai_addr, res->ai_addrlen) == -1) {
perror("bind");
return 1;
}
/* Listen */
if (listen(sock, BACKLOG) == -1) {
perror("listen");
return 1;
}
freeaddrinfo(res);
/* Main loop */
while (1) {
printf("waiting on accept\n");
fflush(stdout);
socklen_t size = sizeof(struct sockaddr_in);
struct sockaddr_in their_addr;
int newsock = accept(sock, (struct sockaddr*)&their_addr, &size);
if (newsock == -1) {
perror("accept");
}
else {
printf("Got a connection from %s on port %d\n",
inet_ntoa(their_addr.sin_addr), htons(their_addr.sin_port));
handle(newsock);
}
}
close(sock);
return 0;
}
As pointed out by #eyllanesc, the pending strings "Hello world each second!\r\n" come out eventually after several seconds.
The moduleThread shoud be a thread and not a different process: this is verified by commadn "ps aux" that shows only "../program/testApp test 32001".
Could any of you kindly give me a hint on what is going on here please?
Thanks for your time,
Francesco
Related
As title, but the measurement result is unreasonable. Let me describe the current status.
I'm using syscall getuid as measurement target, I started by measureing the complete overhead with two clock_gettime bounded around, then measure the entry (what SYSCALL instruction does before executing the actual getuid code) and leaving overhead saparately (with eBPF program hook onto the entry and leaving point).
The result for the complete overhead is ~65ns, and regarding to the entry and leaving overhead, it's ~77ns and ~70ns respectively.
It's obvious that my measurement has some additional overhead except the typical overhead. However, it's weird that since clock_gettime is a vDSO syscall, it should barely have noticeable overhead. And BPF, which is a lightweight instrumental tool (JIT-ed and etc.) these day in Linux, shouldn't have noticeable overhead too.
Is there anyone have idea what additional overhead my measurement incurs?
Following is my measurement code:
userland (measuring the return-from-kernel overhead):
#define _GNU_SOURCE
#include <bpf.h>
#include <libbpf.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <string.h>
#include <asm/errno.h>
#include <linux/if_link.h>
#include <errno.h>
#include <sys/resource.h>
#include <unistd.h>
#include <asm/unistd.h>
#include <time.h>
#include <linux/perf_event.h>
#include <linux/hw_breakpoint.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <sched.h>
#define likely(x) __builtin_expect((x),1)
#define unlikely(x) __builtin_expect((x),0)
#define TEST_CNT 1000000
#define BPF_FILE_NAME "mkern.o"
#define BPF_MAP_NAME "msys"
static inline int sys_perf_event_open(struct perf_event_attr *attr, pid_t pid,
int cpu, int group_fd,
unsigned long flags)
{
attr->size = sizeof(*attr);
return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags);
}
static int attach_kprobe(int prog_fd)
{
int err, fd, id;
char buf[32];
struct perf_event_attr attr = {};
err = system("echo 'r:kp_sys_batch __x64_sys_getuid' > /sys/kernel/debug/tracing/kprobe_events");
if (err < 0) {
fprintf(stderr, "Failed to create kprobe, error '%s'\n", strerror(errno));
return -1;
}
fd = open("/sys/kernel/debug/tracing/events/kprobes/kp_sys_batch/id", O_RDONLY, 0);
if (fd < 0) {
fprintf(stderr, "Failed to open event %s\n", "sys_batch");
return -1;
}
err = read(fd, buf, sizeof(buf));
if (err < 0 || err >= sizeof(buf)) {
fprintf(stderr, "read from '%s' failed '%s'\n", "sys_batch", strerror(errno));
return -1;
}
close(fd);
buf[err] = 0;
id = atoi(buf);
attr.config = id;
attr.type = PERF_TYPE_TRACEPOINT;
attr.sample_type = PERF_SAMPLE_RAW;
attr.sample_period = 1;
attr.wakeup_events = 1;
fd = sys_perf_event_open(&attr, 0/*this process*/, -1/*any cpu*/, -1/*group leader*/, 0);
if (fd < 0) {
perror("sys_perf_event_open");
fprintf(stderr, "Failed to open perf_event (id: %llu)\n", attr.config);
return -1;
}
err = ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
if (err < 0) {
fprintf(stderr, "ioctl PERF_EVENT_IOC_ENABLE failed err %s\n",
strerror(errno));
return -1;
}
err = ioctl(fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
if (err < 0) {
fprintf(stderr, "ioctl PERF_EVENT_IOC_SET_BPF failed: %s\n",
strerror(errno));
return -1;
}
return 0;
}
static void maxi_memlock_rlimit(void)
{
struct rlimit rlim_new = {
.rlim_cur = RLIM_INFINITY,
.rlim_max = RLIM_INFINITY,
};
if (setrlimit(RLIMIT_MEMLOCK, &rlim_new)) {
fprintf(stderr, "Failed to increase RLIMIT_MEMLOCK limit!\n");
exit(-1);
}
}
static int find_map_fd(struct bpf_object *bpf_obj, const char *mapname)
{
struct bpf_map *map;
int map_fd = -1;
map = bpf_object__find_map_by_name(bpf_obj, mapname);
if (!map) {
fprintf(stderr, "Failed finding map by name: %s\n", mapname);
exit(-1);
}
map_fd = bpf_map__fd(map);
return map_fd;
}
int main(int argc, char **argv)
{
int bpf_map_fd;
int bpf_prog_fd = -1;
int err;
int key = 0;
struct timespec tp;
struct bpf_object *bpf_obj;
struct reals map;
struct bpf_prog_load_attr xattr = {
.prog_type = BPF_PROG_TYPE_KPROBE,
.file = BPF_FILE_NAME,
};
maxi_memlock_rlimit();
err = bpf_prog_load_xattr(&xattr, &bpf_obj, &bpf_prog_fd);
if (err) {
fprintf(stderr, "Failed loading bpf object file\n");
exit(-1);
}
if (attach_kprobe(bpf_prog_fd)) {
fprintf(stderr, "Failed attaching kprobe\n");
exit(-1);
}
bpf_map_fd = find_map_fd(bpf_obj, BPF_MAP_NAME);
if (find_map_fd < 0) {
fprintf(stderr, "Failed finding map fd\n");
exit(-1);
}
/* warm up */
for (int i = 0; i < TEST_CNT; i++) {
syscall(__NR_getuid); /* dummy call */
clock_gettime(CLOCK_MONOTONIC, &tp);
if (unlikely(bpf_map_lookup_elem(bpf_map_fd, &key, &map))) {
fprintf(stderr, "Failed to lookup map element\n");
perror("lookup");
exit(-1);
}
}
uint64_t delta = 0;
for (int i = 0; i < TEST_CNT; i++) {
syscall(__NR_getuid); /* dummy call */
clock_gettime(CLOCK_MONOTONIC, &tp);
if (unlikely(bpf_map_lookup_elem(bpf_map_fd, &key, &map))) {
fprintf(stderr, "Failed to lookup map element\n");
perror("lookup");
exit(-1);
}
delta += (1000000000 * tp.tv_sec + tp.tv_nsec) - map.ts;
}
printf("avg: %fns\n", (double) delta / TEST_CNT);
return 0;
}
user land (measuring the enter-kernel overhead, almost same as the above, except what I pointed out):
err = system("echo 'p:kp_sys_batch sys_batch' > /sys/kernel/debug/tracing/kprobe_events");
...
clock_gettime(CLOCK_MONOTONIC, &tp);
syscall(__NR_getuid); /* dummy call */
...
delta += map.ts - (1000000000 * tp.tv_sec + tp.tv_nsec);
kernel land:
SEC("getuid")
int kp_sys_batch(struct pt_regs *ctx)
{
__u32 i = 0;
struct reals *r;
r = bpf_map_lookup_elem(&reals, &i);
if (!r)
return 1;
r->ts = bpf_ktime_get_ns();
return 0;
}
Except the additional overhead I mentioned above, inside the return-from-kernel measurement code, if the echo 'r:kp_sys_batch sys_batch' is changed to echo 'p:kp_sys_batch sys_batch' (which means that the measurement would take the syscall execution overhead into account), the result would be ~48ns, this means that the result includes overhead of syscall execution and return-from-kernel. Any idea why this could be only ~48ns?
Thanks!
My operating system is Centos 6.4.
For select the linux programmer's manual says: "select may update the timeout parameter to indicate how much time was left...."
I wonder why it is 'may' rather than 'must'? Is there a system version or a kernel version?
I make a test.
test_code01.cpp
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/time.h>
#include <pthread.h>
#define BUFSIZE 1024
struct timeval g_timeout;
int main(int argc, char* argv[])
{
int sock_clientfd, ret_recvsize, i;
struct sockaddr_in dest, mine;
char buffer[BUFSIZE + 1];
// create socket fd
if ((sock_clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
perror("Socket");
exit(EXIT_FAILURE);
}
// init server address that client will connetct to.
bzero(&dest, sizeof(dest));
dest.sin_family = AF_INET;
dest.sin_port = htons(9567);
if(argc != 2)
{
printf("Usage: %s <dest ip>\n", argv[0]);
printf("Usage: %s 127.0.0.1\n", argv[0]);
return -1;
}
printf("-----\n");
if (inet_aton(argv[1], (struct in_addr *) &dest.sin_addr.s_addr) == 0)
{
perror(argv[1]);
exit(1);
}
// connect to server
printf("will connect!\n");
if (connect(sock_clientfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
{
perror("Connect ");
exit(EXIT_FAILURE);
}
int ret_select = 0;
fd_set readfds;
FD_SET(sock_clientfd, &readfds);
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
while(1)
{
ret_select = select(sock_clientfd + 1, &readfds, NULL, NULL, &g_timeout);
// Becasue select can update timeout, we need to set value for timeout.
// g_timeout.tv_sec = 3;
// g_timeout.tv_usec = 0;
// Recover sock_clienfd state
FD_SET(sock_clientfd, &readfds);
if(ret_select == 0)
{
printf("select wait timeout.\n");
continue;
}
bzero(buffer, BUFSIZE + 1);
ret_recvsize = recv(sock_clientfd, buffer, BUFSIZE, 0);
if(ret_recvsize > 0)
{
printf("get %d message:%s", strlen(buffer), buffer);
ret_recvsize=0;
}
else
{
printf("no data from server\n");
}
}
// close sock_clientfd
close(sock_clientfd);
return 0;
}
I execute this test_code01.cpp, and the result select wait for 3 seconds for only first times.In my opinion, the select modify timeout.So I modify code follow:
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/time.h>
#include <pthread.h>
#define BUFSIZE 1024
// in order to facilitate the ovservation data
// the timeout is set to global varible
struct timeval g_timeout;
void* thredproc_looktimeout(void*)
{
while(1)
{
printf("left time: %ds %dms\n", g_timeout.tv_sec, g_timeout.tv_usec);
}
}
int main(int argc, char* argv[])
{
int sock_clientfd, ret_recvsize, i;
struct sockaddr_in dest, mine;
char buffer[BUFSIZE + 1];
// create socket fd
if ((sock_clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
perror("Socket");
exit(EXIT_FAILURE);
}
// init server address that client will connetct to.
bzero(&dest, sizeof(dest));
dest.sin_family = AF_INET;
dest.sin_port = htons(9567);
if(argc != 2)
{
printf("Usage: %s <dest ip>\n", argv[0]);
printf("Usage: %s 127.0.0.1\n", argv[0]);
return -1;
}
printf("-----\n");
if (inet_aton(argv[1], (struct in_addr *) &dest.sin_addr.s_addr) == 0)
{
perror(argv[1]);
exit(1);
}
// connect to server
printf("will connect!\n");
if (connect(sock_clientfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
{
perror("Connect ");
exit(EXIT_FAILURE);
}
int ret_select = 0;
fd_set readfds;
FD_SET(sock_clientfd, &readfds);
printf("fdset add fd start+++\n");
for(int i = 0; i < 2000; i++)
{
FD_SET(1025+i, &readfds);
}
printf("%d--%s\n", errno, strerror(errno));
printf("fdset add 2000 fd finish...\n");
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
/**
* look timeout left time by create thread
**/
pthread_t tid;
int ret_pthreadcreate = pthread_create(&tid, NULL, thredproc_looktimeout, NULL);
printf("pthread_create return value is %d\n", ret_pthreadcreate);
while(1)
{
ret_select = select(sock_clientfd + 1, &readfds, NULL, NULL, &g_timeout);
// Becasue select can update timeout, we need to set value for timeout.
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
// Recover sock_clienfd state
FD_SET(sock_clientfd, &readfds);
if(ret_select == 0)
{
printf("select wait timeout.\n");
continue;
}
bzero(buffer, BUFSIZE + 1);
ret_recvsize = recv(sock_clientfd, buffer, BUFSIZE, 0);
if(ret_recvsize > 0)
{
printf("get %d message:%s", strlen(buffer), buffer);
ret_recvsize=0;
}
else
{
printf("no data from server\n");
}
}
// close sock_clientfd
close(sock_clientfd);
return 0;
}
I think we should see a change in timeout in thredproc_looktimeout.But no change.I don't know why?
The manual page means that on return the timeout might be updated to indicate the amount of time left. It absolutely doesn't mean that the timeout will be updated while the wait is still in progress.
And even if it was, there is no possible way you could detect it. Your attempt to do with threads won't work because it has no synchronization of any kind. Among other problems, the compiler could optimize the while loop to read g_timeout only once. And there is no conceivable synchronization mechanism you could use.
Because a valid timeout may also be
null in the case you want to wait forever
both fields of the timeval are set to 0, select returns immediately
In both those cases subtraction doesn't make sense, hence no "must."
Your addition of code makes this a duplicate of https://stackoverflow.com/a/7382740/4080377. You're seeing a copy of the variable, not the real thing
I get a answer.In my system, select actualy modify timeout value.I don't fully understand select may update timeout before.
Now,
(1)all_time:it is initial value of timeout.For example, timeout.tv_sec is equal to three and timeout.tv_usec is equal to zero.
(2)consume_time:it is the time value that from start to return of select to spended.
(3)left_time:it does't use time for select.
Linux manual actually implies the meaning that left_time add consume_time is equal to all_time.
Why it "may update".I think Tom Scanlan's answer is right.
The follow code:
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/time.h>
#include <pthread.h>
#define BUFSIZE 1024
// in order to facilitate the ovservation data
// the timeout is set to global varible
struct timeval g_timeout;
void* thredproc_looktimeout(void*)
{
while(1)
{
if(g_timeout.tv_sec != 3 && g_timeout.tv_usec != 0)
{
printf("left time: %ds %dms\n", g_timeout.tv_sec, g_timeout.tv_usec);
}
}
}
int main(int argc, char* argv[])
{
int sock_clientfd, ret_recvsize, i;
struct sockaddr_in dest, mine;
char buffer[BUFSIZE + 1];
// create socket fd
if ((sock_clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
perror("Socket");
exit(EXIT_FAILURE);
}
// init server address that client will connetct to.
bzero(&dest, sizeof(dest));
dest.sin_family = AF_INET;
dest.sin_port = htons(9567);
if(argc != 2)
{
printf("Usage: %s <dest ip>\n", argv[0]);
printf("Usage: %s 127.0.0.1\n", argv[0]);
return -1;
}
printf("-----\n");
if (inet_aton(argv[1], (struct in_addr *) &dest.sin_addr.s_addr) == 0)
{
perror(argv[1]);
exit(1);
}
// connect to server
printf("will connect!\n");
if (connect(sock_clientfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
{
perror("Connect ");
exit(EXIT_FAILURE);
}
int ret_select = 0;
fd_set readfds;
FD_SET(sock_clientfd, &readfds);
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
/**
* look timeout left time by create thread
**/
pthread_t tid;
int ret_pthreadcreate = pthread_create(&tid, NULL, thredproc_looktimeout, NULL);
printf("pthread_create return value is %d\n", ret_pthreadcreate);
while(1)
{
ret_select = select(sock_clientfd + 1, &readfds, NULL, NULL, &g_timeout);
// Recover sock_clienfd state
FD_SET(sock_clientfd, &readfds);
if(ret_select == 0)
{
printf("select wait timeout.\n");
// Becasue select can update timeout, we need to set value for timeout.
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
continue;
}
bzero(buffer, BUFSIZE + 1);
ret_recvsize = recv(sock_clientfd, buffer, BUFSIZE, 0);
if(ret_recvsize > 0)
{
printf("get %d message:%s", strlen(buffer), buffer);
ret_recvsize=0;
}
else
{
printf("no data from server\n");
}
sleep(3);
// Becasue select can update timeout, we need to set value for timeout.
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
}
// close sock_clientfd
close(sock_clientfd);
return 0;
}
This code can get no-zero value in thredproc_looktimeout thread function.
In addition, such is in case, As David Schwartz said: it absolutely doesn't mean that the timeout will be updated while the wait is still in progress.
You can try this code. I added some code and some comment.
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/time.h>
#include <pthread.h>
#define BUFSIZE 1024
// in order to facilitate the ovservation data
// the timeout is set to global varible
struct timeval g_timeout;
void* thredproc_looktimeout(void*)
{
while(1)
{
if(g_timeout.tv_sec != 3 && g_timeout.tv_usec != 0)
{
printf("left time: %ds %dms\n", g_timeout.tv_sec, g_timeout.tv_usec);
}
}
}
int main(int argc, char* argv[])
{
int sock_clientfd, ret_recvsize, i;
struct sockaddr_in dest, mine;
char buffer[BUFSIZE + 1];
// create socket fd
if ((sock_clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
{
perror("Socket");
exit(EXIT_FAILURE);
}
// init server address that client will connetct to.
bzero(&dest, sizeof(dest));
dest.sin_family = AF_INET;
dest.sin_port = htons(9567);
if(argc != 2)
{
printf("Usage: %s <dest ip>\n", argv[0]);
printf("Usage: %s 127.0.0.1\n", argv[0]);
return -1;
}
printf("-----\n");
if (inet_aton(argv[1], (struct in_addr *) &dest.sin_addr.s_addr) == 0)
{
perror(argv[1]);
exit(1);
}
// connect to server
printf("will connect!\n");
if (connect(sock_clientfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
{
perror("Connect ");
exit(EXIT_FAILURE);
}
int ret_select = 0;
fd_set readfds;
/*new code added*/
FD_ZERO(&readfs);
/* before you set any fd in readfs, you should set 0 all fds in reads. For security, because some times
readfs can alloced some old trash data place, and a chance in readfs may some data.*/
FD_SET(sock_clientfd, &readfds);
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
/**
* look timeout left time by create thread
**/
pthread_t tid;
int ret_pthreadcreate = pthread_create(&tid, NULL, thredproc_looktimeout, NULL);
printf("pthread_create return value is %d\n", ret_pthreadcreate);
while(1)
{
ret_select = select(sock_clientfd + 1, &readfds, NULL, NULL, &g_timeout);
if(ret_select == -1){
/*return some error or what you want here. Because if ret_select is equal to 1. That is a problem*/
}
// Recover sock_clienfd state
/* FD_SET(sock_clientfd, &readfds);*/
/* you don't need to same fd set again. That shuold be always some data ready to read()*/
if(ret_select == 0)
{
printf("select wait timeout.\n");
// Becasue select can update timeout, we need to set value for timeout.
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
continue;
}
/*bzero(buffer, BUFSIZE + 1);
ret_recvsize = recv(sock_clientfd, buffer, BUFSIZE, 0);
if(ret_recvsize > 0)
{
printf("get %d message:%s", strlen(buffer), buffer);
ret_recvsize=0;
}
else
{
printf("no data from server\n");
}*/
/*Here you can check just sock_clienfd. If that is true, some data ready to read on socket.*/
if(FD_ISSET(sock_clientfd,&readfs){
bzero(buffer, BUFSIZE + 1);
ret_recvsize = recv(sock_clientfd, buffer, BUFSIZE, 0);
}
/*sleep(3);*/ /*your select() function already wait 3 second before go down*/
// Becasue select can update timeout, we need to set value for timeout.
g_timeout.tv_sec = 3;
g_timeout.tv_usec = 0;
}
// close sock_clientfd
close(sock_clientfd);
return 0;
}
I have two programs, a server and a client. The server opens a file, writes data to it, and then send its file descriptor to the client over a unix domain socket. Everything works fine untill I introduce a socat proxy in between.
socat -x -v UNIX-LISTEN:/tmp/unixSockSendFe,mode=775,reuseaddr,fork UNIX-CONNECT:/tmp/unixSockSendFd
Explanation
The server listens on /tmp/unixSockSendFd, socat connects to it(UNIX-CONNECT:/tmp/unixSockSendFd), and creates another Unix domain socket(UNIX-LISTEN:/tmp/unixSockSendFe,mode=775,reuseaddr,fork), on which the client connects. Any communication between the client and server gets relayed through socat, which prints the bytes sent in their binary (-x option), and ascii (-v option) form.
If I don't use socat, and client directly connects to server(on /tmp/unixSockSendFd socket), everything works fine, but when socat is used as a proxy, the client crashes with a segmentation fault.
Server
/*Server code - sendfd.c*/
#include <sys/socket.h>
#include <sys/un.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
char *socket_path = "/tmp/unixSockSendFd";
char *file="/tmp/abcd.txt" ;/*file whose fd is to be sent*/
int sendfd(int sock, int fd);
int recvfd(int s);
char data[]="sahil\0";
int main(int argc, char *argv[]) {
struct sockaddr_un addr;
char buf[100];
buf[0]='\n';
int fd,rc,confd;
int fd_to_send;
int temp,len;
temp=1;
fd_to_send=open(file,O_TRUNC|O_RDWR|O_CREAT,S_IRWXU|S_IRWXG|S_IRWXO);
if(fd_to_send==-1)
{
perror("file open error");
return -1;
}
if (argc > 1) socket_path=argv[1];
if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
perror("socket error");
exit(-1);
}
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
if (*socket_path == '\0') {
*addr.sun_path = '\0';
strncpy(addr.sun_path+1, socket_path+1, sizeof(addr.sun_path)-2);
} else {
strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path)-1);
}
unlink(socket_path);
if(bind(fd,(struct sockaddr*)&addr,sizeof(addr))==-1){
perror("bind error");
return -1;
}
/*Writing data to file before sending fd*/
len=write(fd_to_send,data,(int)strlen(data));
fsync(fd_to_send);
printf("(len=%d)data written in file(content between ## marks) ##%s##\n",len,data);
listen(fd,1);
for(;;){
confd=accept(fd,NULL,NULL);
if(confd==-1)
{
perror("accept error");
continue;
}
else{
printf("new client connected ... sending fd ... \n");
sendfd(confd,fd_to_send);
close(confd);
}
}
return 0;
}
int sendfd(int sock, int fd)
{
struct msghdr hdr;
struct iovec data;
char cmsgbuf[CMSG_SPACE(sizeof(int))];
char dummy = '*';
data.iov_base = &dummy;
data.iov_len = sizeof(dummy);
memset(&hdr, 0, sizeof(hdr));
hdr.msg_name = NULL;
hdr.msg_namelen = 0;
hdr.msg_iov = &data;
hdr.msg_iovlen = 1;
hdr.msg_flags = 0;
hdr.msg_control = cmsgbuf;
hdr.msg_controllen = CMSG_LEN(sizeof(int));
struct cmsghdr* cmsg = CMSG_FIRSTHDR(&hdr);
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
*(int*)CMSG_DATA(cmsg) = fd;
//memcpy((CMSG_DATA(cmsg)), &fd, sizeof(fd)); -- from ivshmem server code - this too works instead of previous line
int n = sendmsg(sock, &hdr, 0);
if(n == -1)
printf("sendmsg() failed: %s (socket fd = %d)\n", strerror(errno), sock);
return n;
}
int recvfd(int s)
{
int n;
int fd;
char buf[1];
struct iovec iov;
struct msghdr msg;
struct cmsghdr *cmsg;
char cms[CMSG_SPACE(sizeof(int))];
iov.iov_base = buf;
iov.iov_len = 1;
memset(&msg, 0, sizeof msg);
msg.msg_name = 0;
msg.msg_namelen = 0;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_control = (caddr_t)cms;
msg.msg_controllen = sizeof cms;
if((n=recvmsg(s, &msg, 0)) < 0)
return -1;
if(n == 0){
perror("unexpected EOF");
return -1;
}
cmsg = CMSG_FIRSTHDR(&msg);
memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
return fd;
}
Client
/*Client code - recvfd.c*/
#include <sys/socket.h>
#include <sys/un.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <errno.h>
char *socket_path = "/tmp/unixSockSendFe";
int sendfd(int sock, int fd);
int recvfd(int s);
int fd_received;
int main(int argc, char *argv[]) {
struct sockaddr_un addr;
char buf[100];
buf[0]='\n';
int fd,rc,confd;
int temp,len;
temp=1;
if (argc > 1) socket_path=argv[1];
if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
perror("socket error");
exit(-1);
}
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
if (*socket_path == '\0') {
*addr.sun_path = '\0';
strncpy(addr.sun_path+1, socket_path+1, sizeof(addr.sun_path)-2);
} else {
strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path)-1);
}
if(connect(fd,(struct sockaddr*)&addr,sizeof(addr))==-1)
{
perror("connect error");
exit(-1);
}
fd_received=recvfd(fd);
lseek(fd_received,0,SEEK_SET);
len=read(fd_received,buf,5);
if(len<0)
{
perror("read error");
}
printf("(fd_received=%d,len=%d) first %d characters read from the file whoes fd was received(content within ##) ##%.*s##\n",fd_received,len,5,5,buf);
return 0;
}
int sendfd(int sock, int fd)
{
struct msghdr hdr;
struct iovec data;
char cmsgbuf[CMSG_SPACE(sizeof(int))];
char dummy = '*';
data.iov_base = &dummy;
data.iov_len = sizeof(dummy);
memset(&hdr, 0, sizeof(hdr));
hdr.msg_name = NULL;
hdr.msg_namelen = 0;
hdr.msg_iov = &data;
hdr.msg_iovlen = 1;
hdr.msg_flags = 0;
hdr.msg_control = cmsgbuf;
hdr.msg_controllen = CMSG_LEN(sizeof(int));
struct cmsghdr* cmsg = CMSG_FIRSTHDR(&hdr);
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
*(int*)CMSG_DATA(cmsg) = fd;
int n = sendmsg(sock, &hdr, 0);
if(n == -1)
printf("sendmsg() failed: %s (socket fd = %d)\n", strerror(errno), sock);
return n;
}
int recvfd(int s)
{
int n;
int fd;
char buf[1];
struct iovec iov;
struct msghdr msg;
struct cmsghdr *cmsg;
char cms[CMSG_SPACE(sizeof(int))];
iov.iov_base = buf;
iov.iov_len = 1;
memset(&msg, 0, sizeof msg);
msg.msg_name = 0;
msg.msg_namelen = 0;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
msg.msg_control = (caddr_t)cms;
msg.msg_controllen = sizeof cms;
if((n=recvmsg(s, &msg, 0)) < 0)
return -1;
if(n == 0){
perror("unexpected EOF");
return -1;
}
cmsg = CMSG_FIRSTHDR(&msg);
memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
return fd;
}
On running client (recvfd) I get segmentation fault.
./recvfd
[1] 6104 segmentation fault (core dumped) ./recvfd
Here are lines from running gdb with coredump
Core was generated by `./recvfd'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x0000000000400cf9 in recvfd (s=3) at recvfd.c:146
146 memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
Here is the core dump - Link.
I want to sniff the communication happening between the two processes when the file descriptor is being sent. I am not able to figure out why the client crashes when run with socat, but doesn't when run without it.
Update 1
While using socat to sniff communication happening between two processes of a well established open source project (ivshmem - used for sharing memory between running virtual machines, also a part of Intel DPDK, Link), I observed the following.
None of the processes crash on using socat
When socat is used, the file descriptor is not properly sent, and does not get added to the recipient process.
When socat is not used, and the two processes are connected directly, the file descriptor gets sent properly, and gets added to the recipient process.
I am trying to understand procfs for communication between userspace and kernel module. My module has basic two functions for procfs write_proc, driver_mmap.
I call multiple times write_proc by calling fputs("123456789",fd). where fd is file descriptor to procfs entry in /proc directory. But I don't see write_proc called multiple time. Code is attached by here.
<code>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <sound/core.h>
#include <sound/initval.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/proc_fs.h>
#include <asm/uaccess.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/mm.h> /* mmap related stuff */
#define BUF_SIZE 64 * 1024
int *MmapBuffer;
int Factor = 1;
static int write_proc(struct file *filp, int *buf, size_t count, loff_t *offp)
{
int rc,i;
printk("in Write \n");
for (i = 1; i <= 16*1024 ; i++)
MmapBuffer[i-1] = (i+1)*Factor;
Factor++;
return count;
}
static int driver_mmap(struct file *file, struct vm_area_struct *vma)
{
int ret;
vma->vm_flags |= VM_LOCKED|VM_SHARED;
ret = remap_pfn_range(vma, vma->vm_start,
virt_to_phys(MmapBuffer) >> PAGE_SHIFT,
vma->vm_end-vma->vm_start, vma->vm_page_prot);
if(ret != 0)
printk("MMAP Failed \n");
SetPageReserved(virt_to_page(MmapBuffer));
printk("MMAP Succeeded \n");
return 0;
}
// file operations
struct file_operations proc_fops =
{
.write = write_proc,
.mmap = driver_mmap,
};
// init module
int init_module_test(void)
{
printk("<1>Hello world\n");
MmapBuffer = kzalloc(BUF_SIZE,__GFP_COLD|GFP_DMA);
if(MmapBuffer == NULL)
printk("Kzalloc failed. reduce buffer size \n");
proc_create ("Test_fs",0,NULL, &proc_fops);
return 0;
}
// exit module
void cleanup_module_test(void)
{
kfree(MmapBuffer);
remove_proc_entry ("Test_fs", NULL);
printk("Goodbye world\n");
}
module_init(init_module_test);
module_exit(cleanup_module_test);
MODULE_LICENSE("GPL");
</code>
Application code
<code>
#include<stdio.h>
#include<stdlib.h>
#include<sys/mman.h>
#include<errno.h>
#include <fcntl.h>
int main(void)
{
int fd;
int i,j;
int *msg ;
printf("Allocation started \n ");
msg=(int*)malloc(64*1024);
if(msg == NULL)
printf("Allocation failed \n");
//unsigned int *addr;
printf("Starting opening \n ");
if((fd=open("/proc/Test_fs", O_RDONLY ))<0)
{
printf("File not opened ");
}
printf("Starting mapping \n ");
msg = mmap(NULL, 64*1024, PROT_READ, MAP_SHARED , fd, 0);
printf("done from module \n ");
if(msg == MAP_FAILED)
{
printf("MAP failed and error is %s", strerror(errno));
return 0;
}
close(fd);
printf("Successful mapping");
FILE *f;
f=fopen("/proc/Test_fs", "wr");
if(!f)
{
printf("File not opened ");
}
for (j = 0; j < 10 ; j++)
{
if(fputs("1234567890,",f) <= 0)
printf("write failed, ");
for (i = 0; i < 16*1024 ; i++)
printf("%d, ", msg[i]);
printf("\n \n done \n \n ");
}
fclose(f);
return 0;
}
</code>
Hi am trying to implement net link user code and kernel code every thing works fine for unicast (src_addr.nl_groups = 0;). For mulicast, user code bind call always fails for non zero src_addr.nl_groups value. Really am not sure what value to put for multicast and how to proceed further. I checked the usage of netlink_broadcast in kernel source tree, so I put the same group Id value (RTMGRP_LINK) here. For unicast I found good number of help in internet but for multicast I don't think so . So Please help me to proceed further.
Error am getting is:
bind: No such file or directory
./a.out: can't bind socket (3)and err : -1: No such file or directory
#include <sys/socket.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <stdio.h>
#include <linux/netlink.h>
#include <linux/rtnetlink.h>
#define NETLINK_TEST 28
#define GROUP_IB 1
#define MAX_PAYLOAD 1024
struct sockaddr_nl src_addr, dst_addr;
struct nlmsghdr *nlh = NULL;
struct msghdr msg;
struct iovec iov;
int sock_fd;
int main(int argc, char ** argv)
{
int err;
sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_TEST);
if (sock_fd<0) {
char s[BUFSIZ];
sprintf( s, "%s: can't assign fd for socket", argv[0] );
perror(s);
return -1;
}
memset(&src_addr, 0, sizeof(src_addr));
src_addr.nl_family = AF_NETLINK;
src_addr.nl_pid = getpid();
src_addr.nl_groups = 0; // Unicast
//src_addr.nl_groups = RTMGRP_LINK; /* Multicast, bind call always fails for non zero values */
err = bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr));
perror("bind");
if (err<0) {
char s[BUFSIZ];
sprintf( s, "%s: can't bind socket (%d)and err : %d", argv[0], sock_fd,err );
perror(s);
return -1;
}
memset(&dst_addr, 0, sizeof(dst_addr));
nlh = (struct nlhmsghdr *) malloc(NLMSG_SPACE(MAX_PAYLOAD));
memset(nlh, 0, NLMSG_SPACE(MAX_PAYLOAD));
iov.iov_base = (void *)nlh;
iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD);
msg.msg_name = (void *)&dst_addr;
msg.msg_namelen = sizeof(dst_addr);
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
printf("pid : %d\n Waiting for messages from kernel...\n",getpid());
recvmsg(sock_fd, &msg, 0);
printf("Message : %s\n", NLMSG_DATA(nlh));
close(sock_fd);
return 0;
}
Netlink socket binds are sensitive to what USER you are- I've seen them reliably fail if you are not running the program in question as 'root', at least on RedHat 6.
Try running as root 1st, before changing your logic. If you get the same failure as you do in normal operation, then you know it isn't (necessarily) a permissions issue.
The issue is
sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_TEST);
Does you kernel module define the NETLINK_TEST family? your own family might must be supported at kernel module and it should post the message in the proper group using nlmsg_multicast()
RTMGRP_LINK is group defined in NETLINK_ROUTE.
This sample code is example for multicast
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <linux/netlink.h>
#include <unistd.h>
#define MYPROTO NETLINK_USERSOCK
#define MYMGRP 21
int open_netlink(void)
{
int sock;
struct sockaddr_nl addr;
int group = MYMGRP;
sock = socket(AF_NETLINK, SOCK_RAW, MYPROTO);
if (sock < 0) {
printf("sock < 0.\n");
return sock;
}
memset((void *) &addr, 0, sizeof(addr));
addr.nl_family = AF_NETLINK;
addr.nl_pid = getpid();
/* addr.nl_groups = MYMGRP; */
if (bind(sock, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
printf("bind < 0.\n");
return -1;
}
if (setsockopt(sock, 270, NETLINK_ADD_MEMBERSHIP, &group, sizeof(group)) < 0) {
printf("setsockopt < 0\n");
return -1;
}
return sock;
}
void read_event(int sock)
{
struct sockaddr_nl nladdr;
struct msghdr msg;
struct iovec iov;
char buffer[65536];
int ret;
iov.iov_base = (void *) buffer;
iov.iov_len = sizeof(buffer);
msg.msg_name = (void *) &(nladdr);
msg.msg_namelen = sizeof(nladdr);
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
printf("Ok, listening.\n");
ret = recvmsg(sock, &msg, 0);
if (ret < 0)
printf("ret < 0.\n");
else
printf("Received message payload: %s\n", NLMSG_DATA((struct nlmsghdr *) &buffer));
}
int main(int argc, char *argv[])
{
int nls;
nls = open_netlink();
if (nls < 0)
return nls;
while (1)
read_event(nls);
return 0;
}
kernel module:
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/netlink.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#define MYPROTO NETLINK_USERSOCK
#define MYGRP 21
static struct sock *nl_sk = NULL;
static void send_to_user(void)
{
struct sk_buff *skb;
struct nlmsghdr *nlh;
char *msg = "Hello from kernel";
int msg_size = strlen(msg) + 1;
int res;
pr_info("Creating skb.\n");
skb = nlmsg_new(NLMSG_ALIGN(msg_size + 1), GFP_KERNEL);
if (!skb) {
pr_err("Allocation failure.\n");
return;
}
nlh = nlmsg_put(skb, 0, 1, NLMSG_DONE, msg_size + 1, 0);
strcpy(nlmsg_data(nlh), msg);
pr_info("Sending skb.\n");
res = nlmsg_multicast(nl_sk, skb, 0, MYGRP, GFP_KERNEL);
if (res < 0)
pr_info("nlmsg_multicast() error: %d\n", res);
else
pr_info("Success.\n");
}
static int __init hello_init(void)
{
pr_info("Inserting hello module.\n");
nl_sk = netlink_kernel_create(&init_net, MYPROTO, NULL);
if (!nl_sk) {
pr_err("Error creating socket.\n");
return -10;
}
send_to_user();
netlink_kernel_release(nl_sk);
return 0;
}
static void __exit hello_exit(void)
{
pr_info("Exiting hello module.\n");
}
module_init(hello_init);
module_exit(hello_exit);
MODULE_LICENSE("GPL");