Two processes share UDP socket, one stops receiving messages after being restarted - linux

Question

On a Linux platform I want to use socket sharing between 2 processes. One of the processes sends data on the socket, the other one receives data. I read on this site (here) that this is done by setting an option SO_REUSEADDR and/or SO_REUSEPORT.
So I set a test scenario with 3 processes:
1) An echo server bound to localhost that listens for messages on 127.0.0.1:44000. When it receives a message it replies to the sender immediately;
2) A sender that binds to 127.0.01:44001 and issues periodic messages to the echo server;
3) A receiver that binds to 127.0.01:44001 and listens for messages from the echo server;
The problem: Receiver stops receiving replies from the echo server. This depends on the socket option used:
With SO_REUSEADDR:
If sender(2) is started after the receiver(3) the latter does not receive anything. If the receiver is started last, but the sender is restarted, again the receiver stops receiving.
With SO_REUSEPORT (or together with SO_REUSEADDR):
The situation is just the opposite - receiver has to be started first for things to work, as sender is started last you can restart sender as many times as you want, everything will work well. But if you restart the sender (or just start it last) it will not get any message.
This is the code I use:
#define CC_LISTEN_PORT 44000
#define DRN_LISTEN_PORT 44001
static void runCC_EchoMode(struct sockaddr_in* ccaddr)
{
char buf[100];
int s = socket(AF_INET, SOCK_DGRAM, 0);
struct sockaddr_in remaddr;
int recvlen, sentlen;
// bind
if(bind(s, (struct sockaddr *)ccaddr, sizeof(struct sockaddr_in)) < 0)
{
debug("%s: bind failed", __func__);
return;
}
/* now loop, receiving data and printing what we received */
unsigned int addrlen = sizeof(remaddr);
int count = 0;
for (;;) {
debug("waiting on port %d\n", ntohs(ccaddr->sin_port));
recvlen = recvfrom(s, buf, sizeof(buf), 0, (struct sockaddr *)&remaddr, &addrlen);
debug("received %d bytes\n", recvlen);
if (recvlen > 0) {
buf[recvlen] = 0;
printf("received message: \"%s\"\n", buf);
// send echo back
sprintf(buf, "Echo #%d", count++);
sentlen = sendto(s, buf, strlen(buf), 0, (struct sockaddr *)&remaddr, sizeof(remaddr));
debug("sent %d bytes to %s:%d\n", sentlen,
inet_ntoa(remaddr.sin_addr), ntohs(remaddr.sin_port));
}
}
close(s);
}
static void runDrn_SendMode(struct sockaddr_in* ccaddr, struct sockaddr_in* drnaddr)
{
char buf[100];
int s = socket(AF_INET, SOCK_DGRAM, 0);
int sentlen;
int one = 1;
if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(int)) < 0) {
debug("setsockopt(SO_REUSEADDR) failed\n");
}
if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(int)) < 0) {
debug("setsockopt(SO_REUSEPORT) failed\n");
}
// bind
if(bind(s, (struct sockaddr *)drnaddr, sizeof(struct sockaddr_in)) < 0)
{
debug("%s: bind failed", __func__);
return;
}
int count = 0;
for (;;) {
sleep(2);
sprintf(buf, "Hello #%d", count++);
debug("sending \"%s\" to server...\n", buf);
sentlen = sendto(s, buf, strlen(buf), 0, (struct sockaddr *)ccaddr, sizeof(struct sockaddr_in));
debug("sent %d bytes\n", sentlen);
}
close(s);
}
static void runDrn_RcvMode(struct sockaddr_in* ccaddr, struct sockaddr_in* drnaddr)
{
char buf[100];
int s = socket(AF_INET, SOCK_DGRAM, 0);
struct sockaddr_in remaddr;
int recvlen;
int one = 1;
if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(int)) < 0) {
debug("setsockopt(SO_REUSEADDR) failed\n");
}
if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(int)) < 0) {
debug("setsockopt(SO_REUSEPORT) failed\n");
}
// bind
if(bind(s, (struct sockaddr *)drnaddr, sizeof(struct sockaddr_in)) < 0)
{
debug("%s: bind failed", __func__);
return;
}
/* now loop, receiving data and printing what we received */
unsigned int addrlen = sizeof(remaddr);
for (;;) {
debug("waiting on port %d\n", ntohs(drnaddr->sin_port));
recvlen = recvfrom(s, buf, sizeof(buf), 0, (struct sockaddr *)&remaddr, &addrlen);
debug("received %d bytes\n", recvlen);
if (recvlen > 0) {
buf[recvlen] = 0;
printf("received message: \"%s\"\n", buf);
}
}
close(s);
}
int main(int argc, char *argv[])
{
int mode;
if (argc < 3) {
fprintf(stderr, "Usage: %s <host> <mode>\n", argv[0]);
exit(EXIT_FAILURE);
}
printf("Starting process with PID: %d\n", getpid());
// this is a simple wrapper of getaddrinfo()
AddressResolver resv1(argv[1]);
resv1.print();
struct sockaddr_in ccaddr, drnaddr;
ccaddr = *(resv1.getAddress(0));
ccaddr.sin_port = htons(CC_LISTEN_PORT);
drnaddr = *(resv1.getAddress(0));
drnaddr.sin_port = htons(DRN_LISTEN_PORT);
mode = atoi(argv[2]);
switch(mode) {
case 0: // cc
runCC_EchoMode(&ccaddr);
break;
case 1: // drone sender
runDrn_SendMode(&ccaddr, &drnaddr);
break;
case 2: // drone receiver
runDrn_RcvMode(&ccaddr, &drnaddr);
break;
default:
debug("Mode is not available\n");
break;
}
return 0;
}
And this is how I start the 3 processes:
./testUDP localhost 0
./testUDP localhost 1
./testUDP localhost 2
This is a the output of a test run:
./testUDP localhost 0
Starting process with PID: 10651
IP: 127.0.0.1
waiting on port 44000
received 8 bytes
received message: "Hello #0"
sent 7 bytes to 127.0.0.1:44001
waiting on port 44000
received 8 bytes
received message: "Hello #1"
sent 7 bytes to 127.0.0.1:44001
waiting on port 44000
received 8 bytes
received message: "Hello #2"
sent 7 bytes to 127.0.0.1:44001
waiting on port 44000
^C
...
./testUDP localhost 1
Starting process with PID: 10655
IP: 127.0.0.1
sending "Hello #0" to server...
sent 8 bytes
sending "Hello #1" to server...
sent 8 bytes
sending "Hello #2" to server...
sent 8 bytes
^C
...
./testUDP localhost 2
Starting process with PID: 10652
IP: 127.0.0.1
waiting on port 44001
received 7 bytes
received message: "Echo #0"
waiting on port 44001
received 7 bytes
received message: "Echo #1"
waiting on port 44001
received 7 bytes
received message: "Echo #2"
waiting on port 44001
^C

Answer 1

The behavior of two different processes listening on the same interface and port is non-deterministic: it will vary by operating system, kernel version, and other factors.
In general, a port is intended to be associated with a single process and socket. SO_REUSE is intended for UDP multicast receive or for TCP to bypass the WAIT state after a connection drop. While some systems will let you bind a single port to multiple sockets, threads, or processes for other purposes, the behavior is too varied to be useful.
What you are probably looking for is some kind of packet duplication or round-robin distribution. SO_REUSE does not guarantee that.