in my multithreaded linux server application, a thread gets stuck forever in send function, which I believe should be nonblocking. The code looks like this:
while(size) {
const ssize_t sent = send(unixSocketFD, data, size, MSG_NOSIGNAL | MSG_DONTWAIT);
if(sent > 0) {
size -= sent;
data += sent;
} else {
if(-1 == sent) {
if(EINTR == errno) {
continue;
}
if(EWOULDBLOCK == errno || EAGAIN == errno) {
return 0; // need to be called again later
}
}
return -1; // indicate error ...
}
}
... and the stacktrace:
(gdb) bt
#0 0x00007ffff7bca9ff in __libc_send (fd=5, buf=0x7fffedd25880, n=87643, flags=16448) at ../sysdeps/unix/sysv/linux/x86_64/send.c:26
#1 0x00000000004076cf in Output::sendBlock (this=0x7fffdc0a3190, unixSocketFD=5) at OutputBlock.cpp:24
...
I believe that the send should return immediately when flags contain MSG_DONTWAIT. What's wrong with my code/assumptions? Thanks you for any suggestions.
I'm sorry, this was my mistake: the send was not blocking, instead it was enterered again and again even if it indicated that the socket is not ready to accept more data.
Related
I have a server class that has a method called handle_client as follows:
void server::handle_client()
{
do {
// Accept a client socket
EnterCriticalSection(&listenSocketCriticalSection);
SOCKET clientSocket = accept(listenSocket, NULL, NULL);
LeaveCriticalSection(&listenSocketCriticalSection);
// ... rest of the client handling code that reads
// from the client socket and sends appropriate response
// ...
} while(true);
}
I have a run method as follows:
void server::run()
{
// create the threads
for (int i = 0; i < THREAD_POOL_SIZE; i++) {
DWORD dwThreadId;
thread_pool_handle[i] = CreateThread(NULL, 0, thread_function, this, 0, &dwThreadId);
}
WaitForMultipleObjects(THREAD_POOL_SIZE, thread_pool_handle, true, INFINITE);
}
I have a thread_function as follows:
DWORD WINAPI thread_function(LPVOID lpParam)
{
server* pServer = (server*)lpParam;
pServer->handle_client();
}
I am creating a pool of threads that are all waiting for a client socket connection to be accepted. Since I have wrapped the accept within a critical section, only one thread will succeed at a time. Once a client socket is accepted by the server thread, that thread continues to go on to handle the request. The idea is that the thread will loop back indefinitely to the accept call after completing a request.
Questions:
Is the Critical Section necessary? (I think so, because otherwise the accept call from the multiple threads on the same listenSocket would clobber things. Is this correct?)
If handle_client loops indefinitely, what is the best way to cleanly terminate all the threads and exit the server? Should I use a special message from the client to trigger the thread terminations? Any other suggestions?
How should I handle the server process termination gracefully (as it pertains to the thread pool)?
It Is recommended to use Select model to store socket objects in multithreaded sockets. In Select model, you can use FD_CLR() to clear sockets when there are no network events.
I have the code for the server with the select socket, You can try to run and modify.
#include <iostream>
#include<WinSock2.h>
#include<windows.h>
#include<WS2tcpip.h>
#pragma comment(lib,"ws2_32.lib")
#pragma warning(disable:4996)//inet_addr
int main()
{
//1.Obtain version info
WSADATA wsaData = { 0 };
SOCKET hServer = { 0 };
WSAStartup(MAKEWORD(2, 2), &wsaData);
if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2)
{
printf("version failed %d\n", GetLastError());
return -1;
}
else
{
printf("version succeed \n");
}
//2.create socket
hServer = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (hServer == SOCKET_ERROR)
{
printf("create socket tcp failed %d\n", GetLastError());
return -1;
}
else
{
printf("create socket tcp succeed \n");
}
//3. Create a protocol address family
sockaddr_in ServerAddr = { 0 };
ServerAddr.sin_family = AF_INET6;
ServerAddr.sin_zero[8];
ServerAddr.sin_port = htons(8888);
ServerAddr.sin_addr.S_un.S_addr = inet_addr("192.168.2.50");;//modify your address
//4.bind
int nRet = bind(hServer, (sockaddr*)&ServerAddr, sizeof ServerAddr);
if (nRet == SOCKET_ERROR)
{
printf("bind failed %d\n", GetLastError());
closesocket(hServer);
WSACleanup();
return -1;
}
else
{
printf("bind succeed \n");
}
//5.listen
nRet = listen(hServer, 3);
if (nRet == SOCKET_ERROR)
{
printf("listen failed %d\n", GetLastError());
closesocket(hServer);
WSACleanup();
return -1;
}
else
{
printf("listen succeed \n");
}
sockaddr_in clientAddr = { 0 };// The protocol address family used to receive the client
int len = sizeof(clientAddr);// The size of the accepted client protocol address family information
// Create a select model to store socket objects
FD_SET fd_read;
FD_ZERO(&fd_read);
FD_SET(hServer, &fd_read);
//6. Accept client connections
while (1) {
FD_SET fd_tmp = fd_read;// Read backup can only be in
const timeval tv = { 1,0 };
int Ret = select(NULL, &fd_tmp, NULL, NULL, &tv);
if (Ret == 0) // No network events, TMP is automatically deleted
{
Sleep(1000);
continue;
}
for (int i = 0; i < fd_tmp.fd_count; i++)
{
// If there are network events for a listening socket, it proves that a client is connecting to the socket
if (fd_tmp.fd_array[i] == hServer)
{
SOCKET hclient;
hclient = accept(hServer, (sockaddr*)&clientAddr, &len);// If you do not want to store the protocol address family information of the client, you can pass a NULL address
if (hclient == SOCKET_ERROR)
{
printf("recieve information of client failed %d\n", GetLastError());
closesocket(hServer);
return -1;
}
printf("connecting: %s******** \n", inet_ntoa(clientAddr.sin_addr));
FD_SET(hclient, &fd_read);
}
else // The client socket has network events that prove that the client is sending data and the server is accepting the data
{
char buff[32] = { 0 };
int nRet = recv(fd_tmp.fd_array[i], (char*)buff, 32, NULL);
if (nRet > 0)
{
printf("message: %s\n", buff);
}
else// Removes the current socket fd_array from the fd_read
{
FD_CLR(fd_tmp.fd_array[i], &fd_read);
printf("Disconnect \n", GetLastError());
closesocket(hServer);
closesocket(fd_read.fd_array[i]);
break;
}
}
break;
}
}
//7.close socket
WSACleanup();
getchar();
return 0;
}
I have read lots of articles and Q&As, but I still have lost of troubles..
The socket is ET, non-blocking, and I used epoll, and now I want to recv some data on it.
The code are below my questions.(Is correct I guess?)
My questions are:
How can I know whether I have received all the data? When I run this server, each time the while loop break with EAGAIN. I found some people say that "EAGAIN means there is no data available for reading on a non-blocking socket". Well that make sense.
But is that equals to "Server have received all data from client when EAGIAN?" A neg example is maybe the socket buffer is full, and this time we get EAGAIN, but in fact the client still have some words to send to. Can this be possible ?
And can I add this lines else if (nread < BUFF_SIZE) break; ? Can I say if the nread smaller than BUFF_SIZE, I have already received all data?
Help me plz T_T
string read_buf;
char buff[BUFF_SIZE];
while (true) {
int nread = recv(fd, buff, BUFF_SIZE, 0);
if (nread < 0) {
if (errno == EINTR) continue;
else if (errno == EAGAIN || errno == EWOULDBLOCK) {
break;
}
else {
perror("read error");
return -1;
}
}
else if (nread == 0) {
close(fd);
break;
}
read_buf += std::string(buff, buff + nread);
}
I have a function that will generate multiple threads and i pass to them a different string every time, but it seems that the threads have the same string. The strings are coming from a socket. Here is the code:
pthread_t *MirrorManager;
MirrorManager = malloc(sizeof(pthread_t)*th_size);
if( MirrorManager == NULL ) { perror("malloc"); exit(1); }
/* -------------------------------------- */
int th_num = 0;
while( true )
{
received = 0;
/* Read the desired readable size */
if( read(newsock, &size, sizeof(size)) < 0 )
{ perror("Read"); exit(1); }
/* Read all data */
while( received < size )
{
if( (nread = read(newsock, buffer + received, size - received)) < 0 )
{ perror("Read"); exit(1); }
received += nread;
}
if( strcmp(buffer, "end") == 0 ) { break; }
printf("Received string: %s\n",buffer);
/* Create thread */
strcpy(th_str, buffer);
if( (err = pthread_create(&MirrorManager[th_num], NULL, thread_start, (void*) th_str)) == true )
{ show_error("pthread_create", err); }
/* Take next thread */
th_num++;
}
Here i generate two threads. The two threads have the same string, actually this string is the last string that will come out of the socket. Why this is happening and how can i prevent this? Please help i have stuck here for a few days now.
You should post the complete code.
Given what you have posted, it looks like your issue is that all of your threads share the same parameter th_str:
pthread_create(&MirrorManager[th_num], NULL, thread_start, (void*) th_str))
Instead you should be allocating a separate th_str for each thread, as you're passing a pointer to it for each thread, rather than the string itself.
th_str = malloc(strlen(buffer));
strcpy(th_str, buffer);
And then be sure to have each thread free the pointer that was passed into it.
PS: I'd strongly recommend using strncmp and strncpy on all data from your socket.
I am using windows named pipe example. When I run sample programs to create pipe, write something and receive it in client program everything is just fine. When I move client code into dll, which runs in windows service it just won't receive sent bytes.
Server's code is as follows:
ThreadParams * params = reinterpret_cast<ThreadParams*>(args);
CString * connectionString = params->connectString;
HANDLE hPipe;
DWORD dwBytesRead;
TCHAR buf[1024];
int len;
hPipe = CreateNamedPipe(PIPE_NAME, // Name
PIPE_ACCESS_OUTBOUND | WRITE_OWNER, // OpenMode
PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT, // PipeMode
2, // MaxInstances
1024, // OutBufferSize
1024, // InBuffersize
2000, // TimeOut
NULL); // Security
if (hPipe == INVALID_HANDLE_VALUE)
{
Globals::WriteLog("Could not create the pipe",1);
exit(1);
}
Globals::WriteLog("connect...",1);
ConnectNamedPipe(hPipe, NULL);
Globals::WriteLog("...connected",1);
swprintf(buf, connectionString->GetBuffer());
len = wcslen(buf);
if (!WriteFile(hPipe, buf, len*sizeof(TCHAR), &dwBytesRead, NULL))
Globals::WriteLog("WriteFile failed",1);
else
wprintf(L"written %d bytes\n",dwBytesRead);
DisconnectNamedPipe(hPipe);
CloseHandle(hPipe);
And client:
CString finalResult = _T("");
HANDLE hOut = INVALID_HANDLE_VALUE;
TCHAR buf[1024];
DWORD len;
DWORD dwWritten;
Global::WriteLog("pwrite: waiting for the pipe...",1);
if (WaitNamedPipe(PIPE_NAME, NMPWAIT_WAIT_FOREVER) == 0)
{
Global::WriteLog("WaitNamedPipe failed. error=%d",1,GetLastError());
goto cleanup;
}
Global::WriteLog("the pipe is ready",1);
hOut = CreateFile(PIPE_NAME,
GENERIC_READ,
0,
NULL, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
NULL);
if (hOut == INVALID_HANDLE_VALUE)
{
Global::WriteLog("CreateFile failed with error %d",1,GetLastError());
goto cleanup;
}
Global::WriteLog("Opened the pipe",1);
for (;;)
{
if (!ReadFile(hOut, buf, sizeof(buf), &dwWritten, NULL))
{
Global::WriteLog("ReadFile failed -- probably EOF. Read %d bytes.",1,dwWritten);
goto cleanup;
}
else
break;
}
finalResult = CString(buf);
Global::WriteLog("String from pipe:%S",1,buf);
cleanup:
if(hOut != INVALID_HANDLE_VALUE)
CloseHandle(hOut);
Server's code runs in a thread if that changes anything (I've tested it with this thread version in sample program and there was no problem with this).
Why does it not work?
Thanks in advance
Okay, it seems I figured it out. It seems I didn't understood documentation properly.
At server's side WriteFile function doesn't block until the string is read. My program simply wrote data and then closed handle - pipe. Client didn't catch the message and threw error stating that there is no process on the other side of pipe.
Also from client I removed for(;;) loop.
To wait for read operation on client side to complete I added
FlushFileBuffers(hPipe);
after successful write operation.
Hope that help somebody
I need async connect and disconnect for tcp client using epoll for Linux. There are ext. functions in Windows, such as ConnectEx, DisconnectEx, AcceptEx, etc...
In tcp server standard accept function is working, but in tcp client doesn't working connect and disconnect... All sockets are nonblocking.
How can I do this?
Thanks!
To do a non-blocking connect(), assuming the socket has already been made non-blocking:
int res = connect(fd, ...);
if (res < 0 && errno != EINPROGRESS) {
// error, fail somehow, close socket
return;
}
if (res == 0) {
// connection has succeeded immediately
} else {
// connection attempt is in progress
}
For the second case, where connect() failed with EINPROGRESS (and only in this case), you have to wait for the socket to be writable, e.g. for epoll specify that you're waiting for EPOLLOUT on this socket. Once you get notified that it's writable (with epoll, also expect to get an EPOLLERR or EPOLLHUP event), check the result of the connection attempt:
int result;
socklen_t result_len = sizeof(result);
if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &result, &result_len) < 0) {
// error, fail somehow, close socket
return;
}
if (result != 0) {
// connection failed; error code is in 'result'
return;
}
// socket is ready for read()/write()
In my experience, on Linux, connect() never immediately succeeds and you always have to wait for writability. However, for example, on FreeBSD, I've seen non-blocking connect() to localhost succeeding right away.
From experience, when detect non-blocking connection , epoll is a little different from select and poll.
with epoll:
After connect() call is made, check return code.
If the connection can not be completed immediately, then register EPOLLOUT event with epoll.
Call epoll_wait().
if the connection failed, your events will be fill with EPOLLERR or EPOLLHUP, otherwise EPOLLOUT will be triggered.
I have a "complete" answer here in case anyone else is looking for this:
#include <sys/epoll.h>
#include <errno.h>
....
....
int retVal = -1;
socklen_t retValLen = sizeof (retVal);
int status = connect(socketFD, ...);
if (status == 0)
{
// OK -- socket is ready for IO
}
else if (errno == EINPROGRESS)
{
struct epoll_event newPeerConnectionEvent;
int epollFD = -1;
struct epoll_event processableEvents;
unsigned int numEvents = -1;
if ((epollFD = epoll_create (1)) == -1)
{
printf ("Could not create the epoll FD list. Aborting!");
exit (2);
}
newPeerConnectionEvent.data.fd = socketFD;
newPeerConnectionEvent.events = EPOLLOUT | EPOLLIN | EPOLLERR;
if (epoll_ctl (epollFD, EPOLL_CTL_ADD, socketFD, &newPeerConnectionEvent) == -1)
{
printf ("Could not add the socket FD to the epoll FD list. Aborting!");
exit (2);
}
numEvents = epoll_wait (epollFD, &processableEvents, 1, -1);
if (numEvents < 0)
{
printf ("Serious error in epoll setup: epoll_wait () returned < 0 status!");
exit (2);
}
if (getsockopt (socketFD, SOL_SOCKET, SO_ERROR, &retVal, &retValLen) < 0)
{
// ERROR, fail somehow, close socket
}
if (retVal != 0)
{
// ERROR: connect did not "go through"
}
}
else
{
// ERROR: connect did not "go through" for other non-recoverable reasons.
switch (errno)
{
...
}
}
I have tried the Sonny's solution and the epoll_ctl will return invalid argument. So i think maybe the right way to do this is as follow:
1.create socketfd and epollfd
2.use epoll_ctl to associate the socketfd and epollfd with epoll event.
3.do connect(socketfd,...)
4.check the return value or errno
5.if errno == EINPROGRESS, do epoll_wait