Delphi RIO PAServer in Linux raises IOError 11 - linux

I have a strange problem I cannot solve on my own:
Using Delphi 10.3 Rio Enterprise, on Linux (I tried fresh installs of Ubuntu 16.04 and Ubuntu 18.04 Desktop and Ubuntu 18.04 Server with GUI in VMWare Player 15) PAServer crashes with I/O Error 11 every time I start debugging. I can pull the SDK and Delphi deploys all files correctly. But trying to run ends with the crash and Delphi complaining that GDB Server is no longer available. I can run the program on the Linux host by starting it manually (of course without debugging).
It does not depend on the user in Linux, it also happens as root. Also, a non graphic appliation crashes even on non GUI Ubuntu server, always with IO Error 11.
Debugging with PAServer works well on OSX, Android and Windows. I of course tried a dead simple one form project without content with the same results.
I traced the Delphi PAClient with ProcMon, no evidence on that side. Then I tracked the Linux process with strace, which gave me this output:
write(1, "Starting Platform Assistant Serv"..., 49Starting Platform Assistant Server on port 64211
) = 49
write(1, "\n", 1
) = 1
write(1, "Type ? for available commands\n", 30Type ? for available commands
) = 30
socket(PF_INET6, SOCK_STREAM, IPPROTO_IP) = 5
close(5) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 5
getsockopt(5, SOL_SOCKET, SO_TYPE, [1], [4]) = 0
getsockopt(5, SOL_TCP, TCP_NODELAY, [0], [4]) = 0
setsockopt(5, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
setsockopt(5, SOL_SOCKET, SO_REUSEPORT, [1], 4) = 0
bind(5, {sa_family=AF_INET, sin_port=htons(64211), sin_addr=inet_addr("0.0.0.0")}, 16) = 0
getsockname(5, {sa_family=AF_INET, sin_port=htons(64211), sin_addr=inet_addr("0.0.0.0")}, [16]) = 0
setsockopt(5, SOL_TCP, TCP_NODELAY, [1], 4) = 0
socket(PF_INET6, SOCK_STREAM, IPPROTO_IP) = 6
getsockopt(6, SOL_SOCKET, SO_TYPE, [1], [4]) = 0
getsockopt(6, SOL_TCP, TCP_NODELAY, [0], [4]) = 0
setsockopt(6, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
setsockopt(6, SOL_SOCKET, SO_REUSEPORT, [1], 4) = 0
bind(6, {sa_family=AF_INET6, sin6_port=htons(64211), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = 0
getsockname(6, {sa_family=AF_INET6, sin6_port=htons(64211), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, [28]) = 0
setsockopt(6, SOL_TCP, TCP_NODELAY, [1], 4) = 0
listen(5, 15) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f8ab39f9000
mprotect(0x7f8ab39f9000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f8ab41f8ff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f8ab41f99d0, tls=0x7f8ab41f9700, child_tidptr=0x7f8ab41f99d0) = 28987
sched_getparam(28987, [0]) = 0
sched_getscheduler(28987) = 0 (SCHED_OTHER)
listen(6, 15) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f8ab31f8000
mprotect(0x7f8ab31f8000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f8ab39f7ff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f8ab39f89d0, tls=0x7f8ab39f8700, child_tidptr=0x7f8ab39f89d0) = 28988
sched_getparam(28988, [0]) = 0
sched_getscheduler(28988) = 0 (SCHED_OTHER)
brk(0x1f55000) = 0x1f55000
brk(0x1f76000) = 0x1f76000
brk(0x1f97000) = 0x1f97000
brk(0x1fba000) = 0x1fba000
brk(0x1fdb000) = 0x1fdb000
mmap(NULL, 200704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8ab82b8000
mmap(NULL, 200704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f8ab8287000
write(1, ">", 1>) = 1
read(0, 0xe0bf90, 128) = -1 EAGAIN (Resource temporarily unavailable)
read(0, 0xe0bf90, 128) = -1 EAGAIN (Resource temporarily unavailable)
shutdown(5, SHUT_WR) = 0
close(5) = 0
select(4, [3], NULL, NULL, {1, 0}) = 1 (in [3], left {0, 999997})
ioctl(3, FIONREAD, [1]) = 0
read(3, "\0", 1) = 1
select(4, [3], NULL, NULL, {1, 0}) = 1 (in [3], left {0, 817727})
ioctl(3, FIONREAD, [1]) = 0
read(3, "\0", 1) = 1
select(4, [3], NULL, NULL, {1, 0}) = 1 (in [3], left {0, 999997})
ioctl(3, FIONREAD, [1]) = 0
read(3, "\0", 1) = 1
shutdown(6, SHUT_WR) = 0
close(6) = 0
select(4, [3], NULL, NULL, {1, 0}) = 1 (in [3], left {0, 974854})
ioctl(3, FIONREAD, [1]) = 0
read(3, "\0", 1) = 1
shutdown(7, SHUT_WR) = 0
close(7) = 0
shutdown(8, SHUT_WR) = 0
close(8) = 0
select(0, NULL, NULL, NULL, {0, 500000}) = 0 (Timeout)
write(1, "EInOutError: I/O error 11\n", 26EInOutError: I/O error 11
) = 26
Thanks in advance for help!

Please check
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1813873
Ubuntu kernel 4.15.0-44 introduced this trouble, and Ubuntu will fix this trouble on next kernel update, Mar 4th.

Completely revised answer.
It doesn't seem to be related to vmWare Player 15 but somehow to Ubuntu, 16 as well as 18. In all non Linux OS debugging works and also in Rhedhat 7 and the Debian Stretch.
In Debian 9 I had to use the sdk generated with the Ubuntu machine as some files were missing when generated in Debian. May be easy to solve but the other SDK does its job.

Related

Why can't I see the send action?

I am using reqwest in rust to do a simple POST action:
let client = reqwest::blocking::Client::new();
let file = File::open("somefile");
let res = client
.post("http://127.0.0.1:5001/api/v0/add")
.body(file)
.send()?;
I got 400 response. So I used wireshark to check the request sent by reqwest. Interestingly, there was none. All I got was the 400 response.
So I used strace, and compared with cURL.
For cURL, I got:
socket(AF_INET6, SOCK_DGRAM, IPPROTO_IP) = 3
socketpair(AF_UNIX, SOCK_STREAM, 0, [3, 4]) = 0
socket(AF_INET, SOCK_STREAM, IPPROTO_IP) = 5
setsockopt(5, SOL_TCP, TCP_NODELAY, [1], 4) = 0
setsockopt(5, SOL_SOCKET, SO_KEEPALIVE, [1], 4) = 0
setsockopt(5, SOL_TCP, TCP_KEEPIDLE, [60], 4) = 0
setsockopt(5, SOL_TCP, TCP_KEEPINTVL, [60], 4) = 0
connect(5, {sa_family=AF_INET, sin_port=htons(5001), sin_addr=inet_addr("127.0.0.1")}, 16) = -1 EINPROGRESS (Operation now in progress)
getsockopt(5, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
getpeername(5, {sa_family=AF_INET, sin_port=htons(5001), sin_addr=inet_addr("127.0.0.1")}, [128 => 16]) = 0
getsockname(5, {sa_family=AF_INET, sin_port=htons(59784), sin_addr=inet_addr("127.0.0.1")}, [128 => 16]) = 0
sendto(5, "POST /api/v0/add HTTP/1.1\r\nHost: 127.0.0.1:5001\r\nUser-Agent: curl/7.80.0\r\nAccept: */*\r\nContent-Length: 1720244\r\nContent-Type: multipart/form-data; boundary=------------------------dfd2b9478efb2b2d\r\nExpect: 100-continue\r\n\r\n", 222, MSG_NOSIGNAL, NULL, 0) = 222
recvfrom(5, "HTTP/1.1 100 Continue\r\n\r\n", 102400, 0, NULL, NULL) = 25
and more after.
But for my app, I got:
socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 7
connect(7, {sa_family=AF_INET, sin_port=htons(5001), sin_addr=inet_addr("127.0.0.1")}, 16) = -1 EINPROGRESS (Operation now in progress)
getsockopt(7, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
setsockopt(7, SOL_TCP, TCP_NODELAY, [0], 4) = 0
getpeername(7, {sa_family=AF_INET, sin_port=htons(5001), sin_addr=inet_addr("127.0.0.1")}, [128 => 16]) = 0
recvfrom(7, "HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain; charset=utf-8\r\nVary: Origin\r\nX-Content-Type-Options: nosniff\r\nDate: Sat, 04 Dec 2021 13:02:05 GMT\r\nContent-Length: 33\r\n\r\nfile argument 'path' is required\n", 8192, 0, NULL, NULL) = 206
recvfrom(7, 0x7f923c0f0e00, 8192, 0, NULL, NULL) = -1 EAGAIN (Resource temporarily unavailable)
So why is there no sendto call?
The strace calls are the same: strace -f -e trace=network -s 10000 $cmd.

Vim stuck when opening file with `EAGAIN (Resource temporarily unavailable)` backtrace

Vim stuck on opening file. That's the ouput with strace. Anyone could advise here?
strace vim test.py
....
poll([{fd=3, events=POLLIN|POLLOUT}], 1, -1) = 1 ([{fd=3, revents=POLLOUT}])
writev(3, [{"l\0\v\0\0\0\22\0\20\0\0\0", 12}, {"", 0}, {"MIT-MAGIC-COOKIE-1", 18}, {"\0\0", 2}, {"\231\r\1\377\"\0025\371\232\264nj%:\32\277", 16}, {"", 0}], 6) = 48
recvfrom(3, 0x2bec270, 8, 0, NULL, NULL) = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=3, events=POLLIN}], 1, -1
I suffered from this and dig into it a little.
Here comes my log.
09:46:07.386053 socket(AF_INET6, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 3
09:46:07.386078 setsockopt(3, SOL_TCP, TCP_NODELAY, [1], 4) = 0
09:46:07.386102 setsockopt(3, SOL_SOCKET, SO_KEEPALIVE, [1], 4) = 0
09:46:07.386125 connect(3, {sa_family=AF_INET6, sin6_port=htons(6010), inet_pton(AF_INET6, "::1", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id=0}, 28) = 0
09:46:07.386225 getpeername(3, {sa_family=AF_INET6, sin6_port=htons(6010), inet_pton(AF_INET6, "::1", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id=0}, [124->28]) = 0
09:46:07.386258 uname({sysname="Linux", nodename="localhost.localdomain", ...}) = 0
09:46:07.386324 access("/root/.Xauthority", R_OK) = 0
09:46:07.386385 openat(AT_FDCWD, "/root/.Xauthority", O_RDONLY) = 4
09:46:07.386414 fstat(4, {st_mode=S_IFREG|0600, st_size=469, ...}) = 0
09:46:07.386439 read(4, "\1\0\0\25localhost.localdomain\0\00216\0\22M"..., 4096) = 469
09:46:07.386548 close(4) = 0
09:46:07.386611 getsockname(3, {sa_family=AF_INET6, sin6_port=htons(49466), inet_pton(AF_INET6, "::1", &sin6_addr), sin6_flowinfo=htonl(0), sin6_scope_id=0}, [124->28]) = 0
09:46:07.386688 fcntl(3, F_GETFL) = 0x2 (flags O_RDWR)
09:46:07.386773 fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK) = 0
09:46:07.386831 fcntl(3, F_SETFD, FD_CLOEXEC) = 0
09:46:07.386853 poll([{fd=3, events=POLLIN|POLLOUT}], 1, -1) = 1 ([{fd=3, revents=POLLOUT}])
09:46:07.386968 writev(3, [{iov_base="l\0\v\0\0\0\22\0\20\0\0\0", iov_len=12}, {iov_base="", iov_len=0}, {iov_base="MIT-MAGIC-COOKIE-1", iov_len=18}, {iov_base="\0\0", iov_len=2}, {iov_base="yD\310A\20\2\202\33\232\242~/\351E\260\336", iov_len=16}, {iov_base="", iov_len=0}], 6) = 48
09:46:07.387053 recvfrom(3, 0x561e505ded90, 8, 0, NULL, NULL) = -1 EAGAIN (Resource temporarily unavailable)
09:46:07.387081 poll([{fd=3, events=POLLIN}], 1, -1) = 1 ([{fd=3, revents=POLLIN}])
09:46:09.393095 recvfrom(3, "\1\0\v\0\0\0\362\0", 8, 0, NULL, NULL) = 8
09:46:09.393186 recvfrom(3, "\277B\241\3\0\0\200\0\377\377\37\0\0\1\0\0\16\0\377\377\1\7\0\0 \10\377\20\377\202\0"..., 968, 0, NULL, NULL) = 968
09:46:09.393266 poll([{fd=3, events=POLLIN|POLLOUT}], 1, -1
<<===========type enter when it stucks.
) = 1 ([{fd=3, revents=POLLOUT}])
09:46:09.393314 writev(3, [{iov_base="b\0\5\0\f\0\0\0BIG-REQUESTS", iov_len=20}], 1) = 20
And I gdb it and find the source(https://github.com/stapelberg/libxcb/blob/master/src/xcb_in.c#L267).
And the sad message is there is no issue and it is just waiting for data from xserver.
The workaround will be disable the xterm.
I like the +xterm_clipboard feature so I have to bear it.

Delay on all network activity on CentOS with DNS

We have a CentOS 7 VM instance on ESXi that is exhibiting a delay every time network calls are made that require DNS resolution (for example "ping"). The DNS server is on a Windows server, and no other hosts on the network using the same DNS server have the problem so it seems isolated to something in CentOS.
The configuration for the network adapter is as follows:
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=no
IPV6_AUTOCONF=no
IPV6_DEFROUTE=no
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens160
UUID=61c69ed9-a97b-40ea-a602-5fe82b9e79cc
DEVICE=ens160
ONBOOT=yes
PEERDNS=no
IPADDR=10.223.27.3
NETMASK=255.255.255.0
GATEWAY=10.223.27.1
DNS1=10.223.27.2
DOMAIN=arndev.com
HWADDR=00:50:56:11:11:01
The DNS server is at IP address 10.223.27.2
If I "nslookup" one of the other hosts on the network it returns almost immediately:
$ time nslookup devtest1
Server: 10.223.27.2
Address: 10.223.27.2#53
Name: devtest1.arndev.com
Address: 10.223.27.51
real 0m0.011s
user 0m0.006s
sys 0m0.005s
If I "strace" a "ping" command, I see it resolve the IP address almost immediately as well, and then a delay where there are two additional calls made prior to the ping succeeding to reach the host.
Originally each of these two calls were taking 5 seconds before they timed out, making a 10 second delay in total. I did find an "options timeout:1" entry to add to "/etc/resolv.conf" that means I can reduce this to 2 seconds (2 x 1 second delay) which is obviously better, but in no way ideal.
The "/etc/resolv.conf" file is defined as follows:
options single-request-reopen
options timeout:1
search arndev.com
nameserver 10.223.27.2
If I ping just by the resolved IP address rather than the name, then there is no delay.
The command I am running for the "strace" of the "ping" command is as follows, and I have marked the location where the delay occurs in the output:
$ sudo strace -s 128 -t ping devtest1
PING devtest1.arndev.com (10.223.27.51) 56(84) bytes of data.
<-- delay occurs here, but it's already resolved the IP address -->
64 bytes from 10.223.27.51 (10.223.27.51): icmp_seq=1 ttl=128 time=0.359 ms
64 bytes from 10.223.27.51 (10.223.27.51): icmp_seq=2 ttl=128 time=0.319 ms
64 bytes from 10.223.27.51 (10.223.27.51): icmp_seq=3 ttl=128 time=0.334 ms
64 bytes from 10.223.27.51 (10.223.27.51): icmp_seq=4 ttl=128 time=0.338 ms
I've marked where the delay occurs in the output below (towards the end):
08:09:41 execve("/bin/ping", ["ping", "devtest1"], [/* 17 vars */]) = 0
08:09:41 brk(NULL) = 0x55f55a5ea000
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e11000
08:09:41 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
08:09:41 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
08:09:41 fstat(3, {st_mode=S_IFREG|0644, st_size=37762, ...}) = 0
08:09:41 mmap(NULL, 37762, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3668e07000
08:09:41 close(3) = 0
< lines cut for brevity >
08:09:41 connect(4, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
08:09:41 close(4) = 0
08:09:41 socket(AF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 4
08:09:41 connect(4, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
08:09:41 close(4) = 0
08:09:41 open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=1746, ...}) = 0
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e10000
08:09:41 read(4, "#\n# /etc/nsswitch.conf\n#\n# An example Name Service Switch config file. This file should be\n# sorted with the most-used services "..., 4096) = 1746
08:09:41 read(4, "", 4096) = 0
08:09:41 close(4) = 0
08:09:41 munmap(0x7f3668e10000, 4096) = 0
08:09:41 stat("/etc/resolv.conf", {st_mode=S_IFREG|0644, st_size=113, ...}) = 0
08:09:41 open("/etc/host.conf", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=9, ...}) = 0
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e10000
08:09:41 read(4, "multi on\n", 4096) = 9
08:09:41 read(4, "", 4096) = 0
08:09:41 close(4) = 0
08:09:41 munmap(0x7f3668e10000, 4096) = 0
08:09:41 open("/etc/resolv.conf", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=113, ...}) = 0
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e10000
08:09:41 read(4, "options single-request-reopen\noptions single-request\noptions timeout:1\n\nsearch arndev.com\nnameserver 10.223.27.2\n", 4096) = 113
08:09:41 read(4, "", 4096) = 0
08:09:41 close(4) = 0
08:09:41 munmap(0x7f3668e10000, 4096) = 0
08:09:41 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=37762, ...}) = 0
08:09:41 mmap(NULL, 37762, PROT_READ, MAP_PRIVATE, 4, 0) = 0x7f3668e07000
08:09:41 close(4) = 0
08:09:41 open("/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 4
08:09:41 read(4, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000!\0\0\0\0\0\0#\0\0\0\0\0\0\0x\350\0\0\0\0\0\0\0\0\0\0#\0008\0\7\0#\0!\0 \0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0(\262\0\0\0\0\0\0(\262\0\0\0\0\0\0\0\0 \0\0\0\0\0\1\0\0\0\6\0\0\0"..., 832) = 832
08:09:41 fstat(4, {st_mode=S_IFREG|0755, st_size=61624, ...}) = 0
08:09:41 mmap(NULL, 2173016, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 4, 0) = 0x7f3660d14000
08:09:41 mprotect(0x7f3660d20000, 2093056, PROT_NONE) = 0
08:09:41 mmap(0x7f3660f1f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 4, 0xb000) = 0x7f3660f1f000
08:09:41 mmap(0x7f3660f21000, 22616, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3660f21000
08:09:41 close(4) = 0
08:09:41 mprotect(0x7f3660f1f000, 4096, PROT_READ) = 0
08:09:41 munmap(0x7f3668e07000, 37762) = 0
08:09:41 open("/etc/hosts", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=305, ...}) = 0
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e10000
08:09:41 read(4, "127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4\n::1 localhost localhost.localdomain local"..., 4096) = 305
08:09:41 read(4, "", 4096) = 0
08:09:41 close(4) = 0
08:09:41 munmap(0x7f3668e10000, 4096) = 0
08:09:41 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=37762, ...}) = 0
08:09:41 mmap(NULL, 37762, PROT_READ, MAP_PRIVATE, 4, 0) = 0x7f3668e07000
08:09:41 close(4) = 0
08:09:41 open("/lib64/libnss_dns.so.2", O_RDONLY|O_CLOEXEC) = 4
08:09:41 read(4, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \20\0\0\0\0\0\0#\0\0\0\0\0\0\0pr\0\0\0\0\0\0\0\0\0\0#\0008\0\7\0#\0!\0 \0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\344N\0\0\0\0\0\0\344N\0\0\0\0\0\0\0\0 \0\0\0\0\0\1\0\0\0\6\0\0\0"..., 832) = 832
08:09:41 fstat(4, {st_mode=S_IFREG|0755, st_size=31408, ...}) = 0
08:09:41 mmap(NULL, 2121952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 4, 0) = 0x7f3660b0d000
08:09:41 mprotect(0x7f3660b12000, 2097152, PROT_NONE) = 0
08:09:41 mmap(0x7f3660d12000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 4, 0x5000) = 0x7f3660d12000
08:09:41 close(4) = 0
08:09:41 mprotect(0x7f3660d12000, 4096, PROT_READ) = 0
08:09:41 munmap(0x7f3668e07000, 37762) = 0
08:09:41 socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 4
08:09:41 connect(4, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.223.27.2")}, 16) = 0
08:09:41 poll([{fd=4, events=POLLOUT}], 1, 0) = 1 ([{fd=4, revents=POLLOUT}])
08:09:41 sendto(4, "\203\336\1\0\0\1\0\0\0\0\0\0\10devtest1\6arndev\3com\0\0\1\0\1", 37, MSG_NOSIGNAL, NULL, 0) = 37
08:09:41 poll([{fd=4, events=POLLIN}], 1, 1000) = 1 ([{fd=4, revents=POLLIN}])
08:09:41 ioctl(4, FIONREAD, [53]) = 0
08:09:41 recvfrom(4, "\203\336\205\200\0\1\0\1\0\0\0\0\10devtest1\6arndev\3com\0\0\1\0\1\300\f\0\1\0\1\0\0\16\20\0\4\n\337\0333", 1024, 0, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.223.27.2")}, [16]) = 53
08:09:41 close(4) = 0
08:09:41 socket(AF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
08:09:41 connect(4, {sa_family=AF_INET, sin_port=htons(1025), sin_addr=inet_addr("10.223.27.51")}, 16) = 0
08:09:41 getsockname(4, {sa_family=AF_INET, sin_port=htons(37068), sin_addr=inet_addr("10.223.27.3")}, [16]) = 0
08:09:41 close(4) = 0
08:09:41 setsockopt(3, SOL_RAW, ICMP_FILTER, ~(1<<ICMP_ECHOREPLY|1<<ICMP_DEST_UNREACH|1<<ICMP_SOURCE_QUENCH|1<<ICMP_REDIRECT|1<<ICMP_TIME_EXCEEDED|1<<ICMP_PARAMETERPROB), 4) = 0
08:09:41 setsockopt(3, SOL_IP, IP_RECVERR, [1], 4) = 0
08:09:41 setsockopt(3, SOL_SOCKET, SO_SNDBUF, [324], 4) = 0
08:09:41 setsockopt(3, SOL_SOCKET, SO_RCVBUF, [65536], 4) = 0
08:09:41 getsockopt(3, SOL_SOCKET, SO_RCVBUF, [131072], [4]) = 0
08:09:41 fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e10000
08:09:41 write(1, "PING devtest1.arndev.com (10.223.27.51) 56(84) bytes of data.\n", 62) = 62
08:09:41 setsockopt(3, SOL_SOCKET, SO_TIMESTAMP, [1], 4) = 0
08:09:41 setsockopt(3, SOL_SOCKET, SO_SNDTIMEO, "\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 0
08:09:41 setsockopt(3, SOL_SOCKET, SO_RCVTIMEO, "\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 16) = 0
08:09:41 getpid() = 16440
08:09:41 rt_sigaction(SIGINT, {0x55f559e15dd0, [], SA_RESTORER|SA_INTERRUPT, 0x7f3667aa6280}, NULL, 8) = 0
08:09:41 rt_sigaction(SIGALRM, {0x55f559e15dd0, [], SA_RESTORER|SA_INTERRUPT, 0x7f3667aa6280}, NULL, 8) = 0
08:09:41 rt_sigaction(SIGQUIT, {0x55f559e15dc0, [], SA_RESTORER|SA_INTERRUPT, 0x7f3667aa6280}, NULL, 8) = 0
08:09:41 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
08:09:41 ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
08:09:41 ioctl(1, TIOCGWINSZ, {ws_row=35, ws_col=142, ws_xpixel=1136, ws_ypixel=455}) = 0
08:09:41 sendto(3, "\10\0k0#8\0\1\265\373\222]\0\0\0\0Bj\3\0\0\0\0\0\20\21\22\23\24\25\26\27\30\31\32\33\34\35\36\37 !\"#$%&'()*+,-./01234567", 64, 0, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("10.223.27.51")}, 16) = 64
08:09:41 recvmsg(3, {msg_name(16)={sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("10.223.27.51")}, msg_iov(1)=[{"E\0\0Tv\304\0\0\200\1w\361\n\337\0333\n\337\33\3\0\0s0#8\0\1\265\373\222]\0\0\0\0Bj\3\0\0\0\0\0\20\21\22\23\24\25\26\27\30\31\32\33\34\35\36\37 !\"#$%&'()*+,-./01234567", 192}], msg_controllen=32, [{cmsg_len=32, cmsg_level=SOL_SOCKET, cmsg_type=0x1d /* SCM_??? */}], msg_flags=0}, 0) = 84
08:09:41 stat("/etc/resolv.conf", {st_mode=S_IFREG|0644, st_size=113, ...}) = 0
08:09:41 open("/etc/hosts", O_RDONLY|O_CLOEXEC) = 4
08:09:41 fstat(4, {st_mode=S_IFREG|0644, st_size=305, ...}) = 0
08:09:41 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3668e0f000
08:09:41 read(4, "127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4\n::1 localhost localhost.localdomain local"..., 4096) = 305
08:09:41 read(4, "", 4096) = 0
08:09:41 close(4) = 0
08:09:41 munmap(0x7f3668e0f000, 4096) = 0
<-- delay starts here -->
08:09:41 socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_IP) = 4
08:09:41 connect(4, {sa_family=AF_INET, sin_port=htons(53), sin_addr=inet_addr("10.223.27.2")}, 16) = 0
08:09:41 poll([{fd=4, events=POLLOUT}], 1, 0) = 1 ([{fd=4, revents=POLLOUT}])
08:09:41 sendto(4, "o\262\1\0\0\1\0\0\0\0\0\0\00251\00227\003223\00210\7in-addr\4arpa\0\0\f\0\1", 43, MSG_NOSIGNAL, NULL, 0) = 43
08:09:41 poll([{fd=4, events=POLLIN}], 1, 1000) = 0 (Timeout)
08:09:42 poll([{fd=4, events=POLLOUT}], 1, 0) = 1 ([{fd=4, revents=POLLOUT}])
08:09:42 sendto(4, "o\262\1\0\0\1\0\0\0\0\0\0\00251\00227\003223\00210\7in-addr\4arpa\0\0\f\0\1", 43, MSG_NOSIGNAL, NULL, 0) = 43
08:09:42 poll([{fd=4, events=POLLIN}], 1, 1000) = 0 (Timeout)
08:09:43 close(4) = 0
<-- delay ends here -->
08:09:43 open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 4
08:09:43 fstat(4, {st_mode=S_IFREG|0644, st_size=37762, ...}) = 0
08:09:43 mmap(NULL, 37762, PROT_READ, MAP_PRIVATE, 4, 0) = 0x7f3668df7000
08:09:43 close(4) = 0
08:09:43 open("/lib64/libnss_myhostname.so.2", O_RDONLY|O_CLOEXEC) = 4
Anyone have any idea what could be causing this delay? If you need output from additional commands I'd be happy to supply...
I resolved the problem by creating a "Reverse Lookup Zone" on the Windows DNS Server that mapped to the same subnet.
As my IP addresses were 10.223.27.xxx the Reverse Lookup Zone was created (and automatically named) as "27.223.10.in-addr.arpa".
I believe this name matched the two socket calls that timed out in the "strace" command.
08:09:42 sendto(4, "o\262\1\0\0\1\0\0\0\0\0\0\00251\00227\003223\00210\7in-addr\4arpa\0\0\f\0\1", 43, MSG_NOSIGNAL, NULL, 0) = 43
The pings and other network related operations all now execute without any delay at all.

Why is strace missing network syscalls?

When I to run strace on dig (strace dig google.com), there are no sendto or recvfrom syscalls as you would expect for a DNS query. dig is definitely performing network requests: I can see the queries in Wireshark. strace is capturing the socket and close syscalls, though. The full trace is available at https://pastebin.com/aU816wLq, but I pasted the relevant section below:
futex(0x7f3e959bbe34, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3e959bbf80, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3e963f9290, FUTEX_WAKE_PRIVATE, 2147483647) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 3
close(3) = 0
socket(PF_INET6, SOCK_STREAM, IPPROTO_IP) = 3
getsockname(3, {sa_family=AF_INET6, sin6_port=htons(0), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, [28]) = 0
close(3) = 0
socket(PF_LOCAL, SOCK_STREAM, 0) = 3
close(3) = 0
futex(0x7f3e959bc3ac, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3e959bbf08, FUTEX_WAKE_PRIVATE, 2147483647) = 0
mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3e965c8000
mmap(NULL, 8392704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f3e9028f000
mprotect(0x7f3e9028f000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f3e90a8efb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f3e90a8f9d0, tls=0x7f3e90a8f700, child_tidptr=0x7f3e90a8f9d0) = 3083
mmap(NULL, 8392704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f3e8fa8e000
mprotect(0x7f3e8fa8e000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f3e9028dfb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f3e9028e9d0, tls=0x7f3e9028e700, child_tidptr=0x7f3e9028e9d0) = 3084
brk(0x55fd6e9cd000) = 0x55fd6e9cd000
pipe([3, 4]) = 0
fcntl(3, F_GETFL) = 0 (flags O_RDONLY)
fcntl(3, F_SETFL, O_RDONLY|O_NONBLOCK) = 0
epoll_create(64) = 5
epoll_ctl(5, EPOLL_CTL_ADD, 3, {EPOLLIN, {u32=3, u64=3}}) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f3e8f28d000
mprotect(0x7f3e8f28d000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f3e8fa8cfb0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f3e8fa8d9d0, tls=0x7f3e8fa8d700, child_tidptr=0x7f3e8fa8d9d0) = 3085
open("/usr/share/locale/en_US.UTF-8/libdst.cat", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libdst.cat", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/libdst.cat", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_MESSAGES/libdst.cat", O_RDONLY) = -1 ENOENT (No such file or directory)
futex(0x7f3e963f9b20, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3e963f9b24, FUTEX_WAKE_PRIVATE, 2147483647) = 0
brk(0x55fd6e9ee000) = 0x55fd6e9ee000
open("/usr/lib/ssl/openssl.cnf", O_RDONLY) = 6
fstat(6, {st_mode=S_IFREG|0644, st_size=10835, ...}) = 0
read(6, "#\n# OpenSSL example configuratio"..., 4096) = 4096
read(6, "Netscape crash on BMPStrings or "..., 4096) = 4096
read(6, " this to avoid interpreting an e"..., 4096) = 2643
read(6, "", 4096) = 0
close(6) = 0
futex(0x7f3e94c480a8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/usr/lib/x86_64-linux-gnu/openssl-1.0.0/engines/libgost.so", O_RDONLY|O_CLOEXEC) = 6
read(6, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220T\0\0\0\0\0\0"..., 832) = 832
fstat(6, {st_mode=S_IFREG|0644, st_size=93280, ...}) = 0
mmap(NULL, 2188320, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 6, 0) = 0x7f3e8f076000
mprotect(0x7f3e8f08b000, 2093056, PROT_NONE) = 0
mmap(0x7f3e8f28a000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x14000) = 0x7f3e8f28a000
close(6) = 0
mprotect(0x7f3e8f28a000, 4096, PROT_READ) = 0
open("/home/ubuntu-user/.digrc", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/resolv.conf", O_RDONLY) = 6
fstat(6, {st_mode=S_IFREG|0644, st_size=208, ...}) = 0
read(6, "# Dynamic resolv.conf(5) file fo"..., 4096) = 208
read(6, "", 4096) = 0
close(6) = 0
getppid() = 3080
rt_sigaction(SIGHUP, {0x7f3e957899e0, ~[RTMIN RT_1], SA_RESTORER, 0x7f3e9553f390}, NULL, 8) = 0
rt_sigsuspend([], 8) = ? ERESTARTNOHAND (To be restarted if no handler)
--- SIGTERM {si_signo=SIGTERM, si_code=SI_TKILL, si_pid=3082, si_uid=1000} ---
rt_sigreturn({mask=[HUP INT TERM]}) = -1 EINTR (Interrupted system call)
futex(0x55fd6df085a0, FUTEX_WAIT_PRIVATE, 2, NULL) = 0
futex(0x55fd6df085a0, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7f3e90a8f9d0, FUTEX_WAIT, 3083, NULL) = 0
write(4, "\0\0\0\0\377\377\377\377", 8) = 8
epoll_ctl(5, EPOLL_CTL_DEL, 3, 0x7fff8ed49d10) = 0
close(5) = 0
close(3) = 0
close(4) = 0
futex(0x7f3e965cf07c, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x7f3e965cf078, {FUTEX_OP_SET, 0, FUTEX_OP_CMP_GT, 1}) = 1
futex(0x7f3e965cf028, FUTEX_WAKE_PRIVATE, 1) = 1
munmap(0x7f3e8f076000, 2188320) = 0
munmap(0x7f3e965c8000, 266240) = 0
write(1, "\n; <<>> DiG 9.10.3-P4-Ubuntu <<>"..., 484) = 484
exit_group(0) = ?
+++ exited with 0 +++
I see clone syscalls there. Is dig forking and handling the DNS query in another process?
Versions:
strace -- version 4.11
DiG 9.10.3-P4-Ubuntu
Linux ubuntu 4.4.0-71-generic #92-Ubuntu SMP Fri Mar 24 12:59:01 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
The dig process is creating threads. Use strace -f to follow them.

Why is this read blocking when O_NONBLOCK is set?

Strace excerpt (please ignore sending xml to ssh):
dup(12) = 13
getsockname(13, {sa_family=AF_INET, sin_port=htons(46811), sin_addr=inet_addr("127.0.0.1")}, [16]) = 0
getsockopt(13, SOL_SOCKET, SO_TYPE, [1], [4]) = 0
fstat(13, {st_mode=S_IFSOCK|0777, st_size=0, ...}) = 0
getsockname(13, {sa_family=AF_INET, sin_port=htons(46811), sin_addr=inet_addr("127.0.0.1")}, [16]) = 0
getsockopt(13, SOL_SOCKET, SO_TYPE, [1], [4]) = 0
fcntl(13, F_SETFD, FD_CLOEXEC) = 0
fcntl(13, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(13, F_SETFL, O_RDWR|O_NONBLOCK) = 0
setsockopt(13, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
setsockopt(13, SOL_SOCKET, SO_REUSEPORT, [0], 4) = 0
setsockopt(13, SOL_TCP, TCP_NODELAY, [0], 4) = 0
setsockopt(13, SOL_TCP, TCP_CORK, [0], 4) = 0
fcntl(12, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(12, F_SETFL, O_RDWR) = 0
close(12) = 0
getpeername(13, {sa_family=AF_INET, sin_port=htons(22), sin_addr=inet_addr("127.0.0.1")}, [16]) = 0
sendto(13, "<stream:stream xmlns:stream='htt"..., 133, MSG_NOSIGNAL, NULL, 0) = 133
read(13, "SSH-2.0-OpenSSH_6.6.1\r\n", 4096) = 23
read(13, <unfinished ...>
(At the end I kill -9 it.)
The read() at the end was blocking; even though O_NONBLOCK was set via fcntl().
How is this possible?
Version info:
$ uname -r
3.16.4-1-ARCH
Because FD 13 is a dup() of FD 12, this line removes your O_NONBLOCK:
fcntl(12, F_SETFL, O_RDWR) = 0

Resources