Please help me to resolve the issue I am facing in my Linux Server.
I have a Linux System having 16GB RAM.
My java server is running on this machine it accepts only tcp connections from its clients.
But there are alsways 8 -10 clients stay in TCP DISCONNECTION state.
When I have checked my linux system by running
netstat -an | grep -c SYN
it gives always a value between 4 to 10 or sometimes more than 10.
CAN any one help to get rid of it.
how can I have 0 SYN_RECV sockets.
Thank you in advance.
sysct.conf is as follows:-
net.ipv4.ip_forward = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.core.wmem_max = 16777216
net.core.rmem_max = 16777216
net.core.netdev_max_backlog = 30000
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_no_metrics_save = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_congestion_control = htcp
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_mtu_probing = 0
fs.file-max = 4573502
net.ipv4.tcp_max_tw_buckets = 1000000
Related
I traced open, read, close and dup system calls in gimp-2.8.22 using strace, with the following command:
strace -eread,openat,open,close,dup,dup2 gimp
In gimp, I opened an image named babr.jpg. The trace shows that this image was opened (file descriptor is 14), read and closed. But, immediately after that, the same file descriptor (14 is not opened after the last close) is used for reading. How is it possible?
Here is the relevant portion of trace:
read(14, "\371\331\25\233M\311j\261b\271\332\240\33\315d\234\340y\236\217\323\206(\214\270x2\303S\212\252\254"..., 4096) = 4096
read(14, "t\260\265fv<\243.5A\324\17\221+\36\207\265&+rL\247\343\366\372\236\353\353'\226\27\27"..., 4096) = 318
close(14) = 0
openat(AT_FDCWD, "/home/ahmad/Pictures/babr.jpg", O_RDONLY) = 14
read(14, "\377\330\377\340\0\20JFIF\0\1\1\1\1,\1,\0\0\377\355(\212Photosho"..., 4096) = 4096
close(14) = 0
openat(AT_FDCWD, "/opt/gimp-2.8.22/lib/gimp/2.0/plug-ins/file-jpeg", O_RDONLY) = 19
read(19, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P[\0\0\0\0\0\0"..., 4096) = 4096
close(19) = 0
close(20) = 0
read(19, "", 8) = 0
close(19) = 0
close(17) = 0
close(16) = 0
read(4, "\2\0\0\0\0\0\0\0", 16) = 8
Gtk-^[[1;32mMessage^[[0m: ^[[34m15:09:02.956^[[0m: Failed to load module "canberra-gtk-module"
read(14, "\0\0\0\5", 4) = 4
read(14, "\0\0\0\23", 4) = 4
read(14, "gimp-progress-init\0", 19) = 19
read(14, "\0\0\0\2", 4) = 4
I also checked this using Pin and found the same result.
The second file descriptor #14 is very likely a pipe between the plugin and Gimp (the handle being free has been reused). And you don't trace the creation of pipes.
From gimpplugin.c:
/* Open two pipes. (Bidirectional communication).
*/
if ((pipe (my_read) == -1) || (pipe (my_write) == -1))
{
gimp_message (plug_in->manager->gimp, NULL, GIMP_MESSAGE_ERROR,
"Unable to run plug-in \"%s\"\n(%s)\n\npipe() failed: %s",
gimp_object_get_name (plug_in),
gimp_file_get_utf8_name (plug_in->file),
g_strerror (errno));
return FALSE;
}
>>> packet = IP(dst="ntpid")/UDP(dport=123,sport=50000)/("\x1b\x00\x00\x00"+"\x00"*11*4)
>>> packet.show()
###[ IP ]###
version = 4
ihl = None
tos = 0x0
len = None
id = 1
flags =
frag = 0
ttl = 64
proto = udp
chksum = None
src = xxx.xxx.xxx.xxx
dst = xxx.xxx.xxx.xxx
\options \
###[ UDP ]###
sport = 50000
dport = ntp
len = None
chksum = None
###[ Raw ]###
load = '\x1b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
>>> rep,non_rep = sr(packet)
Begin emission:
Finished sending 1 packets.
..........................^C
Received 26 packets, got 0 answers, remaining 1 packets
I would like to get the ntp_monlist response of my NTP server.
Actually the packet is sent but I didn't receive anything.
Can someone tell me the reason?
I try for it like this？ But the received ntp is question mark。
>>> packet = IP(dst=ntpserver)/UDP(dport=123,sport=50000)/ NTP(version=4)
>>>rep,non_rep = sr(packet)
>>> rep.show()
0000 IP / UDP / NTP v??, ?? ==> IP / UDP / NTP v??, ??
I was able to get this to work with the following command:
data = "\x17\x00\x03\x2a" + "\x00" * 4
send(IP(src=src_ip, dst=dst_ip)/UDP(sport=RandShort(),dport=123)/Raw(load=data),count=packets_chunk)
Source: https://vulners.com/packetstorm/PACKETSTORM:127492
I was trying to find the virtual set size and resident set size of a c program. I wrote a kernel module to traverse the vm_areas and calculated vss and rss. I also wrote one c program to validate the changes in vss and rss.
// sample test program
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#define N 10000000
int main() {
// setup...
int *arg1 = malloc(5*sizeof(int));
char *arg2 = malloc(sizeof(char)*1024);
pid_t _pid = getpid();
int rss , prev_rss , vss, prev_vss;
printf("pid of this process = %d\n",_pid);
//...
// first observatrion
arg1[0] = (int)(_pid);
long res = syscall(333,arg1,&arg2);
vss = prev_vss = arg1[1]; // agr1[1] stores the vss from the kernel module
rss = prev_rss = arg1[2]; // agr1[2] stores the rss from the kernel module
printf("vss = %d rss = %d\n",vss,rss);
unsigned int *ptr = malloc(1<<21); // 2 MB
printf("ptr = %p\n",ptr);
// second observatrion
arg1[0] = (int)(_pid);
res = syscall(333,arg1,&arg2);
vss = arg1[1];
rss = arg1[2];
printf("vss = %d rss = %d\n",vss,rss);
if(vss - prev_vss > 0) {
printf("chnage in vss = %d\n", vss - prev_vss);
}
if(rss - prev_rss > 0) {
printf("chnage in rss = %d\n", rss - prev_rss);
}
prev_vss = vss;
prev_rss = rss;
// ...
return 0;
}
The output of the above program:
pid of this process = 12964
vss = 4332 rss = 1308
ptr = 0x7f4077464010
vss = 6384 rss = 1312
chnage in vss = 2052
chnage in rss = 4
Here are the dmesg output :
First observation:
[11374.065527] 1 = [0000000000400000-0000000000401000] RSS=4KB sample
[11374.065529] 2 = [0000000000600000-0000000000601000] RSS=4KB sample
[11374.065530] 3 = [0000000000601000-0000000000602000] RSS=4KB sample
[11374.065532] 4 = [0000000000c94000-0000000000cb5000] RSS=4KB
[11374.065539] 5 = [00007f4077665000-00007f407781f000] RSS=1064KB libc-2.19.so
[11374.065546] 6 = [00007f407781f000-00007f4077a1f000] RSS=0KB libc-2.19.so
[11374.065547] 7 = [00007f4077a1f000-00007f4077a23000] RSS=16KB libc-2.19.so
[11374.065549] 8 = [00007f4077a23000-00007f4077a25000] RSS=8KB libc-2.19.so
[11374.065551] 9 = [00007f4077a25000-00007f4077a2a000] RSS=16KB
[11374.065553] 10 = [00007f4077a2a000-00007f4077a4d000] RSS=140KB ld-2.19.so
[11374.065554] 11 = [00007f4077c33000-00007f4077c36000] RSS=12KB
[11374.065556] 12 = [00007f4077c49000-00007f4077c4c000] RSS=12KB
[11374.065557] 13 = [00007f4077c4c000-00007f4077c4d000] RSS=4KB ld-2.19.so
[11374.065559] 14 = [00007f4077c4d000-00007f4077c4e000] RSS=4KB ld-2.19.so
[11374.065561] 15 = [00007f4077c4e000-00007f4077c4f000] RSS=4KB
[11374.065563] 16 = [00007ffcdf974000-00007ffcdf995000] RSS=8KB
[11374.065565] 17 = [00007ffcdf9c3000-00007ffcdf9c6000] RSS=0KB
[11374.065566] 18 = [00007ffcdf9c6000-00007ffcdf9c8000] RSS=4KB
Second observation:
[11374.065655] 1 = [0000000000400000-0000000000401000] RSS=4KB sample
[11374.065657] 2 = [0000000000600000-0000000000601000] RSS=4KB sample
[11374.065658] 3 = [0000000000601000-0000000000602000] RSS=4KB sample
[11374.065660] 4 = [0000000000c94000-0000000000cb5000] RSS=4KB
[11374.065667] 5 = [00007f4077464000-00007f4077665000] RSS=4KB
[11374.065673] 6 = [00007f4077665000-00007f407781f000] RSS=1064KB libc-2.19.so
[11374.065679] 7 = [00007f407781f000-00007f4077a1f000] RSS=0KB libc-2.19.so
[11374.065681] 8 = [00007f4077a1f000-00007f4077a23000] RSS=16KB libc-2.19.so
[11374.065683] 9 = [00007f4077a23000-00007f4077a25000] RSS=8KB libc-2.19.so
[11374.065685] 10 = [00007f4077a25000-00007f4077a2a000] RSS=16KB
[11374.065687] 11 = [00007f4077a2a000-00007f4077a4d000] RSS=140KB ld-2.19.so
[11374.065688] 12 = [00007f4077c33000-00007f4077c36000] RSS=12KB
[11374.065690] 13 = [00007f4077c49000-00007f4077c4c000] RSS=12KB
[11374.065691] 14 = [00007f4077c4c000-00007f4077c4d000] RSS=4KB ld-2.19.so
[11374.065693] 15 = [00007f4077c4d000-00007f4077c4e000] RSS=4KB ld-2.19.so
[11374.065695] 16 = [00007f4077c4e000-00007f4077c4f000] RSS=4KB
[11374.065697] 17 = [00007ffcdf974000-00007ffcdf995000] RSS=8KB
[11374.065699] 18 = [00007ffcdf9c3000-00007ffcdf9c6000] RSS=0KB
[11374.065701] 19 = [00007ffcdf9c6000-00007ffcdf9c8000] RSS=4KB
The virtual address of the ptr was found to be : ptr = 0x7f4077464010 which corresponds to the 5th vm_area in the second obervation.
[00007f4077464000-00007f4077665000] VSS=2052KB // shown from the VSS outputs
My quesitons are :
Why there is a difference of between desired malloc size (which was of 2048 KB) and the vss output for the 5th vm_area (2052 KB)?
We have not accessed the memory region pointed by ptr yet. So then why one physical page s allocated as shown in the rss result of the seocnd observation for the 5th vm_area? ( is it possibly because of the new vm_area_struct ?)
Thank You !
malloc(xxx) does not exactly allocate xxx size of memory. malloc is not a system call, but a library function.
In general, malloc has following steps.
extend the heap space via brk (if it needs)
do mmap to map virtual address with physical address
allocate some metadata (for managing heap space, usually linked list).
In step 3, the one page would be accessed. it means one physical page is accessed and results in increasing the RSS size by 4KB (a page size).
We are getting very frequently below message in /var/log/messages
kernel: SLUB: Unable to allocate memory on node -1 (gfp=0x8020)
In some cases followed by an allocation table
kernel: cache: sigqueue(12019:454c4ebd186d964699132181ad7367c669700f7d8991c47d4bc053ed101675bc), object size: 160, buffer size: 160, default order: 0, min order: 0
kernel: node 0: slabs: 57, objs: 23313, free: 0
kernel: node 1: slabs: 35, objs: 14315, free: 0
Ok, free is 0, but how may this be tuned?
Following is set information
OS - Centos7.3
Kernel - 3.10.0-327.36.3.el7.x86_64
Docker - 1.12.6
Kubernetes - 1.5.5
We have private cloud powered by kurbernetes, having 10 nodes; it was working fine till last month and now we are getting these alerts very frequently on every nodes, pods/container also increased in last few days.
We have enough memory and cpu available on each node.
Any fine tuning for these alert will be very helpful.
Additional information:
sysctl.conf options
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_max_syn_backlog = 4096
net.core.somaxconn = 1024
net.ipv4.tcp_syncookies = 1
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.rmem_default = 65535
net.core.wmem_default = 65535
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.ip_local_port_range = 1024 65535
vm.max_map_count = 262144
vm.swappiness=10
vm.vfs_cache_pressure=100
Please look at this: https://pingcap.com/blog/try-to-fix-two-linux-kernel-bugs-while-testing-tidb-operator-in-k8s/. It's a kernel bug.
problems seems to be with kernel, first a fall check whether swap memory is properly allocated or not by free -m and mkswap -c, if swap is not properly allocated, do it. if swap is fine, then you might need to update the kernel.
all
I have a embedded board, run linux OS. and I use yaffs2 as rootfs.
I run a program on it, but after some times, it got a error "error No space left on device.". but I checked the flash, there still have a lot free space.
I just write some config file. the config file is rarely update. the program will write some log to flash. log size is limited to 2M.
I don't know why, and how to solve.
Help me please!(my first language is not English,sorry. hope you understand what I say)
some debug info:
# ./write_test
version 1.0
close file :: No space left on device
return errno 28
# cat /proc/yaffs
YAFFS built:Nov 23 2015 16:57:34
Device 0 "rootfs"
start_block........... 0
end_block............. 511
total_bytes_per_chunk. 2048
use_nand_ecc.......... 1
no_tags_ecc........... 1
is_yaffs2............. 1
inband_tags........... 0
empty_lost_n_found.... 0
disable_lazy_load..... 0
refresh_period........ 500
n_caches.............. 10
n_reserved_blocks..... 5
always_check_erased... 0
data_bytes_per_chunk.. 2048
chunk_grp_bits........ 0
chunk_grp_size........ 1
n_erased_blocks....... 366
blocks_in_checkpt..... 0
n_tnodes.............. 749
n_obj................. 477
n_free_chunks......... 23579
n_page_writes......... 6092
n_page_reads.......... 11524
n_erasures............ 96
n_gc_copies........... 5490
all_gcs............... 1136
passive_gc_count...... 1136
oldest_dirty_gc_count. 95
n_gc_blocks........... 96
bg_gcs................ 96
n_retired_writes...... 0
n_retired_blocks...... 0
n_ecc_fixed........... 0
n_ecc_unfixed......... 0
n_tags_ecc_fixed...... 0
n_tags_ecc_unfixed.... 0
cache_hits............ 0
n_deleted_files....... 0
n_unlinked_files...... 289
refresh_count......... 1
n_bg_deletions........ 0
Device 2 "data"
start_block........... 0
end_block............. 927
total_bytes_per_chunk. 2048
use_nand_ecc.......... 1
no_tags_ecc........... 1
is_yaffs2............. 1
inband_tags........... 0
empty_lost_n_found.... 0
disable_lazy_load..... 0
refresh_period........ 500
n_caches.............. 10
n_reserved_blocks..... 5
always_check_erased... 0
data_bytes_per_chunk.. 2048
chunk_grp_bits........ 0
chunk_grp_size........ 1
n_erased_blocks....... 10
blocks_in_checkpt..... 0
n_tnodes.............. 4211
n_obj................. 24
n_free_chunks......... 658
n_page_writes......... 430
n_page_reads.......... 467
n_erasures............ 7
n_gc_copies........... 421
all_gcs............... 20
passive_gc_count...... 13
oldest_dirty_gc_count. 3
n_gc_blocks........... 6
bg_gcs................ 4
n_retired_writes...... 0
n_retired_blocks...... 0
n_ecc_fixed........... 0
n_ecc_unfixed......... 0
n_tags_ecc_fixed...... 0
n_tags_ecc_unfixed.... 0
cache_hits............ 0
n_deleted_files....... 0
n_unlinked_files...... 2
refresh_count......... 1
n_bg_deletions........ 0
#
log and config file stored in "data".
thanks!!
In General this could be your disk space (here Flash), first of all check your flash space with with df -h (or other commands you have.. df is present in BusyBox). But if your flash space (specially on your program partition) is ok, this could be your "inode" (directory) space problem, you could see your inode usage with df -i command. (a good link for this: https://wiki.gentoo.org/wiki/Knowledge_Base:No_space_left_on_device_while_there_is_plenty_of_space_available)
If non of these is the problem cause, I think you have to have a deeper look at your code, specially if you deal with disk I/O!
Also good to mention that be aware of memory & heap space & free all allocated spaces in you functions.