alternative for netcat utility - linux

Is there any alternative for netcat utility?
I want to run docker API and netcat utility is not installed on client system.
docker command example - echo -e "GET /info HTTP/1.0\r\n" | nc -U /var/run/docker.sock

According to this,
(exec 3<>/dev/tcp/url/port; cat >&3; cat <&3; exec 3<&-)
can replace nc/netcat. It should work in any bash based terminal.
Example:
printf "Hello World!" | (exec 3<>/dev/tcp/termbin.com/9999; cat >&3; cat <&3; exec 3<&-)
returns a link.

socat is a more powerful version of nc and netcat.

Have you got Perl? You could do something like this maybe:
perl -MLWP::Simple -e "getprint('http://localhost')"

Follow a implementation in Python for making sockets connections and sending data in tcp and udp:
import socket
def netcat(hostname, port, content=None, protocol='tcp'):
print('')
if protocol == 'tcp':
s = socket.socket() # (socket.AF_INET, socket.SOCK_STREAM)
if protocol == 'udp':
s = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
if protocol != 'tcp' and protocol != 'udp':
print("Error: Protocol must be 'tcp' or 'udp'")
try:
s.connect((hostname, port))
print('Connection Success to ' + hostname + ':' + str(port) + '/' + protocol)
except:
print('Connection failed to ' + hostname + ':' + str(port) + '/' + protocol)
return None
if content != None:
print('Starting to send content: ' + str(content))
s.send(str.encode(content))
# s.sendall(content)
hasHacievedAnyData = False
while True:
s.settimeout(10)
try:
data = s.recv(1024)
except Exception:
if hasHacievedAnyData:
print('Info: Timeout while expecting to receve more data')
else:
print('Error: Timeout while expecting to receve data')
break
if len(data) == 0:
break
print('Received:' + str(repr(data)))
hasHacievedAnyData = True
s.shutdown(socket.SHUT_WR)
s.close()
print('Connection closed.')
#Examples of usage
netcat('localhost', 443)
netcat('localhost', 3478)
netcat('localhost', 3478, protocol='udp')
netcat('localhost', 16384, 'Hello', 'udp')

Python is ubiquitous these days and the socket module is all you need.
Here are a few examples:
You can use it to test connectivity on port 443 to a list of 3 hosts:
import socket
def test_socket(ip,port):
s = socket.socket()
try:
s.settimeout(3)
s.connect((ip,port))
except socket.error as msg:
s.close()
print 'could not open %s:%s %s' % (ip,port,msg)
return(1)
else:
s.close()
print '%s:%s is OK' % (ip,port)
return(0)
hosts=['host1.example.com','host2.example.com','host3.example.com']
for host in hosts:
print "testing %s 443" % host
test_socket(host,443)
This one liner can read stdin or files and send to hostname termbin.com on port 9999 upload a file to termbin:
python -c "import socket,fileinput; s = socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.connect(('termbin.com', 9999)) ; [ s.send(b) for b in fileinput.input() ]; print s.recv(1024); s.close();" filetoupload.txt

Related

Make a shell pipeline started from subprocess.Popen fail if the left-hand side of a pipe fails

Im running a bash command with subprocess.popen in python:
cmd = "bwa-mem2/bwa-mem2 mem -R \'#RG\\tID:2064-01\\tSM:2064-01\\tLB:2064-01\\tPL:ILLUMINA\\tPU:2064-01\' reference_genome/human_g1k_v37.fasta BHYHT7CCXY.RJ-1967-987-02.2_1.fastq BHYHT7CCXY.RJ-1967-987-02.2_2.fastq -t 14 | samtools view -bS -o dna_seq/aligned/2064-01/2064-01.6.bam -"
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
The problem is that I get returncode 0 even if the first command fails.
I have googled and found out about pipefail and it seems that this is what I should use.
However, I don't understand where to write it. I have tried:
"set -o pipefail && bwa-mem2/bwa-mem2 mem -R \'#RG\\tID:2064-01\\tSM:2064-01\\tLB:2064-01\\tPL:ILLUMINA\\tPU:2064-01\' reference_genome/human_g1k_v37.fasta BHYHT7CCXY.RJ-1967-987-02.2_1.fastq BHYHT7CCXY.RJ-1967-987-02.2_2.fastq -t 14 | samtools view -bS -o dna_seq/aligned/2064-01/2064-01.6.bam -"
which gives: /bin/sh: 1: set: Illegal option -o pipefail
any ideas how I should incorporate this?
Edit:
I'm not sure if it is correct to edit my answer when responding to an answer? there was not enough characters to respond in a comment:/
Anyway,
I tried your second approach without shell=True #Charles Duffy.
(cmd_1 and cmd_2 are equal to what you wrote in your solution)
This is the code I use:
try:
p1 = Popen(shlex.split(cmd_1), stdout=PIPE)
p2 = Popen(shlex.split(cmd_2), stdin=p1.stdout, stdout=PIPE, stderr=STDOUT, text=True)
p1.stdout.close()
output, error = p2.communicate()
p1.wait()
rc_1 = p1.poll()
rc_2 = p2.poll()
print("rc_1:", rc_1)
print("rc_2:", rc_2)
if rc_1 == 0 and rc_2 == 0:
self.log_to_file("DEBUG", "# Process ended with returncode = 0")
if text: self.log_to_file("INFO", f"{text} succesfully
else:
print("Raise exception")
raise Exception(f"stdout: {output} stderr: {error}")
except Exception as e:
print(f"Error: {e} in misc.run_command()")
self.log_to_file("ERROR", f"# Process ended with returncode != 0, {e}")
this is the result i get when deliberately causing an error by renaming one file:
[E::main_mem] failed to open file `/home/jonas/BASE/dna_seq/reads/2064-01/test_BHYHT7CCXY.RJ-1967-987-02.2_2.fastq.gz'.
free(): double free detected in tcache 2
rc_1: -6
rc_2: 0
Raise exception
Error: stdout: stderr: None in misc.run_command()
ERROR: # Process ended with returncode != 0, stdout: stderr: None
It seems to capture the faulty returncode.
But why is stdout empty and stderr= None?
How can I capture the output to have it logged to a logger both when the process is successful and when it fails?
First, With A Shell
Instead of letting shell=True specify sh by default, specify bash explicitly to ensure that pipefail is an available feature:
shell_script = r'''
set -o pipefail || exit
bwa-mem2/bwa-mem2 mem \
-R '#RG\tID:2064-01\tSM:2064-01\tLB:2064-01\tPL:ILLUMINA\tPU:2064-01' \
reference_genome/human_g1k_v37.fasta \
BHYHT7CCXY.RJ-1967-987-02.2_1.fastq \
BHYHT7CCXY.RJ-1967-987-02.2_2.fastq \
-t 14 \
| samtools view -bS \
-o dna_seq/aligned/2064-01/2064-01.6.bam -
'''
process = subprocess.Popen(["bash", "-c", shell_script],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
This works, but it's not the best available option.
Second, With No Shell At All
p1 = subprocess.Popen(
['bwa-mem2/bwa-mem2', 'mem',
'-R', r'#RG\tID:2064-01\tSM:2064-01\tLB:2064-01\tPL:ILLUMINA\tPU:2064-01',
'reference_genome/human_g1k_v37.fasta',
'BHYHT7CCXY.RJ-1967-987-02.2_1.fastq',
'BHYHT7CCXY.RJ-1967-987-02.2_2.fastq', '-t', '14'],
stdout=subprocess.PIPE)
p2 = subprocess.Popen(
['samtools', 'view', '-bS',
'-o', 'dna_seq/aligned/2064-01/2064-01.6.bam', '-'],
stdin=p1.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True)
p1.stdout.close()
output, _ = p2.communicate() # let p2 finish running
p1.wait() # ensure p1 has properly exited
print(f'bwa-mem2 exited with status {p1.returncode}')
print(f'samtools exited with status {p2.returncode}')
...which lets you check p1.returncode and p2.returncode separately.

Subprocess to run lftp on Python 3.6

I'm trying to write a sub-process to do an lftp (bash command lftp) in Python. It works perfectly in Python 2.7 but fails in Python 3.6
Working code in Python 2.7
>>> uname = 'bunnylorr#gmail.com'
>>> pass_wrd = '*********'
>>> flink = 'ftps://ftp.filestorage.com:990'
>>> src_dir = 'Acct\ Files'
>>> src_frmt = 'compact_'
>>> import subprocess
>>> cmd = """
... filelist=$(lftp -c "open -u {usr},{pwd} {url};cd {sdir};cls -1| grep {sname}")
... echo "INFO: List of files to download: \n$filelist \n"
... for file in $filelist; do lftp -c "open -u {usr},{pwd} {url};cd {sdir};get $file"; \
... echo "INFO: File downloaded to local - $file \n"; done """ \
... .format(usr = uname, pwd = pass_wrd, url = flink, sdir = src_dir, sname = src_frmt )
>>> ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT); cmd_opt = ps.communicate()[0]
>>> print(cmd_opt)
Output Received:
INFO: List of files to download:
compact_wizard_20201211.csv
compact_accounts_20201211.csv
compact_register_20201211.csv
INFO: File downloaded to local - compact_wizard_20201211.csv
INFO: File downloaded to local - compact_accounts_20201211.csv
INFO: File downloaded to local - compact_register_20201211.csv
However I tried the same code on python 3.6 I get the below error
>>> uname = 'bunnylorr#gmail.com'
>>> pass_wrd = '*********'
>>> flink = 'ftps://ftp.filestorage.com:990'
>>> src_dir = 'Acct\ Files'
>>> src_frmt = 'compact_'
>>> import subprocess
>>> cmd = """
... filelist=$(lftp -c "open -u {usr},{pwd} {url};cd {sdir};cls -1| grep {sname}")
... echo "INFO: List of files to download: \n$filelist \n"
... for file in $filelist; do lftp -c "open -u {usr},{pwd} {url};cd {sdir};get $file"; \
... echo "INFO: File downloaded to local - $file \n"; done """ \
... .format(usr = uname, pwd = pass_wrd, url = flink, sdir = src_dir, sname = src_frmt )
>>> ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT); cmd_opt = ps.communicate()[0]
>>> print(cmd_opt)
Error: b'/bin/sh: lftp: command not found\n'
Any recommendations or suggestions on this?
Thanks!!
Installed lftp on my server and reran the Python cmd. It worked!!
sudo apt-get install lftp

ssh port forwarding ("ssh -fNL") doesn't work via expect spawn to automatically provide password

I know that to do port forwarding, the command is ssh -L. I also use other options to decorate it. So for example, a final full command may look like this ssh -fCNL *:10000:127.0.0.1:10001 127.0.0.1. And everything just works after entering password.
Then, because there is not only one port need to be forwarded, I decide to leave the job to shell script and use expect(tcl) to provide passwords(all the same).
Although without a deep understanding of expect, I managed to write the code with the help of Internet. The script succeeds spawning ssh and provides correct password. But I end up finding there is no such process when I try to check using ps -ef | grep ssh and netstat -anp | grep 10000.
I give -v option to ssh and the output seems to be fine.
So where is the problem? I have searched through Internet but most of questions are not about port forwarding. I'm not sure whether it is proper to use expect while I just want to let script automatically provide password.
Here the script.
#!/bin/sh
# Port Forwarding
# set -x
## function definition
connection ()
{
ps -ef | grep -v grep | grep ssh | grep $1 | grep $2 > /dev/null
if [ $? -eq 0 ] ; then
echo "forward $1 -> $2 done"
exit 0
fi
# ssh-keygen -f "$HOME/.ssh/known_hosts" -R "127.0.0.1"
/usr/bin/expect << EOF
set timeout 30
spawn /usr/bin/ssh -v -fCNL *:$1:127.0.0.1:$2 127.0.0.1
expect {
"yes/no" {send "yes\r" ; exp_continue}
"password:" {send "1234567\r" ; exp_continue}
eof
}
catch wait result
exit [lindex \$result 3]
EOF
echo "expect ssh return $?"
echo "forward $1 -> $2 done"
}
## check expect available
which expect > /dev/null
if [ $? -ne 0 ] ; then
echo "command expect not available"
exit 1
fi
login_port="10000"
forward_port="10001"
## check whether the number of elements is equal
login_port_num=$(echo ${login_port} | wc -w)
forward_port_num=$(echo ${forward_port} | wc -w)
if [ ${login_port_num} -ne ${forward_port_num} ] ; then
echo "The numbers of login ports and forward ports are not equal"
exit 1
fi
port_num=${login_port_num}
## provide pair of arguments to ssh main function
index=1
while [ ${index} -le ${port_num} ] ; do
login_p=$(echo ${login_port} | awk '{print $'$index'}')
forward_p=$(echo ${forward_port} | awk '{print $'$index'}')
connection ${login_p} ${forward_p}
index=$((index + 1))
done
Here the output from script
spawn /usr/bin/ssh -v -fCNL *:10000:127.0.0.1:10001 127.0.0.1
OpenSSH_7.2p2 Ubuntu-4ubuntu2.10, OpenSSL 1.0.2g 1 Mar 2016
...
debug1: Next authentication method: password
wang#127.0.0.1's password:
debug1: Enabling compression at level 6.
debug1: Authentication succeeded (password).
Authenticated to 127.0.0.1 ([127.0.0.1]:22).
debug1: Local connections to *:10000 forwarded to remote address 127.0.0.1:10001
debug1: Local forwarding listening on 0.0.0.0 port 10000.
debug1: channel 0: new [port listener]
debug1: Local forwarding listening on :: port 10000.
debug1: channel 1: new [port listener]
debug1: Requesting no-more-sessions#openssh.com
debug1: forking to background
expect ssh return 0
forward 10000 -> 10001 done
This should work for you:
spawn -ignore SIGHUP ssh -f ...
UPDATE:
Another workaround is:
spawn bash -c "ssh -f ...; sleep 1"
UPDATE 2 (a bit explanation):
ssh -f calls daemon() to make itself a daemon. See ssh.c in the souce code:
/* Do fork() after authentication. Used by "ssh -f" */
static void
fork_postauth(void)
{
if (need_controlpersist_detach)
control_persist_detach();
debug("forking to background");
fork_after_authentication_flag = 0;
if (daemon(1, 1) == -1)
fatal("daemon() failed: %.200s", strerror(errno));
}
daemon() is implemented like this:
int
daemon(int nochdir, int noclose)
{
int fd;
switch (fork()) {
case -1:
return (-1);
case 0:
break;
default:
_exit(0);
}
if (setsid() == -1)
return (-1);
if (!nochdir)
(void)chdir("/");
if (!noclose && (fd = open(_PATH_DEVNULL, O_RDWR, 0)) != -1) {
(void)dup2(fd, STDIN_FILENO);
(void)dup2(fd, STDOUT_FILENO);
(void)dup2(fd, STDERR_FILENO);
if (fd > 2)
(void)close (fd);
}
return (0);
}
There's a race condition (not sure if its the correct term for here) between _exit() in the parent process and setsid() in the child process. Here _exit() would always complete first since "the function _exit() terminates the calling process immediately" and setsid() is much more heavy weight. So when the parent process exits, setsid() is not effective yet and the child process is still in the same session as the parent process. According to the apue book (I'm referring to the 2005 edition, Chapter 10: Signals), SIGHUP "is also generated if the session leader terminates. In this case, the signal is sent to each process in the foreground process group."
In brief:
Expect allocates a pty and runs ssh on the pty. Here, ssh would be running in a new session and be the session leader.
ssh -f calls daemon(). The parent process (session leader) calls _exit(). At this time, the child process is still in the session so it'll get SIGHUP whose default behavior is to terminate the process.
How the workarounds works:
The nohup way (spawn -ignore SIGHUP) is to explicitly ask the process to ignore SIGHUP so it'll not be terminated.
For bash -c 'sshh -f ...; sleep 1', bash would be the session leader and sleep 1 in the end prevents the session leader from exiting too soon. So after sleep 1, the child ssh process's setsid() has already done and child ssh is already in a new process session.
UPDATE 3:
You can compile ssh with the following modification (in ssh.c) and verify:
static int
my_daemon(int nochdir, int noclose)
{
int fd;
switch (fork()) {
case -1:
return (-1);
case 0:
break;
default:
// wait a while for child's setsid() to complete
sleep(1);
// ^^^^^^^^
_exit(0);
}
if (setsid() == -1)
return (-1);
if (!nochdir)
(void)chdir("/");
if (!noclose && (fd = open(_PATH_DEVNULL, O_RDWR, 0)) != -1) {
(void)dup2(fd, STDIN_FILENO);
(void)dup2(fd, STDOUT_FILENO);
(void)dup2(fd, STDERR_FILENO);
if (fd > 2)
(void)close (fd);
}
return (0);
}
/* Do fork() after authentication. Used by "ssh -f" */
static void
fork_postauth(void)
{
if (need_controlpersist_detach)
control_persist_detach();
debug("forking to background");
fork_after_authentication_flag = 0;
if (my_daemon(1, 1) == -1)
// ^^^^^^^^^
fatal("my_daemon() failed: %.200s", strerror(errno));
}

python3: can't restore the out on console to a file from the program beginning to end& pexpect.EOF issue

Below is my code about using pexpect module achieve SSH logon function.
#!/usr/bin/env python
import pexpect
import sys
#use ssh to logon server
user="inteuser" #username
host="146.11.85.xxx" #host ip
password="xxxx" #password
command="ls -l" #list file on home/user directory
child = pexpect.spawn('ssh -l %s %s %s'%(user, host, command))
child.expect('password:')
child.sendline(password)
childlog = open('prompt.log',"ab") # restore prompt log to file prompt.log
__console__ = sys.stdout # make a backup of system output to console
sys.stdout = childlog # print the system output to childlog
child.expect(pexpect.EOF)
childlog.close()
sys.stdout = __console__ # back to the original state of system output
print(child.before) # print the contents before match expect function
after I execute my script
[~/Liaohaifeng]$ python3 ssh_test.py
b' \r\ntotal 69636\r\n-rw-rw-r-- 1 inteuser inteuser 949 Nov 28 02:01
01_eITK_trtest01_CrNwid.log\r\n
[~/Liaohaifeng]$ cat prompt.log
total 69412
-rw-rw-r-- 1 inteuser inteuser 949 Nov 28 02:01 01_eITK_trtest01_CrNwid.log
I think this result is not my expected. when I remove the code child.expect(pexpect.EOF) in my script, the output about print(child.before) can be correct(it should print the content before matching password)
Below is the output after I remove child.expect(pexpect.EOF)
[~/Liaohaifeng]$ python3 ssh_test.py
b"\r\n-------------------------------------------------------------------------------\r\n...
These computer resources are provided for authorized users only. For legal,
\r\n
security and cost reasons, utilization and access of resources are sxx, in\r\n
accordance with approved internal procedures, at any time if IF YOU ARE NOT AN AUTHORIZED USER; PLEASE EXIT IMMEDIATELY...\r\n "
my purpose is print out all the output to a file after executing the script,but the log file still only contains the output of listing directory. So why this happen? could you please help update my script? thank you very much.
You can use the spawn().logfile_read.
[STEP 101] # cat example.py
import pexpect, sys
child = pexpect.spawn('bash --norc')
if sys.version_info[0] <= 2:
# python2
child.logfile_read = open('/tmp/pexpect.log', 'w')
else:
# python3
fp = open('/tmp/pexpect.log', 'w')
child.logfile_read = fp.buffer
child.expect('bash-[.0-9]+[$#] ')
child.sendline('echo hello world')
child.expect('bash-[.0-9]+[$#] ')
child.sendline('exit')
child.expect(pexpect.EOF)
child.logfile_read.close()
[STEP 102] # python3 example.py
[STEP 103] # cat /tmp/pexpect.log
bash-4.4# echo hello world
hello world
bash-4.4# exit
exit
[STEP 104] #
It is a simple question, just adjust code order is OK.
#!/usr/bin/env python
import pexpect
import sys
#use ssh to logon server
user="inteuser" #username
host="146.11.85.xxx" #host ip
password="xxxx" #password
command="ls -l" #list file on home/user directory
child = pexpect.spawn('ssh -l %s %s %s'%(user, host, command))
childlog = open('prompt.log',"ab")
child.logfile = childlog
child.expect('password:')
child.sendline(password)
child.expect(pexpect.EOF)
childlog.close()

How do we suppress tshark default output

How do we suppress tshark default output. Please see the given below sample output for more detail:
system#client:~$ sudo tshark -q -i eth0 -f "tcp" -a duration:3 -w /tmp/test.pcap
[sudo] password for system:
tshark: Lua: Error during loading:
[string "/usr/share/wireshark/init.lua"]:45: dofile has been disabled
Running as user "root" and group "root". This could be dangerous.
Capturing on eth0
2 packets captured
system#client:~$
===================================================================
Testbed:
Client <---------------> Server
Code Sample:
def execTShark(self,cmd):
try:
self.SSHCONN.sendline('')
self.SSHCONN.expect('[#\$>]')
self.SSHCONN.maxread=20000
self.output = ""
self.SSHCONN.sendline( cmd )
while 1:
i = self.SSHCONN.expect (['password', '[#\$>]'])
self.output = self.output + self.SSHCONN.before
if i==0:
self.SSHCONN.sendline(self.password)
elif i==1:
self.SSHCONN.sendline('')
self.SSHCONN.expect (['[#\$>]', 'Capturing'])
self.output = self.output + self.SSHCONN.before
self.SSHCONN.sendline('')
break
self.SSHCONN.expect('[#\$>]')
self.output = self.output + self.SSHCONN.before
print self.output
except Exception as ex:
print("An exception occurred, " + str(ex))
return False
return True
Problem:
1.When i call the above mentioned function, execTShark with the cmd="sudo tshark -q -i eth0 -f "tcp" -a duration:3 -w /tmp/test.pcap &", sometime it failed, while waiting for this message "Capturing on eth0".

Resources