resolve 'Can't export GPIO problem on Rpi using python script - python-3.x

I am writing a python application which is programming and testing atmel microcontrollers through a SPI port.
The application is running on RaspberryPi model 3B+ and I use the command line application 'avrdude' to do the job. I use subprocess.Popen() from within my python script and in general this runs just fine.
Sometimes, the SPI port gets in a blocked state. The avrdude application then typically reports something like 'Can't export GPIO 8, already exported/busy?: Device or resource busy'
One can observe the exported GPIO's by:
pi#LeptestPost:/ $ ls /sys/class/gpio/
export gpio10 gpio11 gpio8 gpiochip0 gpiochip504 unexport
I get out of this situation by invoking:
pi#LeptestPost:/ $ sudo echo 8 > /sys/class/gpio/unexport
resulting in:
pi#LeptestPost:/ $ ls /sys/class/gpio/
export gpio10 gpio11 gpiochip0 gpiochip504 unexport
So I can unexport them all and move on manually but I would like to have this automated in the application with the following code (after detecting the error in the avrdude output):
args = ['sudo', 'echo', '8', '>', '/sys/class/gpio/unexport']
result, error = self.runCommand(args, wait=True)
def runCommand(self, args, wait = False, outputFileStr = "", errorFileStr = "", workingDir = ""):
# documentation:
#class subprocess.Popen(args,
# bufsize=-1,
# executable=None,
# stdin=None,
# stdout=None,
# stderr=None,
# preexec_fn=None,
# close_fds=True,
# shell=False,
# cwd=None,
# env=None,
# universal_newlines=None,
# startupinfo=None,
# creationflags=0,
# restore_signals=True,
# start_new_session=False,
# pass_fds=(),
# *,
# encoding=None,
# errors=None,
# text=None)
print("Working on executing command " + str(args))
if (outputFileStr != ""):
stdoutFile = open(outputFileStr,'w')
else:
stdoutFile = None
if (errorFileStr != ""):
stderrFile = open(errorFileStr,'w')
else:
stderrFile = None
if (workingDir != ""):
cwdStr = workingDir
else:
cwdStr = None
try:
if (wait):
p = subprocess.Popen(args, stdout = subprocess.PIPE, cwd = cwdStr)
print("started subprocess with PID " + str(p.pid))
p.wait() # Wait for child process to terminate, This will deadlock when using stdout=PIPE or stderr=PIPE
else:
#p = subprocess.Popen(args, stdin = None, stdout = None, stderr = None, close_fds = True)
p = subprocess.Popen(args, stdin = None, stdout = stdoutFile, stderr = stderrFile, close_fds = True, cwd = cwdStr)
print("started subprocess with PID " + str(p.pid))
(result, error) = p.communicate(timeout=15) # Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached.
except subprocess.CalledProcessError as e:
sys.stderr.write("common::run_command() : [ERROR]: output = %s, error code = %s\n" % (e.output, e.returncode))
return e.output, e.returncode
except FileNotFoundError as e:
self.master.printERROR("common::run_command() : [ERROR]: output = %s, error code = " + str(e) + "\n")
return "error", str(e)
except subprocess.TimeoutExpired as e:
self.master.printERROR("Process timeout on PID "+ str(p.pid) + ", trying to kill \n")
p.kill()
outs, errs = p.communicate()
return "error", str(e)
if (outputFileStr != ""):
stdoutFile.close()
if (errorFileStr != ""):
stderrFile.close()
return result, error
But that does not do the job (no error, not the wanted result). I can imagine it's related to how the process is started within its shell or environment - but that's beyond my knowledge.
Any idea how to get this working?
Note: the avrdude application is also called using the 'runcommand' method and running fine.

Related

I have a python script that works perfectly in the Thonny IDE, but fails in terminal

Firstly, here's the code:
#!/usr/bin/python3
import re, pexpect, os
file = '/home/homebridge/flags/Restart.flag'
f = open(file, 'w')
f.close()
os.system("sudo systemctl stop homebridge")
os.system("sudo chmod -R a+rwx /var/lib/homebridge")
child = pexpect.spawn('tuya-cli wizard')
child.expect('\r\n')
child.sendline('y')
child.expect('\r\n')
child.sendline('XXXXXXXXXXXXXXXX')
data = child.read()
data = data.decode("utf-8")
devices = data.split('},')
devicesO = []
class device:
name = ""
ID = ""
key = ""
def __init__(self, name, ID, key):
self.name = name
self.ID = ID
self.key = key
def __lt__(self, other):
return self.name < other.name
for i in devices:
n = re.search("name: \'(.*)\'", str(i)).group(1)
I = re.search("id: \'(.*)\'", str(i)).group(1)
k = re.search("key: \'(.*)\'", str(i)).group(1)
if n != ("Clock"):
devicesO.append(device(n, I, k))
entries = []
devicesO.sort()
for device in devicesO:
if "phone charger" not in device.name:
s1 = "{\n\"name\": \"" + device.name + "\",\n\"id\": \"" + device.ID + "\",\n\"key\": \"" + device.key + "\","
s2 = """
"type": "RGBTWLight",
"manufacturer": "SmartLife",
"model": "Light",
"dpPower": "20",
"dpMode": "21",
"dpBrightness": "22",
"dpColorTemperature": "23",
"dpColor": "24",
"colorFunction": "HSB",
"scaleBrightness": 1000
}"""
else:
s1 = "{\n\"name\": \"" + device.name + "\",\n\"id\": \"" + device.ID + "\",\n\"key\": \"" + device.key + "\","
s2 = """
"type": "Outlet",
"manufacturer": "SmartLife",
"model": "Outlet",
"dpPower": "1"
}"""
entries.append(s1 + s2)
string = ",\n".join([str(entry) for entry in entries])
config = open('/var/lib/homebridge/config.json', 'r+')
x = config.read()
config.close()
#print(x)
x = re.sub("\"TuyaLan\",\n.*\"devices\": \[((.|\n)*?)\]", "\"TuyaLan\",\n\"devices\": [\n" + string + "\n]", x)
#print(x)
#x = re.sub("\"TuyaLan\",\n.*\"devices\": \[((.|\n)*?)\]", "\"TuyaLan\",\n.*\"devices\": [\nTEST\n]", x)
config = open('/var/lib/homebridge/config.json', 'w+')
config.write(x)
config.close()
config = open('/var/lib/homebridge/config.json', 'r+')
print (config.read())
config.close()
os.remove(file)
os.system("sudo systemctl restart homebridge")
This executes as expected in the IDE, stopping the homebridge service, pulling relevant data from the tuya-cli utility, regex and text replacement, all of it. However, when I try and run it in the terminal without sudo, the first regex search returns an empty object and the script fails. When I run it with sudo, it stalls for a while then times out on the pexpect step at the beginning. I've researched before posting, but I have no clue how to solve this one. It doesn't appear to be a path issue, I used pip3 to install both re and pexpect, and os is obviously packaged with the raspbian install. Any clues would be great.
Error without sudo:
pi#raspberrypi:~ $ /home/homebridge/scripts/updateConfig.py
Traceback (most recent call last):
File "/home/homebridge/scripts/updateConfig.py", line 34, in <module>
n = re.search("name: \'(.*)\'", str(i)).group(1)
AttributeError: 'NoneType' object has no attribute 'group'
With sudo:
pi#raspberrypi:~ $ sudo /home/homebridge/scripts/updateConfig.py
Traceback (most recent call last):
File "/home/homebridge/scripts/updateConfig.py", line 10, in <module>
child.expect('\r\n')
File "/usr/local/lib/python3.7/dist-packages/pexpect/spawnbase.py", line 344, in expect
timeout, searchwindowsize, async_)
File "/usr/local/lib/python3.7/dist-packages/pexpect/spawnbase.py", line 372, in expect_list
return exp.expect_loop(timeout)
File "/usr/local/lib/python3.7/dist-packages/pexpect/expect.py", line 181, in expect_loop
return self.timeout(e)
File "/usr/local/lib/python3.7/dist-packages/pexpect/expect.py", line 144, in timeout
raise exc
pexpect.exceptions.TIMEOUT: Timeout exceeded.
<pexpect.pty_spawn.spawn object at 0x766c4510>
command: /usr/bin/tuya-cli
args: ['/usr/bin/tuya-cli', 'wizard']
buffer (last 100 chars): b'\x1b[32m?\x1b[39m \x1b[1mThe API key from tuya.com:\x1b[22m\x1b[0m \x1b[0m\x1b[29D\x1b[29C'
before (last 100 chars): b'\x1b[32m?\x1b[39m \x1b[1mThe API key from tuya.com:\x1b[22m\x1b[0m \x1b[0m\x1b[29D\x1b[29C'
after: <class 'pexpect.exceptions.TIMEOUT'>
match: None
match_index: None
exitstatus: None
flag_eof: False
pid: 1470
child_fd: 5
closed: False
timeout: 30
delimiter: <class 'pexpect.exceptions.EOF'>
logfile: None
logfile_read: None
logfile_send: None
maxread: 2000
ignorecase: False
searchwindowsize: None
delaybeforesend: 0.05
delayafterclose: 0.1
delayafterterminate: 0.1
searcher: searcher_re:
0: re.compile(b'\r\n')
Possible short answer: Your IDE is probably automatically adding carriage returns with your sendlines, which is why your code runs in the IDE, but not at the terminal. Sendline adds a line feed (\n), but not a carriage return (\r). You should add a \r after each sendline (e.g., child.sendline('XXXXXXXXXXXXXXXX\r')) to complete the CRLF (\r\n).
Long explanation:
Based on your code, when you spawned the child, you expected a CRLF. However, pexpect searches are not greedy and will stop at the first CRLF they encounter. Unfortunately, when I tested your code, pexpect stopped at the CRLF after the command you entered, not the prompt afterwards:
child = pexpect.spawn('tuya-cli wizard')
child.expect('\r\n')
print(child.before)
Output
b" tuya-cli wizard"
You should be looking for a prompt or a message instead, such as The API key from tuya.com: or Password::
# tuya-cli wizard
The API key from tuya.com:
The API secret from tuya.com:
Provide a 'virtual ID' of a device currently registered in the app:
or
# sudo tuya-cli wizard
Password:
However, I think both of your errors occurred because you did not include a carriage return (\r) with your sendlines. The first error occurred because, at the prompt, The API key from tuya.com:, you sent 'y', not 'y\r', so nothing was entered at the prompt. You then searched for CRLF, but since you had not included a \r, pexpect found the original CRLF after b" tuya-cli wizard".
The expect call actually caused a carriage return, but, unfortunately, your code was now a step behind, and was interacting with the previous prompt, not the current prompt. That is why data = child.read() ended up reading the wrong output, resulting in a NoneType object.
The second error occurred because the pexpect cursor moved up to the The API key from tuya.com: prompt, looking for a CRLF. Since it is a prompt, it does not end with a CRLF, so the pexpect search timed out (those \x1b escape sequences are just for formatting and color):
pexpect.exceptions.TIMEOUT: Timeout exceeded.
args: ['/usr/bin/tuya-cli', 'wizard']
before (last 100 chars): b'\x1b[32m?\x1b[39m \x1b[1mThe API key from tuya.com:\x1b[22m\x1b[0m \x1b[0m\x1b[29D\x1b[29C'
searcher: searcher_re:
0: re.compile(b'\r\n')
Note that there is no \r\n in the buffer or before byte strings.
I would do something like:
...
while True:
index = child.expect(
["Password:", "The API key from tuya.com:", pexpect.TIMEOUT, pexpect.EOF, ])
if index == 0:
password = getpass() # You will need 'from getpass import getpass'
child.sendline(password) # CR's are usually not needed with variables
elif index == 1:
# This is what you want
child.sendline("XXXXXXXXXXXXXXXX\r")
break
elif index == 2:
raise RuntimeError("Search string not found.")
elif index ==3:
raise RuntimeError("Child closed.")
child.expect("The API secret from tuya.com:")
child.sendline("XXXXXXXXXXXXXXXX\r")
...
Good luck with your code!

python3 - easiest way to share variable from parent to child using scripts

Using script parent.py I would like to set variable parvar and execute child.py and have parvar printed. I am having a hard time wrapping my head around the easiest way to accomplish this. It seems like I could use os.fork() as data present in the parent are presented to the child, but I cannot get it to work. Reading through examples of using multiprocessing I cannot find examples that show sharing data across two different scripts like this.
This is what I have so far:
parent.py
#!/usr/bin/env python3
import subprocess, os
parvar = 'parent var'
pid = os.fork()
if pid == 0:
print('child pid is running')
subprocess.call(['python3', 'child.py'])
exit()
child.py
#!/usr/bin/env python3
childvar = 'child var'
print('this is child var: ', childvar)
print(parvar)
Which returns a NameError:
$ ./parent.py
child pid is running
$ this is child var: child var
Traceback (most recent call last):
File "child.py", line 4, in <module>
print(parvar)
NameError: name 'parvar' is not defined
I think I understand why that is not working. The subprocess call replaces the existing process - spawning a new one. Because that was not forked, whatever I made available to my child PID is now inaccessible to that 3rd process.
Can someone help me with a simple example of getting the above to work?
This is not an answer (not yet at least) but figured I would post this here since it seems to move things in the right direction.
Using the mmap example from this page:
https://blog.schmichael.com/2011/05/15/sharing-python-data-between-processes-using-mmap/
I have (I believe correctly) re-factored it for Python 3 (3.6.8):
a.py
#!/usr/bin/env python3
import ctypes
import mmap
import os
import struct
def main():
# Create new empty file to back memory map on disk
fd = os.open('/tmp/mmaptest', os.O_CREAT | os.O_TRUNC | os.O_RDWR)
# Zero out the file to insure it's the right size
assert os.write(fd, b'\x00' * mmap.PAGESIZE) == mmap.PAGESIZE
# Create the mmap instace with the following params:
# fd: File descriptor which backs the mapping or -1 for anonymous mapping
# length: Must in multiples of PAGESIZE (usually 4 KB)
# flags: MAP_SHARED means other processes can share this mmap
# prot: PROT_WRITE means this process can write to this mmap
buf = mmap.mmap(fd, mmap.PAGESIZE, mmap.MAP_SHARED, mmap.PROT_WRITE)
# Now create an int in the memory mapping
i = ctypes.c_int.from_buffer(buf)
# Set a value
i.value = 10
# And manipulate it for kicks
i.value += 1
assert i.value == 11
# Before we create a new value, we need to find the offset of the next free
# memory address within the mmap
offset = struct.calcsize(i._type_)
# The offset should be uninitialized ('\x00')
assert buf[offset] == 0
# Now ceate a string containing 'foo' by first creating a c_char array
s_type = ctypes.c_char * len('foo')
# Now create the ctypes instance
s = s_type.from_buffer(buf, offset)
# And finally set it
s.raw = b'foo'
print('First 10 bytes of memory mapping: %r' % buf[:10])
input('Now run b.py and press ENTER')
print
print('Changing i')
i.value *= i.value
print('Changing s')
s.raw = b'bar'
new_i = input('Enter a new value for i: ')
i.value = int(new_i)
if __name__ == '__main__':
main()
b.py
#!/usr/bin/env python3
import mmap
import os
import struct
import time
def main():
# Open the file for reading
fd = os.open('/tmp/mmaptest', os.O_RDONLY)
# Memory map the file
buf = mmap.mmap(fd, mmap.PAGESIZE, mmap.MAP_SHARED, mmap.PROT_READ)
i = None
s = None
while 1:
new_i, = struct.unpack('i', buf[:4])
new_s, = struct.unpack('3s', buf[4:7])
if i != new_i or s != new_s:
print('i: %s => %d' % (i, new_i))
print('s: %s => %s' % (s, new_s))
print('Press Ctrl-C to exit')
i = new_i
s = new_s
time.sleep(1)
if __name__ == '__main__':
main()
Execution Example
terminal 1
$ ./a.py
First 10 bytes of memory mapping: b'\x0b\x00\x00\x00foo\x00\x00\x00'
Now run b.py and press ENTER
Changing i
Changing s
Enter a new value for i: 87
terminal 2
$ ./b.py
i: None => 11
s: None => b'foo'
Press Ctrl-C to exit
i: 11 => 121
s: b'foo' => b'bar'
Press Ctrl-C to exit
i: 121 => 87
s: b'bar' => b'bar'
Press Ctrl-C to exit

Python IP logger taking in, and outputting data in csv format

Hello I'm looking for some help with a project I recently took up to help check pc connectivity in my workplace. I am fairly new to python and programming in general so a large portion of this may be wrong. I'm trying to create a simple IP logger that will ping the destination address and return a value to determine the connection state of the machine. It will take the data from a CSV file (excell.xlsx) and splice the IP address from the information provided in cell style format, then output the connection state with a simple "connected" or "disconnected" after relisting the input file. Here's what I've come up with so far:
import csv, platform, subprocess, os as sys
#Defines filepath locations (Adjustment may be required)
in_path = sys.os.path['C:\Users\User\Documents\IpLog\log1.xlsx']
out_path = sys.os.path['C:\Users\User\Documents\IpLog\ip_log.xlsx']
try:
ip = 0
reference = out_path
host = '?'
line_count = 0
with open(in_path, dialect='excel', delimeter=',', newline='') as csvfile:
for row in csvfile:
if line_count == 0:
ip_inp = csv.reader(in_path, dialect='excel') #adjust filename as needed.
ip = ip_inp[ip_inp.index('10.21')]
ip = ip[5::]
line_count += 1
for line in ip_inp:
def ping(ip):
"""
Returns True if host (str) responds to a ping request.
Remember that a host may not respond to a ping (ICMP) request even if the host name is valid.
"""
# Option for the number of packets as a function of
param = '-n' if platform.system().lower() == 'windows' else '-c'
# Building the command. Ex: "ping -c 1 google.com"
command = ['ping', param, '1', ip]
return subprocess.call(command) == 0
if subprocess.call(command) == 0:
status = subprocess.call(command)
else:
status = 1
if status == 0:
if line_count == 0:
net_state = 'Connected'
with open(out_path, delimiter=',') as csvfile2:
print(csvfile)
csv.writer(csvfile2, dialect='excel')
print(net_state)
else:
net_state = 'Disconnected'
with open(out_path, delimiter=',') as csvfile2:
print(csvfile)
csv.writer(csvfile2, dialect='excel')
print(net_state)
except IOError:
print('Error opening the input file please check file input name and file location and try again.')
finally:
raise SystemExit('Operation Completed. Results have been written to (%s)' % out_path)
Right now the error I keep running into is truncation in my global file path variables. Can anyone tell me why this may be happening?

python3 pexpect strange behaviour

I have a single threaded program that simply executes commands over ssh and simply looks for output over ssh. But after a while I start getting extrodinarily strange behaviour:
ssh_cmd = 'ssh %s#%s %s' % (user, addr, options)
ssh = pexpect.spawn(ssh_cmd, timeout=60)
lgsuc = ['(?i)(password)')]
for item in loginsuccess:
lgsuc.append(item)
retval = ssh.expect(lgsuc)
for cmd in cmdlist:
time.sleep(0.1)
#this is regex to match anything. Essentially clears the buffer so you don't get an invalid match from before
ssh.expect(['(?s).*'])
ssh.sendline(cmd)
foundind = ssh.expect([re.escape("root#")], 30) #very slow
#repr escape all the wierd stuff so madness doesn't happen with ctr chars
rettxt = repr(ssh.before.decode("us-ascii") + "root:#")
print("We Found:" + rettxt
And it will be fine for about 20 commands or so then madness occurs Assume the right echo is blablabla each time:
We found 'blablabla \r\n\r\n[edit]\r\nroot#'
We found 'blablabla \r\n\r\n[edit]\r\nroot#'
We found 'blablabla \r\n\r\n[edit]\r\nroot#'
... about 20 commands...
We found 'bl\r\nroot#' # here it just missed part of the string in the middle
We found 'alala\r\nroot#'
here is the remainder of the echo from the previous command!!! and the echo of the current command will show up some time later!! and it gets worse and worse. The thing that is strange is it is in the middle of the return byte array.
Now there are some wierd control codes coming back from this device so if I replace:
rettxt = repr(ssh.before.decode("us-ascii") + "root:#")
with
rettxt = repr(ssh.before.decode("us-ascii") + "root:#")
then
print("We Found:" + rettxt)
returns:
root#e Found lala
Anyway there is really strange stuff going on with pexpect and the buffers, and I can't figure out what it is so any help would be appreciated. I should mention I never get teh timeout, the dive always responds. Also the total number of "root:#"in the log file exceedes the total number of lines sent.
If I go through and remove all ctl codes, the output looks cleaner but the problem still persists, its as if pextect cant handle ctl coodes in its buffer correctly. Any help is appreciated
UPDATE Minimum verifiable example
Ok I have been able to recreate PART of the problem on an isolated ubuntu environment sshing into itself.
first I need to create 4 commands that can be run on a host target, so put the follwing for files in ~/ I did this in ubuntu
~/check.py
#!/usr/bin/python3
import time
import io
#abcd^H^H^H^H^MABC
#mybytes = b'\x61\x62\x63\x64\x08\x08\x08\x0D\x41\x42\x43'
#abcdACB
mybytes = b'\x61\x62\x63\x64\x41\x42\x43'
f = open('test.txt', 'wb')
#time.sleep(1)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
~/check2.py
#!/usr/bin/python3
import time
import io
#0123^H^H^H^H^MABC
mybytes = b'\x30\x31\x32\x33\x08\x0D\x0D\x08\x08\x08\x08\x0D\x41\x42\x43'
f = open('test2.txt', 'wb')
#time.sleep(0.1)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
~/check3.py
#!/usr/bin/python3
import time
import io
#789:^H^H^H^H^DABC
mybytes = b'\x37\x38\x39\x3A\x08\x08\x08\x08\x08\x08\x08\x0D\x0D\x41\x42\x43'
f = open('test3.txt', 'wb')
#time.sleep(0.1)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
And lastly check4.py Sorry it took a wierd combination for the problem to show back up
#!/usr/bin/python3
import time
import io
#abcd^H^H^H^HABC
mybytes = b'\x61\x62\x63\x64\x08\x08\x08\x0D\x41\x42\x43'
f = open('test.txt', 'wb')
time.sleep(4)
f.write(mybytes)
print(mybytes.decode('ascii'))
f.close()
Noticing that the last one has a bigger sleep, this is to encounter texpect timeout. Though on my actual testing this doesn't occue, I have commands that take over 6 minutes to return any text so this might be part of it. Ok and the final file to run everything. It might look ugly but I did a massive trim so I could post it here:
#! /usr/bin/python3
#import paramiko
import time
import sys
import xml.etree.ElementTree as ET
import xml
import os.path
import traceback
import re
import datetime
import pexpect
import os
import os.path
ssh = None
respFile = None
#Error Codes:
DEBUG = True
NO_ERROR=0
EXCEPTION_THROWS=1
RETURN_TEXT_NEVER_FOUND = 2
LOG_CONSOLE = 1
LOG_FILE = 2
LOG_BOTH = 3
def log(out, dummy=None):
print(str(log))
def connect(user, addr, passwd):
global ssh
fout = open('session.log', 'wb')
#options = '-q -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oPubkeyAuthentication=no'
options = ' -oUserKnownHostsFile=/dev/null '
#options = ''
#REPLACE WITH YOU LOCAL USER NAME
#user = 'user'
#REPLACE WITH YOUR LOCAL PASSWORD
#passwd = '123TesT321'
#addr = '127.0.0.1'
ssh_cmd = 'ssh %s#%s %s' % (user, addr, options)
public = None
private = None
retval = 0
try:
ssh = pexpect.spawn(ssh_cmd, timeout=60)
ssh.logfile = fout
#the most common prompts
loginsuccess = [re.escape("#"), re.escape("$")]
lgsuc = ['(?i)(password)', re.escape("connecting (yes/no)? ")]
for item in loginsuccess:
lgsuc.append(item)
retval = ssh.expect(lgsuc)
except pexpect.TIMEOUT as exc:
log("Server never connected to SSH tunnel")
return 0
print('where here ret val = ' + str(retval))
try:
if(retval > 1):
return 1
elif(retval == 1):
hostkey = ssh.before.decode("utf-8")
ssh.sendline("yes")
log("Warning! new host key was added to the database: " + hostkey.split("\n")[1])
lgsuc = ['password: ']
for item in loginsuccess:
lgsuc.append(item)
retval = ssh.expect(lgsuc)
if(retval > 0):
return 1
else:
if(public is not None):
log("Warning public key authentication failed trying password if available...")
else:
if public is not None:
log("Warning public key authentication failed trying password if available...")
if(passwd is None):
log("No password and certificate authentication failed...")
return 0
ssh.sendline(passwd)
login = ['password: ' ]
for item in loginsuccess:
login.append(item)
retval = ssh.expect(login)
except pexpect.TIMEOUT as exc:
log("Server Never responded with expected login prompt: "+lgstr)
return 0
#return 0
if retval > 0:
retval = 1
if retval == 0:
log("Failed to connect to IP:"+addr +" User:"+user+" Password:"+passwd)
return retval
def disconnect():
log("Disconnecting...")
global ssh
if ssh is not None:
ssh.close()
else:
log("Something wierd happened with the SSH client while closing the session. Shouldn't really matter", False)
def RunCommand(cmd, waitTXT, timeout = 5):
global ssh
Error = 0
if DEBUG:
print('Debugging: cmd: '+ cmd+'. timeout: '+str(timeout) +'. len of txt tags: '+ str(len(waitTXT)))
if(type(waitTXT) is str):
waitTXT = [re.excape(waitTXT)]
elif(not hasattr(waitTXT ,'__iter__')):
waitTXT = [re.escape(str(waitTXT))]
else:
cnter = 0
for TXT in waitTXT:
waitTXT[cnter] = re.escape(str(TXT))
cnter +=1
#start = time.time()
#print("type3: "+str(type(ssh)))
#time.sleep(1)
#this is regex to match anything. Essentially clears the buffer so you don't get an invalid match from before
ssh.expect(['(?s).*'])
ssh.sendline(cmd)
print("Debugging: sent: "+cmd)
#GoOn = True
rettxt = ""
try:
foundind = ssh.expect(waitTXT, timeout)
allbytes = ssh.before
newbytes = bytearray()
for onebyte in allbytes:
if onebyte > 31:
newbytes.append(onebyte)
allbytes = bytes(newbytes)
rettxt = repr(allbytes.decode("us-ascii") + waitTXT[foundind])
#rettxt = ssh.before + waitTXT[foundind]
if DEBUG:
print("Debugging: We found "+rettxt)
except pexpect.TIMEOUT as exc:
if DEBUG:
txtret = ""
for strtxt in waitTXT:
txtret += strtxt +", "
print("ERROR Debugging: we timed out waiting for text:"+txtret)
pass
return (rettxt, Error)
def CloseAndExit():
disconnect()
global respFile
if respFile is not None and '_io.TextIOWrapper' in str(type(respFile)):
if not respFile.closed:
respFile.close()
def main(argv):
try:
cmds = ['~/check.py', '~/check2.py', '~/check3.py', '~/check2.py', '~/check3.py','~/check.py', '~/check2.py', '~/check3.py', '~/check2.py', '~/check3.py', '~/check4.py', '~/check3.py','~/check.py', '~/check2.py',]
##CHANGE THESE TO MTACH YOUR SSH HOST
ret = connect('user', '127.0.0.1', 'abcd1234')
for cmd in cmds:
cmdtxt = str(cmd)
#rett = RunCommand(ssh, "ls", "root", 0, 5)
strlen = (170 - (len(cmdtxt)))/2
dashval = ''
starval = ''
tcnt = 0
while(tcnt < strlen):
dashval +='-'
starval +='*'
tcnt +=1
if DEBUG:
print(dashval+cmdtxt+dashval)
#checkval = ['ABC']
#REPLACE THE FOLLOWING LINE WITH YOUR TARGET PROMPT
checkval = ['user-virtual-machine:~$']
rett = RunCommand(cmdtxt, checkval, 2)
if DEBUG:
print(starval+cmdtxt+starval)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
print(traceback.format_exc())
CloseAndExit()
#disconnect()
#respFile.close()
main(sys.argv)
Make sure that all for checks and the main python script are executble in permission via sudo chmod 774 or similar. In the main function call set your username ipaddress and password to where your target is that has the check.py and make sure they are in your ~/ directory.
Once you run this you can look at the session.log and at least on mind there is some wierd stuff going on with the buffer:
~/check4.py^M
~/check3.py
~/check3.py^M
abcd^H^H^H^MABC^M
^[]0;user#user-virtual-machine: ~^Guser#user-virtual-machine:~$ ~/check.py
~/check3.py^M
789:^H^H^H^H^H^H^H^M^MABC^M
^[]0;user#user-virtual-machine: ~^Guser#user-virtual-machine:~$ ~/check.py~/check2.py
And unfortunately its not as corrupt as my actual prbolem but I have several hundred commands I an embedded custom linux kernal that I obviously can't recreate for everyone. But anyway any help is greatly appreciated. Hopefully these examples work you you, I am just on ubuntu 16.04 lts. Also make sure to replace 'user-virtual-machine:~$' with whatever your target login prompt looks like

Piping output using subprocess.Popen and subprocess.PIPE [duplicate]

I know how to run a command using cmd = subprocess.Popen and then subprocess.communicate.
Most of the time I use a string tokenized with shlex.split as 'argv' argument for Popen.
Example with "ls -l":
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
However, pipes seem not to work... For instance, the following example returns noting:
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l | sed "s/a/b/g"'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
Can you tell me what I am doing wrong please?
Thx
I think you want to instantiate two separate Popen objects here, one for 'ls' and the other for 'sed'. You'll want to pass the first Popen object's stdout attribute as the stdin argument to the 2nd Popen object.
Example:
p1 = subprocess.Popen('ls ...', stdout=subprocess.PIPE)
p2 = subprocess.Popen('sed ...', stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()
You can keep chaining this way if you have more commands:
p3 = subprocess.Popen('prog', stdin=p2.stdout, ...)
See the subprocess documentation for more info on how to work with subprocesses.
I've made a little function to help with the piping, hope it helps. It will chain Popens as needed.
from subprocess import Popen, PIPE
import shlex
def run(cmd):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
output, err, exit_code = run("ls -lha /var/log | grep syslog | grep gz")
if exit_code != 0:
print "Output:"
print output
print "Error:"
print err
# Handle error here
else:
# Be happy :D
print output
shlex only splits up spaces according to the shell rules, but does not deal with pipes.
It should, however, work this way:
import subprocess
import shlex
sp_ls = subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_sed = subprocess.Popen(shlex.split(r'sed "s/a/b/g"'), stdin = sp_ls.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_ls.stdin.close() # makes it similiar to /dev/null
output = sp_ls.communicate()[0] # which makes you ignore any errors.
print output
according to help(subprocess)'s
Replacing shell pipe line
-------------------------
output=`dmesg | grep hda`
==>
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
HTH
"""
Why don't you use shell
"""
def output_shell(line):
try:
shell_command = Popen(line, stdout=PIPE, stderr=PIPE, shell=True)
except OSError:
return None
except ValueError:
return None
(output, err) = shell_command.communicate()
shell_command.wait()
if shell_command.returncode != 0:
print "Shell command failed to execute"
return None
return str(output)
Thank #hernvnc, #glglgl, and #Jacques Gaudin for the answers. I fixed the code from #hernvnc. His version will cause hanging in some scenarios.
import shlex
from subprocess import PIPE
from subprocess import Popen
def run(cmd, input=None):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
if input:
p[i]=Popen(shlex.split(cmd_part),stdin=PIPE, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
# close the stdin explicitly, otherwise, the following case will hang.
if input:
p[0].stdin.write(input)
p[0].stdin.close()
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
# test case below
inp = b'[ CMServer State ]\n\nnode node_ip instance state\n--------------------------------------------\n1 linux172 10.90.56.172 1 Primary\n2 linux173 10.90.56.173 2 Standby\n3 linux174 10.90.56.174 3 Standby\n\n[ ETCD State ]\n\nnode node_ip instance state\n--------------------------------------------------\n1 linux172 10.90.56.172 7001 StateFollower\n2 linux173 10.90.56.173 7002 StateLeader\n3 linux174 10.90.56.174 7003 StateFollower\n\n[ Cluster State ]\n\ncluster_state : Normal\nredistributing : No\nbalanced : No\ncurrent_az : AZ_ALL\n\n[ Datanode State ]\n\nnode node_ip instance state | node node_ip instance state | node node_ip instance state\n------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n1 linux172 10.90.56.172 6001 P Standby Normal | 2 linux173 10.90.56.173 6002 S Primary Normal | 3 linux174 10.90.56.174 6003 S Standby Normal'
cmd = "grep -E 'Primary' | tail -1 | awk '{print $3}'"
run(cmd, input=inp)

Resources