How to use tabulate in python3 dictionary while using loops - python-3.x

I'm trying to print the dictionary data into a tabular form , for now i see tabulate module as a easy way to test but somehow the data i'm getting thats coming the good way but the header informaion is repeating on each run for the user ID, please guide or suggest how to do that.....
$ cat checktable.py
#!/usr/bin/python3
import subprocess
import pandas as pd
from tabulate import tabulate
def CheckUid(user):
proc = subprocess.Popen("ldapsearch -h ldapserver -D 'cn=directory manager' -w pass123 -LLLb 'ou=people,o=rraka.com' 'uid=%s' managerlogin" % (user), shell=True, stdout=subprocess.PIPE)
info_str = proc.stdout.read().decode('utf8')
split_str = info_str.split()
if len(split_str) > 1:
raw_data = {'UserID': [split_str[1].split(',')[0].split('=')[1]], 'MangerID': [split_str[-1]]}
headers = ["UserID", "MangerID"]
return tabulate(raw_data, headers, tablefmt="simple")
else:
split_str = 'null'
def CallUid():
with open('hh', mode='rt', encoding='utf-8') as f:
for line in f.readlines():
print(CheckUid(line))
if __name__ == '__main__':
CallUid()
This returns the below data:
$ ./checktable.py
UserID MangerID
-------- ----------
aashishp rpudota
UserID MangerID
-------- ----------
abaillie davem
UserID MangerID
-------- ----------
abishek kalyang
UserID MangerID
Expected output:
$ ./checktable.py
UserID MangerID
-------- ----------
aashishp rpudota
abaillie davem
abishek kalyang
Another alternative code:
#!/usr/bin/python3
import sys
import subprocess
from tabulate import tabulate
def CheckUid(user):
proc = subprocess.Popen("ldapsearch -h its3 -D 'cn=directory manager' -w JatetRE3 -LLLb 'ou=people,o=cadence.com' 'uid=%s' managerlogin" % (user), shell=True, stdout=subprocess.PIPE)
info_str = proc.stdout.read().decode('utf8')
split_str = info_str.split()
if len(split_str) > 1:
raw_data = {'UserID': split_str[1].split(',')[0].split('=')[1], 'Manger': split_str[-1]}
for key, value in raw_data.items():
print(key, ":", value)
else:
split_str = 'null'
def CallUid():
with open('hh', mode='rt', encoding='utf-8') as f:
for line in f.readlines():
CheckUid(line)
if __name__ == '__main__':
CallUid()
It comes as below, where i need every two line two be into one..
$ ./checktable2.py
UserID : aashishp
Manger : rpudota
UserID : abaillie
Manger : davem
While desired would be:
$ ./checktable2.py
UserID : aashishp Manger : rpudota
UserID : abaillie Manger : davem

After Struggling as a learner, I came around with the below variant of codes as a solution to my own questions:
1) The First code is using the pandas module:
$ cat check_ldapUserdata.py
#!/usr/bin/python3
import pandas as pd
import subprocess
user_list = []
mngr_list = []
def CheckUid(user):
proc = subprocess.Popen("ldapsearch -h ldapserver -D 'cn=directory manager' -w JatetRE3 -LLLb 'ou=people,o=rraka.com' 'uid=%s' managerlogin" % (user), shell=True, stdout=subprocess.PIPE)
info_str = proc.stdout.read().decode('utf8')
split_str = info_str.split()
if len(split_str) > 1:
user = split_str[1].split(',')[0].split('=')[1]
manager = split_str[-1]
user_list.append(user)
mngr_list.append(manager)
else:
split_str = 'null'
def DataList():
df = pd.DataFrame({'User':user_list, 'Manager':mngr_list})
df = df[['User', 'Manager']] # To keep the order of columns
#return df
print(df)
def CallUid():
with open('testu', mode='rt', encoding='utf-8') as f:
for line in f.readlines():
CheckUid(line)
if __name__ == '__main__':
CallUid()
DataList()
Result Output is as Follows...
$ ./check_ldapUserdata.py
User Manager
0 karn benjamin
1 niraj vikashg
2 vaithees benjamin
3 mauj benjamin
2) The another way I achived it with using Regular Expression &
BeautifulTable module to get the table Format..
$ cat check_ldapUserdata2.py
#!/usr/bin/python3
import re
import subprocess
from beautifultable import BeautifulTable
table = BeautifulTable()
table.column_headers = ["User", "Manager"]
def CheckUid(user):
proc = subprocess.Popen("ldapsearch -h ldapserver -D 'cn=directory manager' -w pass123 -LLLb 'ou=people,o=rraka.com' 'uid=%s' managerlogin" % (user), shell=True, stdout=subprocess.PIPE)
info_str = proc.stdout.read().decode('utf8')
pat_match = re.match(".*uid=(.*?)\,.*\nmanagerlogin:\s+(.*)",info_str)
if pat_match:
table.append_row([pat_match.group(1), pat_match.group(2)])
def CallUid():
input_file=input("Please enter the file name : ")
with open(input_file, mode='rt', encoding='utf-8') as f:
for line in f.readlines():
CheckUid(line)
print(table)
if __name__ == '__main__':
CallUid()
Result Output as as below....
$ ./check_ldapUserdata2.py
Please enter the file name : testu
+----------+----------+
| User | Manager |
+----------+----------+
| karn | benjamin |
+----------+----------+
| niraj | vikashg |
+----------+----------+
| vaithees | benjamin |
+----------+----------+
| mauj | benjamin |
+----------+----------+
3) Another Simple non tabular Form but working ..
$ cat check_table_working1.py
#!/usr/bin/python3
import subprocess
def CheckUid(user):
proc = subprocess.Popen("ldapsearch -h ldapserver -D 'cn=directory manager' -w pass123 -LLLb 'ou=people,o=rraka.com' 'uid=%s' managerlogin" % (user), shell=True, stdout=subprocess.PIPE)
info_str = proc.stdout.read().decode('utf8')
split_str = info_str.split()
if len(split_str) > 1:
raw_data = {split_str[1].split(',')[0].split('=')[1] : split_str[-1]}
#raw_data = {'UserID': split_str[1].split(',')[0].split('=')[1], 'Manger': split_str[-1]}
for key, value in raw_data.items():
#print(key, ":", value)
print('{} : {}'.format(key, value))
else:
split_str = 'null'
def CallUid():
with open('hh', mode='rt', encoding='utf-8') as f:
for line in f.readlines():
CheckUid(line)
if __name__ == '__main__':
CallUid()
Result output of the above is as below...
$ ./check_table_working1.py
aashishp : rpudota
abaillie : davem
abishek : kalyang
adik : venky
adithya : jagi

Related

Stop Creating New List on Each Iteration

The following code extracts the subtitle data from mkv files. I want the output in one list but it creates a list for each track. How can the code be changed to give one list containing all the tracks rather than a list for each track?
#!/usr/bin/env python3
import os
import re
import json
import subprocess
from itertools import chain
mkvmerge = "/usr/bin/mkvmerge"
keep_lang = "eng"
#############################################################################
def extract_subs(mkv_list):
subtitle_tracks = []
video_tracks = []
audio_tracks = []
extractList = []
for file in mkv_list:
### Commandline auguments for extracting info about the mkv file
command = [mkvmerge, "-i", "-F", "json", file]
# Ask mkvmerge for the json info
process = subprocess.run(command, capture_output=True, text=True, check=True)
stdout = process.stdout
### Process the json response
json__data = json.loads(stdout)
tracks = json__data.get('tracks', [])
### find audio and subtitle tracks
audio = []
subtitle = []
track_list = []
for track in tracks:
track['properties']['id'] = track['id']
if track['type'] == 'audio':
audio.append(track)
elif track['type'] == 'subtitles':
subtitle.append(track)
# filter out files that don't need processing
if len(audio) < 2 and len(subtitle) < 2:
pass
#print("\nNo extracted subs to process.", file)
subtitle_keep = list(filter(lambda a: a ['properties']['language']==keep_lang, subtitle))
for s in subtitle_keep:
track_list.append(f"Track #{s['id']}: " f"{s['properties'].get('language')}" f" - " f"{s['codec']}: " + file)
print (track_list)
output
['Track #2: eng - SubRip/SRT: /home/mp/torrents/test/Belfast (2021)/Belfast (2021).mkv', 'Track #3: eng - SubRip/SRT: /home/mp/torrents/test/Belfast (2021)/Belfast (2021).mkv']
['Track #2: eng - SubRip/SRT: /home/mp/torrents/test/The Rescue (2021)/The Rescue (2021).mkv']
desired output
['Track #2: eng - SubRip/SRT: /home/mp/torrents/test/Belfast (2021)/Belfast (2021).mkv', 'Track #3: eng - SubRip/SRT: /home/mp/torrents/test/Belfast (2021)/Belfast (2021).mkv', 'Track #2: eng - SubRip/SRT: /home/mp/torrents/test/The Rescue (2021)/The Rescue (2021).mkv']
It seems like that your variable track_list has the wrong scope.
Try moving it out of the scope of the for loop.
#!/usr/bin/env python3
import os
import re
import json
import subprocess
from itertools import chain
mkvmerge = "/usr/bin/mkvmerge"
keep_lang = "eng"
#############################################################################
def extract_subs(mkv_list):
subtitle_tracks = []
video_tracks = []
audio_tracks = []
extractList = []
track_list = []
for file in mkv_list:
### Commandline auguments for extracting info about the mkv file
command = [mkvmerge, "-i", "-F", "json", file]
# Ask mkvmerge for the json info
process = subprocess.run(command, capture_output=True, text=True, check=True)
stdout = process.stdout
### Process the json response
json__data = json.loads(stdout)
tracks = json__data.get('tracks', [])
### find audio and subtitle tracks
audio = []
subtitle = []
for track in tracks:
track['properties']['id'] = track['id']
if track['type'] == 'audio':
audio.append(track)
elif track['type'] == 'subtitles':
subtitle.append(track)
# filter out files that don't need processing
if len(audio) < 2 and len(subtitle) < 2:
pass
#print("\nNo extracted subs to process.", file)
subtitle_keep = list(filter(lambda a: a ['properties']['language']==keep_lang, subtitle))
for s in subtitle_keep:
track_list.append(f"Track #{s['id']}: " f"{s['properties'].get('language')}" f" - " f"{s['codec']}: " + file)
print (track_list)

Piping bash commands using Python subprocess.call - Syntax error: "|" unexpected

I was testing pipe using subprocess.call and I came to the following problem.
For example an automated ssh using users as strings and as a text file.
The pipe works perfectly using strings (Example 1) but it fails when opening a text file to check the users (Example 2).
Is there a limitation when piping values from a file?
Example 1:
$ python3 script.py myhost
import subprocess
import sys
host = sys.argv[1]
for j in range(0, 2):
words = ['test1', 'test2']
user = words[j]
print(user)
print('Loggin in %s...' % repr(user))
subp = subprocess.call(['echo %s | ssh %s' % (user, host)],
shell=True, stdout=None, stderr=None)
Output:
test1
Trying 'test1'...
Logged in
test2
Trying 'test2'...
Logged in
Example 2:
$ python3 script.py myhost users.txt
import subprocess
import sys
host = sys.argv[1]
user_list = sys.argv[2]
with open(user_list) as usr:
user = usr.readline()
cnt = 0
while user:
print(user.strip())
user = usr.readline()
subp = subprocess.call(['echo %s | ssh %s' % (user, host)],
shell=True, stdout=None, stderr=None)
cnt += 1
Output:
test1
/bin/sh: 2: Syntax error: "|" unexpected
test2
/bin/sh: 2: Syntax error: "|" unexpected
In case someone stumbles into this, I ended up fixing by getting rid of the first readline, adding a strip and modifying the while condition:
with open(user_list) as usr:
cnt = 0
while usr:
line = usr.readline()
user = line.strip()
subp = subprocess.call(['echo %s | ssh %s' % (user, host)],
shell=True, stdout=None, stderr=None)
cnt += 1

Dataflow pipeline triggering NotImplementedError [while running 'Filter Status 1']

My pipeline has the following simple JSON input
{"mac": "KC:FC:48:AE:F6:94", "status": 8, "datetime": "2015-07-13T21:15:02Z"}
The output should basically go to a BigQuery table with 3 columns (mac, status and datetime) with their corresponding values
My Pipeline looks as follows:
# -*- coding: utf-8 -*-
import os, json, logging, argparse, datetime, apache_beam as beam
from google.cloud import error_reporting
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import StandardOptions
from apache_beam.options.pipeline_options import GoogleCloudOptions
GOOGLE_PUBSUB_CHANNEL = 'projects/project-name/topics/topic-name'
GOOGLE_BIGQUERY_TABLE = 'bq-table'
GOOGLE_DATASET_ID = 'bq-dataset'
GOOGLE_PROJECT_ID = 'project-name'
class GoogleBigQuery():
client_error = error_reporting.Client()
#staticmethod
def get_schema_table(schema):
bigquery_schema = []
for key in range(len(schema)):
bigquery_schema.append('{}:{}'.format(schema[key].get('bigquery_field_name'), schema[key].get('bigquery_field_type')))
return ','.join(bigquery_schema)
fields_contract = (
{ 'bigquery_field_name': 'datetime', 'bigquery_field_type': 'STRING' },
{ 'bigquery_field_name': 'mac', 'bigquery_field_type': 'STRING' },
{ 'bigquery_field_name': 'status', 'bigquery_field_type': 'INTEGER' }
)
def parse_pubsub(line):
record = json.loads(line)
logging.info(record)
return record
class FilterStatus1(beam.DoFn):
def status_filter_1(self, data):
for r in data:
print(r)
logging.info(r)
if r["status"] == 1:
print(r)
logging.info(r)
yield r
def run(argv=None):
parser = argparse.ArgumentParser()
known_args, pipeline_args = parser.parse_known_args(argv)
pipeline_parameters = [
'--runner', 'DirectRunner'
, '--staging_location', 'gs://bucket/staging'
, '--temp_location', 'gs://bucket/temp'
, '--autoscaling_algorithm', 'THROUGHPUT_BASED' #'NONE' to disable autoscaling
, '--num_workers', '1'
, '--max_num_workers', '2'
, '--disk_size_gb', '30'
, '--worker_machine_type', 'n1-standard-1'
]
pipeline_options = PipelineOptions(pipeline_parameters)
pipeline_options.view_as(StandardOptions).streaming = True
pipeline_options.view_as(GoogleCloudOptions).job_name = os.path.basename(__file__).split('.')[0].replace('_', '-')
pipeline_options.view_as(GoogleCloudOptions).project = GOOGLE_PROJECT_ID
with beam.Pipeline(options=pipeline_options, argv=pipeline_parameters) as p:
# Read the pubsub topic into a PCollection.
lines = (
p
| 'ReadPubSubMessage' >> beam.io.ReadFromPubSub(GOOGLE_PUBSUB_CHANNEL).with_output_types(bytes)
| 'Decode UTF-8' >> beam.Map(lambda x: x.decode('utf-8'))
| 'ParsePubSub' >> beam.Map(parse_pubsub)
)
(
lines | 'Filter Status 1' >> beam.ParDo(FilterStatus1())
| 'WriteToBigQueryStatus1' >> beam.io.WriteToBigQuery(
GOOGLE_BIGQUERY_TABLE
, project=GOOGLE_PROJECT_ID
, dataset=GOOGLE_DATASET_ID
, schema=GoogleBigQuery.get_schema_table(fields_contract)
, create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED
, write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND
#, write_disposition=beam.io.BigQueryDisposition.WRITE_TRUNCATE
)
)
logging.info('Pipeline finished')
result = p.run()
result.wait_until_finish()
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
run()
I'm getting the following error:
RuntimeError: NotImplementedError [while running 'Filter Status 1']
My goal here is to filter the status column and when the value is 1 to stream it into BQ.
Thanks in advance for helping me out.
You can try a filtering approach using FlatMap to do such things.
First, define a filtering method:
def FilterStatus1(row):
if row["status"] == 1:
yield row
Then you can apply like:
lines = lines | beam.FlatMap(FilterStatus1) | 'WriteToBigQueryStatus1' ...
Also, try breaking up your code into chunks or explicitly assigned steps. This giant transformation, mappings and filterings happening in a single row usually turn your code into a black-box.
Hope it helps. Thanks.
I fixed my code this way
class FilterStatus1(beam.DoFn):
def process(self, data):
if data["status"] == 1:
result = [{"datetime":data["datetime"], "mac":data["mac"], "status":data["status"]}]
logging.info(result)
return result

Python traceback and attribute errors for AWS and Boto3

Working on getting this script running on Python 3 from earlier version 2 and still getting some traceback and Attribute errors. Below is the error and below that the script in its current state: Special thanks for #Madlania Kalunder for sorting the printing issues. Now I know things are close but need some help.
Traceback (most recent call last):
File "ec2-instances.py", line 97, in <module> sys.exit(main())
File "ec2-instances.py", line 78, in main hosts=(list_instances)(filter)
File "ec2-instances.py", line 18, in list_instances
name = (item for item in i.tags if item ["Key"] == "Name" ).next()
AttributeError: 'generator' object has no attribute 'next'
Below is the code. Sorry for formatting issues here and thanks in advance for the help.
#!/usr/bin/env python
import boto3
import sys
import argparse
import paramiko
def list_instances(Filter):
ec2 = boto3.resource('ec2')
instances = ec2.instances.filter(Filters=Filter)
(columns_format) = ("%-3s %-26s %-16s %-16s %-20s %-12s %-12s %-16s")
print ((columns_format) % ("num", "Name", "Public IP", "Private IP", "ID", "Type", "VPC", "Status"))
num = 1
hosts = []
name = {}
for i in instances:
try:
name = (item for item in i.tags if item ["Key"] == "Name" ).next()
except StopIteration:
name['Value'] = ''
print (columns_format) % (
num,
name['Value'],
i.public_ip_address,
i.private_ip_address,
i.id,
i.instance_type,
i.vpc_id,
i.state['Name']
)
num = num + 1
item={'id': i.id, 'ip': i.public_ip_address, 'hostname': name ['Value'], 'status': i.state['Name'],}
hosts.append(item)
return hosts
def execute_cmd(host,user,cmd):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user)
stdin, stdout, stderr = ssh.exec_command(cmd)
stdout=stdout.read()
stderr=stderr.read()
ssh.close()
return stdout,stderr
except paramiko.AuthenticationException as exception:
return "Authentication Error trying to connect into the host %s with the user %s. Plese review your keys" % (host, user), e
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--name',
help="Filter result by name.")
parser.add_argument('-t', '--type',
help="Filer result by type.")
parser.add_argument('-s', '--status',
help="Filter result by status." )
parser.add_argument('-e', '--execute',
help="Execute a command on instances")
parser.add_argument('-u', '--user', default="ubuntu",
help="User to run commands if -e option is used.\
Ubuntu user is used by default")
arg = parser.parse_args()
# Default filter if no options are specified
filter=[]
if arg.name:
filter.append({'Name': 'tag-value', 'Values': ["*" + arg.name + "*"]})
if arg.type:
filter.append({'Name': 'instance-type', 'Values': ["*" + arg.type + "*"]})
if arg.status:
filter.append({'Name': 'instance-state-name', 'Values': ["*" + arg.status + "*"]})
hosts=list_instances(filter)
names = ""
if arg.execute:
for item in hosts:
names = names + " " + item["hostname"] + "(" + item["id"] + ")"
print ("\nCommand to execute: %s") % arg.execute
print ("Executed by: %s") % arg.user
print ("Hosts list: %s\n") % names
for item in hosts:
if item["status"] == 'running':
print ("::: %s (%s)") % (item["hostname"], item["id"])
stdout,stderr = execute_cmd(item["ip"], arg.user, arg.execute)
print (stdout)
print (stderr)
else:
print ("::: %s (%s) is not running (command execution skiped)") % (item["hostname"], item["id"])
if __name__ == '__main__':
sys.exit(main())
Got some help from a colleague and here is the now correct/working version. More print issues needed fixing with () to include the % and removing the specific exception rule. Also corrected the .next to include the generator.
#!/usr/bin/env python
import boto3
import sys
import argparse
import paramiko
def list_instances(Filter):
ec2 = boto3.resource('ec2')
instances = ec2.instances.filter(Filters=Filter)
(columns_format) = ("%-3s %-26s %-16s %-16s %-20s %-12s %-12s %-16s")
print ((columns_format) % ("num", "Name", "Public IP", "Private IP", "ID", "Type", "VPC", "Status"))
num = 1
hosts = []
name = {}
for i in instances:
try:
name = next(item for item in i.tags if item["Key"] == "Name")
except:
name['Value'] = ''
print ((columns_format) % (
num,
name['Value'],
i.public_ip_address,
i.private_ip_address,
i.id,
i.instance_type,
i.vpc_id,
i.state['Name']
))
num = num + 1
item={'id': i.id, 'ip': i.public_ip_address, 'hostname': name ['Value'], 'status': i.state['Name'],}
hosts.append(item)
return hosts
def execute_cmd(host,user,cmd):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(host, username=user)
stdin, stdout, stderr = ssh.exec_command(cmd)
stdout=stdout.read()
stderr=stderr.read()
ssh.close()
return stdout,stderr
except paramiko.AuthenticationException as exception:
return "Authentication Error trying to connect into the host %s with the user %s. Plese review your keys" % (host, user), e
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--name',
help="Filter result by name.")
parser.add_argument('-t', '--type',
help="Filer result by type.")
parser.add_argument('-s', '--status',
help="Filter result by status." )
parser.add_argument('-e', '--execute',
help="Execute a command on instances")
parser.add_argument('-u', '--user', default="ubuntu",
help="User to run commands if -e option is used.\
Ubuntu user is used by default")
arg = parser.parse_args()
# Default filter if no options are specified
filter=[]
if arg.name:
filter.append({'Name': 'tag-value', 'Values': ["*" + arg.name + "*"]})
if arg.type:
filter.append({'Name': 'instance-type', 'Values': ["*" + arg.type + "*"]})
if arg.status:
filter.append({'Name': 'instance-state-name', 'Values': ["*" + arg.status + "*"]})
hosts=list_instances(filter)
names = ""
if arg.execute:
for item in hosts:
names = names + " " + item["hostname"] + "(" + item["id"] + ")"
print (("\nCommand to execute: %s") % arg.execute)
print (("Executed by: %s") % arg.user)
print (("Hosts list: %s\n") % names)
for item in hosts:
if item["status"] == 'running':
print (("::: %s (%s)") % (item["hostname"], item["id"]))
stdout,stderr = execute_cmd(item["ip"], arg.user, arg.execute)
print (stdout)
print (stderr)
else:
print (("::: %s (%s) is not running (command execution skiped)") % (item["hostname"], item["id"]))
if __name__ == '__main__':
sys.exit(main())

Piping output using subprocess.Popen and subprocess.PIPE [duplicate]

I know how to run a command using cmd = subprocess.Popen and then subprocess.communicate.
Most of the time I use a string tokenized with shlex.split as 'argv' argument for Popen.
Example with "ls -l":
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
However, pipes seem not to work... For instance, the following example returns noting:
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l | sed "s/a/b/g"'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
Can you tell me what I am doing wrong please?
Thx
I think you want to instantiate two separate Popen objects here, one for 'ls' and the other for 'sed'. You'll want to pass the first Popen object's stdout attribute as the stdin argument to the 2nd Popen object.
Example:
p1 = subprocess.Popen('ls ...', stdout=subprocess.PIPE)
p2 = subprocess.Popen('sed ...', stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()
You can keep chaining this way if you have more commands:
p3 = subprocess.Popen('prog', stdin=p2.stdout, ...)
See the subprocess documentation for more info on how to work with subprocesses.
I've made a little function to help with the piping, hope it helps. It will chain Popens as needed.
from subprocess import Popen, PIPE
import shlex
def run(cmd):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
output, err, exit_code = run("ls -lha /var/log | grep syslog | grep gz")
if exit_code != 0:
print "Output:"
print output
print "Error:"
print err
# Handle error here
else:
# Be happy :D
print output
shlex only splits up spaces according to the shell rules, but does not deal with pipes.
It should, however, work this way:
import subprocess
import shlex
sp_ls = subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_sed = subprocess.Popen(shlex.split(r'sed "s/a/b/g"'), stdin = sp_ls.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_ls.stdin.close() # makes it similiar to /dev/null
output = sp_ls.communicate()[0] # which makes you ignore any errors.
print output
according to help(subprocess)'s
Replacing shell pipe line
-------------------------
output=`dmesg | grep hda`
==>
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
HTH
"""
Why don't you use shell
"""
def output_shell(line):
try:
shell_command = Popen(line, stdout=PIPE, stderr=PIPE, shell=True)
except OSError:
return None
except ValueError:
return None
(output, err) = shell_command.communicate()
shell_command.wait()
if shell_command.returncode != 0:
print "Shell command failed to execute"
return None
return str(output)
Thank #hernvnc, #glglgl, and #Jacques Gaudin for the answers. I fixed the code from #hernvnc. His version will cause hanging in some scenarios.
import shlex
from subprocess import PIPE
from subprocess import Popen
def run(cmd, input=None):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
if input:
p[i]=Popen(shlex.split(cmd_part),stdin=PIPE, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
# close the stdin explicitly, otherwise, the following case will hang.
if input:
p[0].stdin.write(input)
p[0].stdin.close()
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
# test case below
inp = b'[ CMServer State ]\n\nnode node_ip instance state\n--------------------------------------------\n1 linux172 10.90.56.172 1 Primary\n2 linux173 10.90.56.173 2 Standby\n3 linux174 10.90.56.174 3 Standby\n\n[ ETCD State ]\n\nnode node_ip instance state\n--------------------------------------------------\n1 linux172 10.90.56.172 7001 StateFollower\n2 linux173 10.90.56.173 7002 StateLeader\n3 linux174 10.90.56.174 7003 StateFollower\n\n[ Cluster State ]\n\ncluster_state : Normal\nredistributing : No\nbalanced : No\ncurrent_az : AZ_ALL\n\n[ Datanode State ]\n\nnode node_ip instance state | node node_ip instance state | node node_ip instance state\n------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n1 linux172 10.90.56.172 6001 P Standby Normal | 2 linux173 10.90.56.173 6002 S Primary Normal | 3 linux174 10.90.56.174 6003 S Standby Normal'
cmd = "grep -E 'Primary' | tail -1 | awk '{print $3}'"
run(cmd, input=inp)

Resources