In pyhocon can is it possible to include a file in runtime?
I know how to merge trees in runtime e.g.
conf = ConfigFactory.parse_file(conf_file)
conf1 = ConfigFactory.parse_file(include_conf)
conf = ConfigTree.merge_configs(conf, conf1)
I would like to emulate the include statement so that hocon variables get evaluated in runtime like in the wishful code below:
conf = ConfigFactory.parse_files([conf_file, include_conf])
Newer pyhocon has a possibility to parse a config file without resolving variables.
import pprint
from pyhocon import ConfigFactory
if __name__ == "__main__":
conf = ConfigFactory\
.parse_file('./app.conf', resolve=False)\
.with_fallback(
config='./app_vars.conf',
resolve=True,
)
pprint.pprint(conf)
app.conf:
smart: {
super_hero: ${super_hero}
}
app_vars.conf:
super_hero: Buggs_Bunny
It's possible to use strings:
from pyhocon import ConfigFactory
# readingconf entrypoint
file = open('application.conf', mode='r')
conf_string = file.read()
file.close()
conf_string_ready = conf_string.replace("PATH_TO_CONFIG","spec.conf")
conf = ConfigFactory.parse_string(conf_string_ready)
print(conf)
application.conf has
include "file://INCLUDE_FILE"
or in runtime without preparation I wrote it myself using python 3.65:
#!/usr/bin/env python
import os
from pyhocon import ConfigFactory as Cf
def is_line_in_file(full_path, line):
with open(full_path) as f:
content = f.readlines()
for l in content:
if line in l:
f.close()
return True
return False
def line_prepender(filename, line, once=True):
with open(filename, 'r+') as f:
content = f.read()
f.seek(0, 0)
if is_line_in_file(filename, line) and once:
return
f.write(line.rstrip('\r\n') + '\n' + content)
def include_configs(base_path, included_paths):
for included_path in included_paths:
line_prepender(filename=base_path, line=f'include file("{included_path}")')
return Cf.parse_file(base_path)
if __name__ == "__main__":
dir_path = os.path.dirname(os.path.realpath(__file__))
print(f"current working dir: {dir_path}")
dir_path = ''
_base_path = f'{dir_path}example.conf'
_included_files = [f'{dir_path}example1.conf', f'{dir_path}example2.conf']
_conf = include_configs(base_path=_base_path, included_paths=_included_files)
print('break point')
Related
i am wondering how to go about making this short test in python3, cmd is supported in python3 but dircache is not... currently only works on python2 but looking to change it to work in python3, thanks!
#!/usr/bin/env python
import cmd
import dircache
class MyCmd(cmd.Cmd):
def __init__(self):
cmd.Cmd.__init__(self)
self.prompt = '(MyCmd)'
def do_test(self, line):
print('cmd test ' + line)
def complete_test(self, text, line, begidx, endidx):
""" auto complete of file name.
"""
line = line.split()
if len(line) < 2:
filename = ''
path = './'
else:
path = line[1]
if '/' in path:
i = path.rfind('/')
filename = path[i+1:]
path = path[:i]
else:
filename = path
path = './'
ls = dircache.listdir(path)
ls = ls[:] # for overwrite in annotate.
dircache.annotate(path, ls)
if filename == '':
return ls
else:
return [f for f in ls if f.startswith(filename)]
if __name__ == '__main__':
mycmd = MyCmd()
mycmd.cmdloop()
Ive tried using this instead of dircache but had no luck
#lru_cache(32)
def cached_listdir(d):
return os.listdir(d)
Here you go check out the load function
import os
import glob
import cmd
def _append_slash_if_dir(p):
if p and os.path.isdir(p) and p[-1] != os.sep:
return p + os.sep
else:
return p
class MyShell(cmd.Cmd):
prompt = "> "
def do_quit(self, line):
return True
def do_load(self, line):
print("load " + line)
def complete_load(self, text, line, begidx, endidx):
before_arg = line.rfind(" ", 0, begidx)
if before_arg == -1:
return # arg not found
fixed = line[before_arg+1:begidx] # fixed portion of the arg
arg = line[before_arg+1:endidx]
pattern = arg + '*'
completions = []
for path in glob.glob(pattern):
path = _append_slash_if_dir(path)
completions.append(path.replace(fixed, "", 1))
return completions
MyShell().cmdloop()
I'm trying to make it so that the user chooses which function to run using if.
import os
import csv
import collections
import datetime
import pandas as pd
import time
import string
import re
import glob, os
folder_path = 'C:/ProgramData/WebPort/system/tags'
folder2_path = 'C:/ProgramData/WebPort/system'
search2_str = '"Prefix"'
print("Choices:\n 1 - Read from CSV\n 2 - Read from WPP")
x = input("Please enter your choice:\n")
x = int(x)
if x == 1:
csv_file_list = glob.glob(folder_path + '/*.csv')
with open("csv.txt", 'w') as wf:
for file in csv_file_list:
print(glob.glob(folder_path + '/*.csv'))
with open(file) as rf:
for line in rf:
if line.strip(): # if line is not empty
if not line.endswith("\n"):
line+="\n"
wf.write(line)
print('Reading from .csv')
elif x == 2:
for root, dirs, files in os.walk(folder2_path):
for file in files:
if file.endswith(".wpp"):
print(os.path.join(root, file))
with open(os.path.join(root, file), 'r') as fr, open ("wpp.txt",'a', encoding='utf-8') as fw:
for i,line in enumerate(fr):
if line.find(search2_str) != -1:
fw.write(line)
print('Reading from .wpp')
else:
print('wrong choice')
Getting Invalid syntax in line 34 using this.
I'm trying to download and iterate over csv file but I'm only reading the headers but no more lines after it
tried using this answer but with no luck
this is my code:
from datetime import datetime
import requests
import csv
def main():
print("python main function")
datetime_object = datetime.now().date()
url = f'https://markets.cboe.com/us/equities/market_statistics/volume_reports/day/{datetime_object}/csv/?mkt=bzx'
print(url)
response = requests.get(url, stream=True)
csv_content = response.content.decode('utf-8')
print(csv_content)
cr = csv.reader(csv_content.splitlines(), delimiter='~')
my_list = list(cr)
for row in my_list:
print(row)
if __name__ == '__main__':
main()
cr = csv.reader(csv_content.splitlines(), delimiter='~')
change to
cr = csv.reader(csv_content.splitlines(), delimiter=',')
And check if You download full file or file with header only use URL in browser ;)
I am a beginner in using python. I have created a plain text file and have to encrypt it to output file. But I am getting an error as below and unable to write it to output file. The code is running but the output file which should be encrypted is created.
#!/usr/bin/env python3
import os
import binascii
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.backends import default_backend
import argparse
def readfile_binary(file):
with open(file, 'rb') as f:
content = f.read()
return content
def writefile_binary(file, content):
with open(file, 'wb') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser(description = 'Encryption and Decryption of the file')
parser.add_argument('-in', dest = 'input', required = True)
parser.add_argument('-out', dest = 'output', required = True)
parser.add_argument('-K', dest = 'key', help = 'The key to be used for encryption must be in hex')
parser.add_argument('-iv', dest = 'iv', help = 'The inintialisation vector, must be in hex')
args = parser.parse_args()
input_content = readfile_binary(args. input)
output_content = writefile_binary(args. output)
if __name__ == "__main__":
main()
The output file should be encrypted and it should be available in the directory.
These two lines:
input_content = readfile_binary(args. input)
output_content = writefile_binary(args. output)
There should not be a space in args.input. Here is an example,
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('filename')
args = parser.parse_args()
# using type hints can help reasoning about code
def write(filename: str, content: str) -> None:
with open(filename, 'wb') as f:
f.write(str.encode(content))
# if the filename was successfully parsed from stdin
if args.filename == 'filename.txt':
print(f"args: {args.filename}")
# write to the appropriate output file
write(filename=args.filename, content="content")
You might need to correct your code's indentation. Python requires indenting code within each function definition, loop, etc.
And as eric points out, there should be no spaces after the periods in args. input and args. output. Change those to args.input and args.output instead.
So:
#!/usr/bin/env python3
import os
import binascii
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.backends import default_backend
import argparse
def readfile_binary(file):
with open(file, 'rb') as f:
content = f.read()
return content
def writefile_binary(file, content):
with open(file, 'wb') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser(description = 'Encryption and Decryption of the file')
parser.add_argument('-in', dest = 'input', required = True)
parser.add_argument('-out', dest = 'output', required = True)
parser.add_argument('-K', dest = 'key', help = 'The key to be used for encryption must be in hex')
parser.add_argument('-iv', dest = 'iv', help = 'The inintialisation vector, must be in hex')
args = parser.parse_args()
input_content = readfile_binary(args.input)
output_content = writefile_binary(args.output)
if __name__ == "__main__":
main()
The XML data(file.xml) for the state will look like below
<?xml version="1.0" encoding="UTF-8" standalone="true"?>
<Activity_Logs xsi:schemaLocation="http://www.cisco.com/PowerKEYDVB/Auditing
DailyActivityLog.xsd" To="2018-04-01" From="2018-04-01" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns="http://www.cisco.com/PowerKEYDVB/Auditing">
<ActivityRecord>
<time>2015-09-16T04:13:20Z</time>
<oper>Create_Product</oper>
<pkgEid>10</pkgEid>
<pkgName>BBCWRL</pkgName>
</ActivityRecord>
<ActivityRecord>
<time>2015-09-16T04:13:20Z</time>
<oper>Create_Product</oper>
<pkgEid>18</pkgEid>
<pkgName>CNNINT</pkgName>
</ActivityRecord>
Parsing and conversion to CSV of above mentioned XML file will be done by the following python code.
import csv
import xml.etree.cElementTree as ET
tree = ET.parse('file.xml')
root = tree.getroot()
data_to_csv= open('output.csv','w')
list_head=[]
Csv_writer=csv.writer(data_to_csv)
count=0
for elements in root.findall('ActivityRecord'):
List_node = []
if count == 0 :
time = elements.find('time').tag
list_head.append(time)
oper = elements.find('oper').tag
list_head.append(oper)
pkgEid = elements.find('pkgEid').tag
list_head.append(pkgEid)
pkgName = elements.find('pkgName').tag
list_head.append(pkgName)
Csv_writer.writerow(list_head)
count = +1
time = elements.find('time').text
List_node.append(time)
oper = elements.find('oper').text
List_node.append(oper)
pkgEid = elements.find('pkgEid').text
List_node.append(pkgEid)
pkgName = elements.find('pkgName').text
List_node.append(pkgName)
Csv_writer.writerow(List_node)
data_to_csv.close()
The code I am using is not giving me any data in CSV. Could some one tell me where excatly am I going wrong?
Using Pandas, parsing all xml fields.
import xml.etree.ElementTree as ET
import pandas as pd
tree = ET.parse("file.xml")
root = tree.getroot()
get_range = lambda col: range(len(col))
l = [{r[i].tag:r[i].text for i in get_range(r)} for r in root]
df = pd.DataFrame.from_dict(l)
df.to_csv('file.csv')
Using pandas and BeautifulSoup you can achieve your expected output easily:
#Code:
import pandas as pd
import itertools
from bs4 import BeautifulSoup as b
with open("file.xml", "r") as f: # opening xml file
content = f.read()
soup = b(content, "lxml")
pkgeid = [ values.text for values in soup.findAll("pkgeid")]
pkgname = [ values.text for values in soup.findAll("pkgname")]
time = [ values.text for values in soup.findAll("time")]
oper = [ values.text for values in soup.findAll("oper")]
# For python-3.x use `zip_longest` method
# For python-2.x use 'izip_longest method
data = [item for item in itertools.zip_longest(time, oper, pkgeid, pkgname)]
df = pd.DataFrame(data=data)
df.to_csv("sample.csv",index=False, header=None)
#output in `sample.csv` file will be as follows:
2015-09-16T04:13:20Z,Create_Product,10,BBCWRL
2015-09-16T04:13:20Z,Create_Product,18,CNNINT
2018-04-01T03:30:28Z,Deactivate_Dhct,,
Use pyxmlparser if it is a one-time operation.
Disclaimer I am the author of the library and it is fairly new. Any feedback is appreciated. It is a command line utility.
https://pypi.org/project/pyxmlparser/
Answer for 2021:
you can use Pandas to read XML and output CSV
https://pandas.pydata.org/pandas-docs/dev/whatsnew/v1.3.0.html#read-and-write-xml-documents
import pandas as pd
df = pd.read_xml(<xml_or_xml_filepath>)
# ...
df.to_csv(<csv_filepath>)
for more details on usage see official documentation:
https://pandas.pydata.org/pandas-docs/dev/reference/api/pandas.read_xml.html
Found the most appropriate way of doing this:
import os
import pandas as pd
from bs4 import BeautifulSoup as b
with open("file.xml", "r") as f: # opening xml file
content = f.read()
soup = b(content, "lxml")
df1 = pd.DataFrame()
for each_file in files_xlm:
with open( each_file, "r") as f: # opening xml file
content = f.read()
soup = b(content, "lxml")
list1 = []
for values in soup.findAll("activityrecord"):
if values.find("time") is None:
time = ""
else:
time = values.find("time").text
if values.find("oper") is None:
oper = ""
else:
oper = values.find("oper").text
if values.find("pkgeid") is None:
pkgeid = ""
else:
pkgeid = values.find("pkgeid").text
if values.find("pkgname") is None:
pkgname = ""
else:
pkgname = values.find("pkgname").text
if values.find("dhct") is None:
dhct = ""
else:
dhct = values.find("dhct").text
if values.find("sourceid") is None:
sourceid = ""
else:
sourceid = values.find("sourceid").text
list1.append(time+','+ oper+','+pkgeid+','+ pkgname+','+dhct+','+sourceid)
df = pd.DataFrame(list1)
df=df[0].str.split(',', expand=True)
df.columns = ['Time','Oper','PkgEid','PkgName','dhct','sourceid']
df.to_csv("new.csv",index=False)