How to write to text file in python? - python-3.x

I am a beginner in using python. I have created a plain text file and have to encrypt it to output file. But I am getting an error as below and unable to write it to output file. The code is running but the output file which should be encrypted is created.
#!/usr/bin/env python3
import os
import binascii
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.backends import default_backend
import argparse
def readfile_binary(file):
with open(file, 'rb') as f:
content = f.read()
return content
def writefile_binary(file, content):
with open(file, 'wb') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser(description = 'Encryption and Decryption of the file')
parser.add_argument('-in', dest = 'input', required = True)
parser.add_argument('-out', dest = 'output', required = True)
parser.add_argument('-K', dest = 'key', help = 'The key to be used for encryption must be in hex')
parser.add_argument('-iv', dest = 'iv', help = 'The inintialisation vector, must be in hex')
args = parser.parse_args()
input_content = readfile_binary(args. input)
output_content = writefile_binary(args. output)
if __name__ == "__main__":
main()
The output file should be encrypted and it should be available in the directory.

These two lines:
input_content = readfile_binary(args. input)
output_content = writefile_binary(args. output)
There should not be a space in args.input. Here is an example,
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('filename')
args = parser.parse_args()
# using type hints can help reasoning about code
def write(filename: str, content: str) -> None:
with open(filename, 'wb') as f:
f.write(str.encode(content))
# if the filename was successfully parsed from stdin
if args.filename == 'filename.txt':
print(f"args: {args.filename}")
# write to the appropriate output file
write(filename=args.filename, content="content")

You might need to correct your code's indentation. Python requires indenting code within each function definition, loop, etc.
And as eric points out, there should be no spaces after the periods in args. input and args. output. Change those to args.input and args.output instead.
So:
#!/usr/bin/env python3
import os
import binascii
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives import padding
from cryptography.hazmat.backends import default_backend
import argparse
def readfile_binary(file):
with open(file, 'rb') as f:
content = f.read()
return content
def writefile_binary(file, content):
with open(file, 'wb') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser(description = 'Encryption and Decryption of the file')
parser.add_argument('-in', dest = 'input', required = True)
parser.add_argument('-out', dest = 'output', required = True)
parser.add_argument('-K', dest = 'key', help = 'The key to be used for encryption must be in hex')
parser.add_argument('-iv', dest = 'iv', help = 'The inintialisation vector, must be in hex')
args = parser.parse_args()
input_content = readfile_binary(args.input)
output_content = writefile_binary(args.output)
if __name__ == "__main__":
main()

Related

How to manage memory consumption in this code using Pool and Itertools

I have a task to make an iteration from "aaa, aab, aac...aa1...999" using multiprocessing. Though it works, if I make this more than 5 symbols long - it will crush. Please help to make it lighter!
This iteration eventually gets written into a txt file
import logging
import multiprocessing
import pathlib
import uuid
from itertools import product
from logging import Logger
from multiprocessing import Pool
from string import ascii_lowercase, digits
from typing import Iterable
STORAGE = pathlib.Path(__file__).parent.joinpath("words_storage")
def init_logger() -> Logger:
logger = logging.getLogger()
logging.basicConfig(level=logging.INFO)
return logger
class Combinator:
def __init__(
self,
alphabet: Iterable,
str_len: int,
purge_after: bool = True,
logger: Logger = init_logger(),
):
self.alphabet = alphabet
self.str_len = abs(str_len)
self.purge_after = purge_after
self.logger = logger
def product(self):
self.logger.info("[START] Combining started [START]")
self.pre_clean()
try:
with Pool(processes=multiprocessing.cpu_count() - 1) as pool:
result = pool.map(self._get_for_char, self.alphabet)
self.logger.info("[PROCESS] List of filenames received [PROCESS]")
for name in result:
self.logger.info(
f"[PROCESS] Extracting combinations from {name} [PROCESS]"
)
with open(
STORAGE.joinpath("result/result.txt"), mode="a"
) as file_result:
with open(name) as producer:
file_result.write(producer.read())
finally:
if self.purge_after:
self.logger.info(
"[CLEANING] Purging created .txt files in storage [CLEANING]"
)
Combinator.purge_files()
self.logger.info(
"[END] All combinations created successfully. See result.txt file [END]"
)
def combinations_generator(self, char):
for starter_combination in product(self.alphabet, repeat=self.str_len - 1):
combination = [char]
combination.extend(starter_combination)
yield combination
def _get_for_char(self, char) -> pathlib.Path:
self.logger.info(f"[PROCESS] Processing char: {char} [PROCESS]")
file_name = STORAGE.joinpath(f"{uuid.uuid4()}.txt")
result = self.combinations_generator(char)
with open(file_name, mode="w") as file:
file.write(
Combinator.writer("".join(combination) for combination in result)
)
return file_name
def pre_clean(self):
try:
STORAGE.joinpath("result/result.txt").unlink()
self.logger.info("[PRE-CLEAN] Result directory cleared [PRE-CLEAN]")
except FileNotFoundError:
self.logger.info("[PRE-CLEAN] Result directory already clear [PRE-CLEAN]")
#staticmethod
def writer(result_list: Iterable[str]) -> str:
return "\n".join(result_list)
#staticmethod
def purge_files():
for file in STORAGE.glob("*.txt"):
STORAGE.joinpath(file).unlink()
if __name__ == "__main__":
repeats = 5
alphabet_for_combinations = ascii_lowercase + digits
combinator = Combinator(alphabet_for_combinations, repeats, purge_after=True)
combinator.product()
I tried regulating the number of processes and and the maxtasksperchild number, but it didn't help.

Extract a part of a string and append it to a tag

I want to find a string that starts with "section_" and add this as a value to a tag in the same line.
Example: Following is the input in a file of type ditamap.
<topicref href="xyz/debug_logging_in_xyz-section_i_y_mn.dita"/>
<topicref href="xyz/workflows_id-section_exf_zaz_lo.dita"/>
<topicref href="xyz/images_id-section_ekl_bbz_lo.dita"/>
Desired output:
<topicref href="xyz/debug_logging_in_xyz-section_i_y_mn.dita" keys="section_i_y_mn"/>
<topicref href="xyz/workflows_id-section_exf_zaz_lo.dita" keys="section_exf_zaz_lo"/>
<topicref href="xyz/images_id-section_ekl_bbz_lo.dita" keys="section_ekl_bbz_lo"/>
I understand BeautifulSoup can be used to achieve this. But, I am new and do not know the syntax. Can anyone help?
Here is the code I am trying to use:
import os
from bs4 import BeautifulSoup as bs
globpath = "C:/DATA" #add your directory path here
def main(path):
with open(path, encoding="utf-8") as f:
s = f.read()
s = bs(s, "xml")
imgs = s.find_all("topicref")
for i in imgs:
if "section" in i["href"]:
i["keys"] = i["href"].replace("*-","").replace(".dita*","")
s = str(s)
with open(path, "w", encoding="utf-8") as f:
f.write(s)
for dirpath, directories, files in os.walk(globpath):
for fname in files:
if fname.endswith(".ditamap"):
path = os.path.join(dirpath, fname)
main(path)
But, it's adding the entire path in the keys attribute. I need only the portion that starts with section and ends before .dita.
Regex worked:Here is the final code
from bs4 import BeautifulSoup as bs
import re
globpath = "C:/DATA" #add your directory path here
def main(path):
with open(path, encoding="utf-8") as f:
s = f.read()
s = bs(s, "xml")
imgs = s.find_all("topicref")
for i in imgs:
if "section" in i["href"]:
try:
i["keys"] = re.findall("section[^\.]*",i["href"])[0]
except:
print("Could not replace")
s = str(s)
with open(path, "w", encoding="utf-8") as f:
f.write(s)```
I think it should be done with Regex (cuz thats the most i can do)
from bs4 import BeautifulSoup
import re
soup = BeautifulSoup('your-string-input-of-tags-goes-here', 'html.parser')
soup.find_all('topicref', {'keys': re.compile(r'(section_([^ "])+)')})
Returns a list of matched tags
Check this code whether it works or not

In pyhocon is it possible to include a file in runtime

In pyhocon can is it possible to include a file in runtime?
I know how to merge trees in runtime e.g.
conf = ConfigFactory.parse_file(conf_file)
conf1 = ConfigFactory.parse_file(include_conf)
conf = ConfigTree.merge_configs(conf, conf1)
I would like to emulate the include statement so that hocon variables get evaluated in runtime like in the wishful code below:
conf = ConfigFactory.parse_files([conf_file, include_conf])
Newer pyhocon has a possibility to parse a config file without resolving variables.
import pprint
from pyhocon import ConfigFactory
if __name__ == "__main__":
conf = ConfigFactory\
.parse_file('./app.conf', resolve=False)\
.with_fallback(
config='./app_vars.conf',
resolve=True,
)
pprint.pprint(conf)
app.conf:
smart: {
super_hero: ${super_hero}
}
app_vars.conf:
super_hero: Buggs_Bunny
It's possible to use strings:
from pyhocon import ConfigFactory
# readingconf entrypoint
file = open('application.conf', mode='r')
conf_string = file.read()
file.close()
conf_string_ready = conf_string.replace("PATH_TO_CONFIG","spec.conf")
conf = ConfigFactory.parse_string(conf_string_ready)
print(conf)
application.conf has
include "file://INCLUDE_FILE"
or in runtime without preparation I wrote it myself using python 3.65:
#!/usr/bin/env python
import os
from pyhocon import ConfigFactory as Cf
def is_line_in_file(full_path, line):
with open(full_path) as f:
content = f.readlines()
for l in content:
if line in l:
f.close()
return True
return False
def line_prepender(filename, line, once=True):
with open(filename, 'r+') as f:
content = f.read()
f.seek(0, 0)
if is_line_in_file(filename, line) and once:
return
f.write(line.rstrip('\r\n') + '\n' + content)
def include_configs(base_path, included_paths):
for included_path in included_paths:
line_prepender(filename=base_path, line=f'include file("{included_path}")')
return Cf.parse_file(base_path)
if __name__ == "__main__":
dir_path = os.path.dirname(os.path.realpath(__file__))
print(f"current working dir: {dir_path}")
dir_path = ''
_base_path = f'{dir_path}example.conf'
_included_files = [f'{dir_path}example1.conf', f'{dir_path}example2.conf']
_conf = include_configs(base_path=_base_path, included_paths=_included_files)
print('break point')

Unable to read data about FCN,i don't know how to do.the result is:Found pickle file! 0 0

import numpy as np
import os
import random
from six.moves import cPickle as pickle
from tensorflow.python.platform import gfile
import glob
import TensorflowUtils as utils
DATA_URL = 'http:\\data.csail.mit.edu\\places\\ADEchallenge\\ADEChallengeData2016.zip'
#download and read dataset
def read_dataset(data_dir):
pickle_filename = "MITSceneParsing.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
SceneParsing_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
result = create_image_lists(os.path.join(data_dir, SceneParsing_folder))
print ("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print ("Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['training']
validation_records = result['validation']
del result
return training_records, validation_records
train_records, valid_records = read_dataset('Data_zoo/MIT_SceneParsing')
print(len(train_records))
print(len(valid_records))
the result is:Found pickle file! 0 0
why the lens about train_records and valid_records are 0?
i don't know whree is wrong and how to correct it.
This code is right. The bug is in 'create_image_lists'.
Note this code in create_image_lists:
filename = os.path.splitext(f.split('/')[-1])[0]
This is no problem in Linux, but in windows, the separator is '\\', so you should modify this code to:
filename = os.path.splitext(f.split('\\')[-1])[0]
Then delete this file 'MITSceneParsing.pickle', and run read_dataset again.

segmentation fault in python3

I am running python3 on a Ubuntu machine and have noticed that the following block of code is fickle. Sometimes it runs just fine, other times it produces a segmentation fault. I don't understand why. Can someone explain what might be going on?
Basically what the code does is try to read S&P companies from Wikipedia and write the list of tickers to a file in the same directory as the script. If no connection to Wikipedia can be established, the script tries instead to read an existing list from file.
from urllib import request
from urllib.error import URLError
from bs4 import BeautifulSoup
import os
import pickle
import dateutil.relativedelta as dr
import sys
sys.setrecursionlimit(100000)
def get_standard_and_poors_500_constituents():
fname = (
os.path.abspath(os.path.dirname(__file__)) + "/sp500_constituents.pkl"
)
try:
# URL request, URL opener, read content.
req = request.Request(
"http://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
)
opener = request.urlopen(req)
# Convert bytes to UTF-8.
content = opener.read().decode()
soup = BeautifulSoup(content, "lxml")
# HTML table we actually need is the first.
tables = soup.find_all("table")
external_class = tables[0].findAll("a", {"class":"external text"})
c = [ext.string for ext in external_class if not "reports" in ext]
with open(fname, "wb") as f:
pickle.dump(c, f)
except URLError:
with open(fname, "rb") as f:
c = pickle.load(f)
finally:
return c
sp500_constituents = get_standard_and_poors_500_constituents()
spdr_etf = "SPY"
sp500_index = "^GSPC"
def main():
X = get_standard_and_poors_500_constituents()
print(X)
if __name__ == "__main__":
main()

Resources