Chose which function to run in script - python-3.x

I'm trying to make it so that the user chooses which function to run using if.
import os
import csv
import collections
import datetime
import pandas as pd
import time
import string
import re
import glob, os
folder_path = 'C:/ProgramData/WebPort/system/tags'
folder2_path = 'C:/ProgramData/WebPort/system'
search2_str = '"Prefix"'
print("Choices:\n 1 - Read from CSV\n 2 - Read from WPP")
x = input("Please enter your choice:\n")
x = int(x)
if x == 1:
csv_file_list = glob.glob(folder_path + '/*.csv')
with open("csv.txt", 'w') as wf:
for file in csv_file_list:
print(glob.glob(folder_path + '/*.csv'))
with open(file) as rf:
for line in rf:
if line.strip(): # if line is not empty
if not line.endswith("\n"):
line+="\n"
wf.write(line)
print('Reading from .csv')
elif x == 2:
for root, dirs, files in os.walk(folder2_path):
for file in files:
if file.endswith(".wpp"):
print(os.path.join(root, file))
with open(os.path.join(root, file), 'r') as fr, open ("wpp.txt",'a', encoding='utf-8') as fw:
for i,line in enumerate(fr):
if line.find(search2_str) != -1:
fw.write(line)
print('Reading from .wpp')
else:
print('wrong choice')
Getting Invalid syntax in line 34 using this.

Related

Dynamically create file folder based on line 1 in csv

This script downloads images, renames them based on line[0] adds a number to the end of the file name and saves them to a file folder. My goal here is to create the file folder name based on line[0] of my csv file, I'm new to python and need to download/sort 15000+ images. Any help would be appreciated! Using python 3.8.6
Note: 1 model may contain many images so the idea is to create the folder, place images for that model inside, move on to the next model, etc.
Csv file content
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dw0c85e188/product-mugs/cooking/ranges/mug/retail/RHV3-484-N-RHV3-484-L-external-mug-rs-84.png
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dwcbd711e5/inspire/caitlin-wilson-portland-dk-339-rs-84.png
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dw3702e52a/inspire/caitlin-wilson-portland-dk-385-rs-84.jpg
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dw0c85e188/product-mugs/cooking/ranges/mug/retail/RHV3-484-N-RHV3-484-L-external-mug-rs-84.png
RHV3-484-L,https://www.fisherpaykel.com/on/demandware.static/-/Sites-fpa-master-catalog/default/dwf99a5a9d/inspire/david-berridge-project-brooklyn-mw-6457-rs-84.jpg
Python script
import sys
import urllib
import urllib.request
from csv import reader
import os.path
import os
csv_filename = "images"
with open(csv_filename+".csv".format(csv_filename), 'r') as csv_file:
n = 1
for line in reader(csv_file):
if not os.path.exists("ImageID"):
os.makedirs("ImageID")
print("Image skipped for {0}".format(line[0]))
else:
if line[1] != '' and line[0] != "ImageID":
urllib.request.urlretrieve(line[1], "ImageID/" + line[0] + "-" + str(n) + ".jpg")
n += 1
print("Image saved for {0}".format(line[0]))
else:
print("No result for {0}".format(line[0]))
This seems to work as desired....
Couple comments in middle. Notably, you need to respect the .jpg or .png file. If you have file extensions that are longer (4 chars) you may need to split out the file name and then split by "."
Good Luck!
import sys
import urllib
import urllib.request
from csv import reader
import os.path
import os
csv_filename = "images.csv"
with open(csv_filename, 'r') as csv_file:
n = 1 # starting point
for line in reader(csv_file):
tgt_folder = line[0]
if not os.path.exists(tgt_folder):
os.makedirs(tgt_folder)
n = 1 # restart n if you find a NEW folder
# there should be no "else" clause here. Just test the folder name above, but don't waste a file
if line[1] != '' and line[0] != "ImageID": # not clear what this is for... ??
filename = ''.join([line[0], '-', str(n), line[1][-4:]])
destination = os.path.join(tgt_folder, filename)
urllib.request.urlretrieve(line[1], destination)
n += 1
print("Image saved for {0}".format(line[1]))
else:
print("No result for {0}".format(line[1]))

Remove all lines startng with < for all files in current folder

This is what I have so far. But it just hangs.
import os
import sys
import re
directory_path=os.path.dirname(os.path.abspath(__file__))
for root, dirs, files in os.walk(directory_path):
for file in files:
s = ''.join(sys.stdin.readlines())
s = re.sub(r'(?m)^\<.*\n?', '', s)
take 2, this still doesnt work but maybe a bit closer?
import fileinput
import glob
import sys
import os
path = os.path.dirname(os.path.abspath(__file__))
search = "<"
replace = " "
for line in fileinput.input(glob.glob(path), inplace=1):
sys.stdout.write(line.replace(search, replace))
You can use python inbuilt functions readlines, filter & writelines to achieve your requirement.
I believe the below code serves your purpose.
import os
source_dir = 'C:/Users/rlal1/Desktop/test_folder'
files = os.listdir(source_dir)
for each_file in files:
with open(os.path.join(source_dir, each_file), 'r+') as fp:
all_lines = fp.readlines()
filtered = filter(lambda x: not x.startswith('<'), all_lines)
fp.seek(0)
fp.truncate(0)
fp.writelines(filtered)

In pyhocon is it possible to include a file in runtime

In pyhocon can is it possible to include a file in runtime?
I know how to merge trees in runtime e.g.
conf = ConfigFactory.parse_file(conf_file)
conf1 = ConfigFactory.parse_file(include_conf)
conf = ConfigTree.merge_configs(conf, conf1)
I would like to emulate the include statement so that hocon variables get evaluated in runtime like in the wishful code below:
conf = ConfigFactory.parse_files([conf_file, include_conf])
Newer pyhocon has a possibility to parse a config file without resolving variables.
import pprint
from pyhocon import ConfigFactory
if __name__ == "__main__":
conf = ConfigFactory\
.parse_file('./app.conf', resolve=False)\
.with_fallback(
config='./app_vars.conf',
resolve=True,
)
pprint.pprint(conf)
app.conf:
smart: {
super_hero: ${super_hero}
}
app_vars.conf:
super_hero: Buggs_Bunny
It's possible to use strings:
from pyhocon import ConfigFactory
# readingconf entrypoint
file = open('application.conf', mode='r')
conf_string = file.read()
file.close()
conf_string_ready = conf_string.replace("PATH_TO_CONFIG","spec.conf")
conf = ConfigFactory.parse_string(conf_string_ready)
print(conf)
application.conf has
include "file://INCLUDE_FILE"
or in runtime without preparation I wrote it myself using python 3.65:
#!/usr/bin/env python
import os
from pyhocon import ConfigFactory as Cf
def is_line_in_file(full_path, line):
with open(full_path) as f:
content = f.readlines()
for l in content:
if line in l:
f.close()
return True
return False
def line_prepender(filename, line, once=True):
with open(filename, 'r+') as f:
content = f.read()
f.seek(0, 0)
if is_line_in_file(filename, line) and once:
return
f.write(line.rstrip('\r\n') + '\n' + content)
def include_configs(base_path, included_paths):
for included_path in included_paths:
line_prepender(filename=base_path, line=f'include file("{included_path}")')
return Cf.parse_file(base_path)
if __name__ == "__main__":
dir_path = os.path.dirname(os.path.realpath(__file__))
print(f"current working dir: {dir_path}")
dir_path = ''
_base_path = f'{dir_path}example.conf'
_included_files = [f'{dir_path}example1.conf', f'{dir_path}example2.conf']
_conf = include_configs(base_path=_base_path, included_paths=_included_files)
print('break point')

Read multiple files multiprocessing

I have a simple function that scans files for a special string, but as these files are on a slow remote file storage, I need to scan them parallel.
I guess I need to use multiprocessing, but I am not sure how to do that correctly.
Here is my function:
from fnmatch import fnmatch
import os
from shutil import copy
from pprint import pprint
def getFailedFile(directory_name, folder_to_write):
for file in os.listdir(directory_name):
if fnmatch(file, '*Response.txt'):
filename = directory_name + file
try:
with open(filename, 'r', encoding='utf-8') as myfile:
data = myfile.read()
if data.find('Exception') != -1:
try:
requestFile = directory_name + file.replace('Response', 'Request')
copy(requestFile, os.getcwd() + folder_to_write)
except FileNotFoundError:
print('no such file - ', requestFile)
except UnicodeDecodeError:
print('error unicode decode -', filename)
directory_name = 'some folder'
folder_to_write = 'some folder_to_write'
getFailedFile(directory_name=directory_name, folder_to_write)
Please help. Currently it takes about 4 hours due to number of files in the destination folder.
Finally figured out how to do that:
from fnmatch import fnmatch
import os
from shutil import copy
from multiprocessing import Pool
import time
import logging
def process_file(file):
directory_name = 'directory with files'
if fnmatch(file, '*Response.txt'):
filename = directory_name + file
try:
with open(filename, 'r', encoding='utf-8') as myfile:
data = myfile.read()
if data.find('xception') != -1:
try:
requestFile = directory_name + file.replace('Response', 'Request')
responseFile = directory_name + file
try:
copy(requestFile, 'directory to write')
copy(responseFile, 'directory to write')
except Exception as e:
logging.info(str(e) + '\n')
print(str(e))
except FileNotFoundError:
print('no such file - ', requestFile)
logging.info('no such file - ' + str(requestFile) + '\n')
except UnicodeDecodeError:
print('error unicode decode -', filename)
logging.info('error unicode decode -' + str(filename) + '\n')
if __name__ == '__main__':
try:
directory_name = 'directory with files'
number_of_processes = 50
logging.info('\n' + 'Number of processes - ' + str(number_of_processes))
logging.info('Directory to scan ' + directory_name)
pool = Pool(number_of_processes)
start_time = time.time()
pool.map(process_file, os.listdir(directory_name))
pool.close()
elapsed_time = time.time() - start_time
logging.info('Elapsed time - ' + str(elapsed_time / 60) + '\n')
except Exception as e:
logging.info(str(e) + '\n')
I know that the code is not so pretty, but it works 27 minutes instead of previous elapsed time.

Unable to read data about FCN,i don't know how to do.the result is:Found pickle file! 0 0

import numpy as np
import os
import random
from six.moves import cPickle as pickle
from tensorflow.python.platform import gfile
import glob
import TensorflowUtils as utils
DATA_URL = 'http:\\data.csail.mit.edu\\places\\ADEchallenge\\ADEChallengeData2016.zip'
#download and read dataset
def read_dataset(data_dir):
pickle_filename = "MITSceneParsing.pickle"
pickle_filepath = os.path.join(data_dir, pickle_filename)
if not os.path.exists(pickle_filepath):
utils.maybe_download_and_extract(data_dir, DATA_URL, is_zipfile=True)
SceneParsing_folder = os.path.splitext(DATA_URL.split("/")[-1])[0]
result = create_image_lists(os.path.join(data_dir, SceneParsing_folder))
print ("Pickling ...")
with open(pickle_filepath, 'wb') as f:
pickle.dump(result, f, pickle.HIGHEST_PROTOCOL)
else:
print ("Found pickle file!")
with open(pickle_filepath, 'rb') as f:
result = pickle.load(f)
training_records = result['training']
validation_records = result['validation']
del result
return training_records, validation_records
train_records, valid_records = read_dataset('Data_zoo/MIT_SceneParsing')
print(len(train_records))
print(len(valid_records))
the result is:Found pickle file! 0 0
why the lens about train_records and valid_records are 0?
i don't know whree is wrong and how to correct it.
This code is right. The bug is in 'create_image_lists'.
Note this code in create_image_lists:
filename = os.path.splitext(f.split('/')[-1])[0]
This is no problem in Linux, but in windows, the separator is '\\', so you should modify this code to:
filename = os.path.splitext(f.split('\\')[-1])[0]
Then delete this file 'MITSceneParsing.pickle', and run read_dataset again.

Resources