I want to take my Python (currently Version 3.9.7) programming skills to a next level. Up to now I just wrote some small scripts for myself, that no one hat to review or reuse. Now, I want to write code that can be considered as "clean" and can be reused by others. For this purpose, I am writing my own signal processing module with which I can generate high- and lowpassfilters in order to filter signals. I have no experience with structuring packages / modules, so I have some questions regarding code structure.
Up to now, I have a class sim_lowpass:
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 22 10:37:19 2022
#author: ilja
"""
from matplotlib import pyplot as plt
import math
class sim_lowpass:
""" Lowpass Simulation Class """
def __init__(self, cutoff: int, order: int, fs: int) -> None:
self.fs = fs
self.nyq = int(0.5 * fs)
self.cutoff = cutoff
self.order = order
def _transfer_func(self,f: float) -> float:
""" Transfer function in the z-domain """
if self.order == 1:
return 1/(1+(f/(2*math.pi*self.cutoff)))
def transfer_func(self) -> list[float]:
""" Transfer function in the z-domain """
if self.order == 1:
# f = np.linspace(self.df, self.nyq, self.N/2)
f = list(range(int(self.nyq)))
return [self._transfer_func(i) for i in f]
def propagate(self, x_hat):
filtered = [i*j for i,j in zip(x_hat, self.impulse_response())]
return filtered
def bode_plot(self, tr_func: list[float]) -> None:
fig, (ax1, ax2) = plt.subplots(2, 1, constrained_layout=True,
figsize = (8,5))
ax1.plot(list(range(self.nyq)), tr_func)
#ax1.set_title('Magnitude')
ax1.set_xscale('log')
ax1.set_yscale('log')
ax1.set_ylabel('Magnitude (dB)')
ax1.grid(True)
# ax2.plot(list(range(self.nyq)), tr_func) # TODO
# ax2.set_title('Phase')
ax2.set_xscale('log')
ax2.set_yscale('log')
ax2.set_xlabel('Frequency (Hz)')
ax2.set_ylabel('Phase (deg)')
ax2.grid(True)
fig.suptitle('Bode Plot', fontsize=16)
def main() -> None:
# define filter params
cutoff = 100
order = 1
fs = 4e6
# create filter
lp = sim_lowpass(cutoff, order, fs)
tf = lp.transfer_func()
lp.bode_plot(tf)
if __name__ == '__main__':
main()
Questions:
First of all: Is the code up to now well structured (in terms of scalability, testability, ... what else is there?)
Second: Now I want to create the class sim_lowpass. How do I continue without copy-pasting the parts I can reuse from the highpass class?
Third: Where do I place this file (and what would be a meaningful name) inside the package hierarchy?
Last but not least: Any other tips for improvement?
I usually get inspiration for code-structure from real projects. For example, since you are using matplotlib, their github could be a place to start: https://github.com/matplotlib/matplotlib/tree/main/lib/matplotlib
Related
I am using the following code to process some pictures for my ML project and I would like to parallelize it.
import multiprocessing as mp
import concurrent.futures
def track_ids(seq):
'''The func is so big I can not put it here'''
ood = {}
for i in seq:
# I load around 500 images and process them
ood[i] = some Value
return ood
seqs = []
for seq in range(1, 10):# len(seqs)+1):
seq = txt+str(seq)
seqs.append(seq)
# serial call of the function
track_ids(seq)
#parallel call of the function
with concurrent.futures.ProcessPoolExecutor(max_workers=mp.cpu_count()) as ex:
ood_id = ex.map(track_ids, seqs)
if I run the code serially it takes 3.0 minutes but for parallel with concurrent, it takes 3.5 minutes.
can someone please explain why is that? and present a way to solve the problem.
btw, I have 12 cores.
Thanks
Here's a brief example of how one might go about profiling multiprocessing code vs serial execution:
from multiprocessing import Pool
from cProfile import Profile
from pstats import Stats
import concurrent.futures
def track_ids(seq):
'''The func is so big I can not put it here'''
ood = {}
for i in seq:
# I load around 500 images and process them
ood[i] = some Value
return ood
def profile_seq():
p = Profile() #one and only profiler instance
p.enable()
seqs = []
for seq in range(1, 10):# len(seqs)+1):
seq = txt+str(seq)
seqs.append(seq)
# serial call of the function
track_ids(seq)
p.disable()
return Stats(p), seqs
def track_ids_pr(seq):
p = Profile() #profile the child tasks
p.enable()
retval = track_ids(seq)
p.disable()
return (Stats(p, stream="dummy"), retval)
def profile_parallel():
p = Profile() #profile stuff in the main process
p.enable()
with concurrent.futures.ProcessPoolExecutor(max_workers=mp.cpu_count()) as ex:
retvals = ex.map(track_ids_pr, seqs)
p.disable()
s = Stats(p)
out = []
for ret in retvals:
s.add(ret[0])
out.append(ret[1])
return s, out
if __name__ == "__main__":
stat, retval = profile_parallel()
stat.print_stats()
EDIT: Unfortunately I found out that pstat.Stats objects cannot be used normally with multiprocessing.Queue because it is not pickleable (which is needed for the operation of concurrent.futures). Evidently it normally will store a reference to a file for the purpose of writing statistics to that file, and if none is given, it will by default grab a reference to sys.stdout. We don't actually need that reference however until we actually want to print out the statistics, so we can just give it a temporary value to prevent the pickle error, and then restore an appropriate value later. The following example should be copy-paste-able and run just fine rather than the pseudocode-ish example above.
from multiprocessing import Queue, Process
from cProfile import Profile
from pstats import Stats
import sys
def isprime(x):
for d in range(2, int(x**.5)):
if x % d == 0:
return False
return True
def foo(retq):
p = Profile()
p.enable()
primes = []
max_n = 2**20
for n in range(3, max_n):
if isprime(n):
primes.append(n)
p.disable()
retq.put(Stats(p, stream="dummy")) #Dirty hack: set `stream` to something picklable then override later
if __name__ == "__main__":
q = Queue()
p1 = Process(target=foo, args=(q,))
p1.start()
p2 = Process(target=foo, args=(q,))
p2.start()
s1 = q.get()
s1.stream = sys.stdout #restore original file
s2 = q.get()
# s2.stream #if we are just adding this `Stats` object to another the `stream` just gets thrown away anyway.
s1.add(s2) #add up the stats from both child processes.
s1.print_stats() #s1.stream gets used here, but not before. If you provide a file to write to instead of sys.stdout, it will write to that file)
p1.join()
p2.join()
I need to take one row from the CSV file to be used in the reinforcement learning class environment as an observation tuple. I have used generator function first it's not retrieving any data and secondly it will provide all the data iteratively which doesn't match with the requirement of my problem. Also, I need the currently selected observation(CSV row) to be used in multiple methods in the class environment for instance in the reward function.
Any idea or suggestion is highly appreciated on how to do this. Thanks
class Environment1:
def __init__(self, data, max_ticks=300):
self.data = data
self.application_latency=1342
self.reward = 0
#self.done = False
self.MAX_TICKS = max_ticks
self.episode_over = False
def step(self, act):
self.take_action(action)
reward = self.get_reward()
ob = self.get_state()
return ob, reward, self.episode_over
#return ob, reward, self.done # obs, reward, done
def get_state(self):
"""Get the observation. it is a tuple """
lst = [tuple(x) for x in data.values]
def gen(last):
for i in last:
print(yield i)
#observation_space= yield i
#ob = (observation_space.Edge_Latency, observation_space.Cloud_latency )
#print(ob)
#return ob
With what I gathered from your question, you want to create a generator of observation tuples from your csv data. Specifically, you want to pass each tuple with edge latency and cloud latency columns to another function. I have written some example code which will make a list of tuples for each row of your data.
import pandas as pd
import numpy as np
def createGenerator(self):
obs_data = [tuple(x) for x in self.data[['Edge_Latency', 'Cloud_latency']].to_numpy()]
for obs in obs_data:
yield obs
I am trying to implement a parallelized function for Geopandas that takes a single vector data (i.e.: a Shapefile containing a Multipolygon data type), and converts it to a standard celular grid with cell x and y sizes defined by the user.
As this function may result in serious Memory issues (i.e.: caused by too high spatial resolution), I was wondering whether it would be possible to save the data iteratively in the given destinated file. That way, as each parallel process runs the "GRID" function, the same process can save the data iteratively in appended mode. That way, I believe that one wouldn't have Memory issues.
Here is my "SHP_to_GRID_Function". Note that the code below still requires that the whole data generated by the multiprocessing be handled by memory directly (so overflow is more than certain for large datasets).
import pandas as pd
import numpy as np
import geopandas as gpd
from shapely.geometry import Polygon
from multiprocessing import Pool
import os
from functools import partial
def info(title):
print(title)
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
def parallelize_df(gdf, func, n_cores, dx=100, dy=100, verbose=False):
Geometries= gdf.loc[:, 'geometry'].values
pool = Pool(processes=n_cores)
func_partial=partial(func, dx, dy, verbose) # prod_x has only one argument x (y is fixed to 10)
results = pool.map(func_partial, Geometries)
pool.close()
pool.join()
print(np.shape(results))
GRID = gpd.GeoSeries(np.array(results).ravel())
print("GRID well created")
return GRID
def generate_grid_from_Poligon(dx=100, dy=100, verbose=False, polygon=None):
if verbose == True:
info('function parallelize_df')
else:
None
xmin,ymin,xmax,ymax = polygon.bounds
lenght = dx
wide = dy
cols = list(np.arange(int(np.floor(xmin)), int(np.ceil(xmax)), wide))
rows = list(np.arange(int(np.floor(ymin)), int(np.ceil(ymax)), lenght))
rows.reverse()
subpolygons = []
for x in cols:
for y in rows:
subpolygons.append( Polygon([(x,y), (x+wide, y), (x+wide, y-lenght), (x, y-lenght)]) )
return subpolygons
def main(GDF, n_cores='standard', dx=100, dy=100, verbose= False):
"""
GDF: geodataframe
n_cores: use standard or a positive numerical (int) value. It will set the number of cores to use in the multiprocessing
args: (dx: dimension in the x coordinate to make the grid
dy: dimenion in the y coordinate to make the grid)
"""
if isinstance(n_cores, str):
import multiprocessing
N_cores = multiprocessing.cpu_count() -1
elif isinstance(n_cores, int):
N_cores =n_cores
GRID_GDF = parallelize_df(GDF, generate_grid_from_Poligon, n_cores=N_cores, dx=dx, dy=dy, verbose=verbose)
return GRID_GDF
I thank you for you time,
Sincerely yours,
Philipe Leal
I finally have come across a solution for my question. It is not perfect, since it requires several writing processes and one final concatenation process over all temporary files created during the run.
Feel free to suggest alternatives.
Here is the solution I found.
import numpy as np
import geopandas as gpd
import pandas as pd
from shapely.geometry import Polygon
from multiprocessing import Pool, Lock, freeze_support
import os
from functools import partial
import time
def info(time_value):
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
print("Time spent: ", time.time() - time_value)
def init(l):
global lock
lock=l
def Data_Arranger(to_filename):
"""This function concatenates and deletes temporary files. It is an arranger
of the multicessing data results"
"""
Base = os.path.join(os.path.dirname(to_filename), 'temp')
Strings = [file for file in os.listdir(Base)]
Strings = [os.path.join(Base, S) for S in Strings]
if not os.path.exists(os.path.dirname(to_filename)):
os.mkdir(os.path.dirname(to_filename))
Sq = [S for S in Strings if S.endswith('.shp')]
gpd.GeoDataFrame(pd.concat([gpd.read_file(sq1) for sq1 in Sq]), crs=GDF.crs).to_file(to_filename)
for sq1 in Sq:
os.remove(sq1)
import shutil
shutil.rmtree(Base, ignore_errors=True)
def parallelize_df(gdf, func, n_cores, dx=100, dy=100, verbose=False, to_filename=None):
Geometries= gdf.loc[:, 'geometry'].values
crs = gdf.crs
pool = Pool(processes=n_cores, initializer=init, initargs=(Lock(), ) )
func_partial=partial(func, dx, dy, verbose, to_filename, crs) # prod_x has only one argument x (y is fixed to 10)
pool.map(func_partial, Geometries)
pool.close()
pool.join()
def generate_grid_from_gdf(dx=100, dy=100, verbose=False, to_filename=None, crs=None, polygon=None):
if verbose == True:
info(time.time())
else:
None
xmin,ymin,xmax,ymax = polygon.bounds
lenght = dx
wide = dy
cols = list(np.arange(int(np.floor(xmin)), int(np.ceil(xmax)), wide))
rows = list(np.arange(int(np.floor(ymin)), int(np.ceil(ymax)), lenght))
rows.reverse()
subpolygons = []
for x in cols:
for y in rows:
subpolygons.append( Polygon([(x,y), (x+wide, y), (x+wide, y-lenght), (x, y-lenght)]) )
lock.acquire()
print('parent process: ', os.getppid(), ' has activated the Lock')
GDF = gpd.GeoDataFrame(geometry=subpolygons, crs=crs)
to_filename = os.path.join(os.path.dirname(to_filename), 'temp', str(os.getpid()) + '_' + str(time.time()) + '.' + os.path.basename(to_filename).split('.')[-1])
if not os.path.exists(os.path.dirname(to_filename)):
os.mkdir(os.path.dirname(to_filename))
try:
print("to_filename: ", to_filename)
GDF.to_file(to_filename)
except:
print("error in the file saving")
lock.release()
print('parent process: ', os.getppid(), ' has unlocked')
def main(GDF, n_cores='standard', dx=100, dy=100, verbose= False, to_filename=None):
"""
GDF: geodataframe
n_cores: use standard or a positive numerical (int) value. It will set the number of cores to use in the multiprocessing
dx: dimension in the x coordinate to make the grid
dy: dimenion in the y coordinate to make the grid)
verbose: whether or not to show info from the processing. Appliable only if applying the function not
in Windows (LINUX, UBUNTU, etc.), or when running in separte console in Windows.
to_filename: the path which will be used to save the resultant file.
"""
if isinstance(n_cores, str):
import multiprocessing
N_cores = multiprocessing.cpu_count() -1
elif isinstance(n_cores, int):
N_cores =n_cores
parallelize_df(GDF, generate_grid_from_gdf, n_cores=N_cores, dx=dx, dy=dy, verbose=verbose, to_filename=to_filename)
Data_Arranger(to_filename)
####################################################################################
if "__main__" == __name__:
freeze_support()
GDF = gpd.read_file("Someone's_file.shp")
to_filename = "To_file_directory/To_file_name.shp"
dx = 500 # resampling to 500 units. Ex: assuming the coordinate reference system is in meters, this function will return polygons of the given geometries in 500m for the longitudinal dimension.
dy = 500 # same here. Assuming CRS is in meters units, the resultant file will be have polygons of 500m in latitudinal dimension
main(GDF, dx=dx, dy=dy, verbose=True, to_filename=to_filename)
I thank you for your time.
Philipe Leal
I run an evaluation at the end of each epoch and need to show an image calculated from the features and labels arguments of the model function model_fn. Including a tf.summary.image(name, image) in evaluation part of the model function does not help and it looks to me that the only way to do so is to pass the correct eval_metric_ops to construct the EstimatorSpec for mode EVAL. So I first sub-class Estimator so that it considers images. The following code is mostly from estimator.py; the only change is the few lines marked by "my change" inside _write_dict_to_summary:
import logging
import io
import numpy as np
import matplotlib.pyplot as plt
import six
from google.protobuf import message
import tensorflow as tf
from tensorflow.python.training import evaluation
from tensorflow.python import ops
from tensorflow.python.estimator.estimator import _dict_to_str, _write_checkpoint_path_to_summary
from tensorflow.core.framework import summary_pb2
from tensorflow.python.framework import tensor_util
from tensorflow.python.summary.writer import writer_cache
def dump_as_image(a):
vmin = np.min(a)
vmax = np.max(a)
img = np.squeeze((img - vmin) / (vmax - vmin) * 255).astype(np.uint8)
s = io.BytesIO()
plt.imsave(s, img, format='png', vmin=0, vmax=255, cmap='gray')
return s.getvalue()
# see https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/estimator/estimator.py
def _write_dict_to_summary(output_dir, dictionary, current_global_step):
logging.info('Saving dict for global step %d: %s', current_global_step, _dict_to_str(dictionary))
summary_writer = writer_cache.FileWriterCache.get(output_dir)
summary_proto = summary_pb2.Summary()
for key in dictionary:
if dictionary[key] is None:
continue
if key == 'global_step':
continue
if (isinstance(dictionary[key], np.float32) or
isinstance(dictionary[key], float)):
summary_proto.value.add(tag=key, simple_value=float(dictionary[key]))
elif (isinstance(dictionary[key], np.int64) or
isinstance(dictionary[key], np.int32) or
isinstance(dictionary[key], int)):
summary_proto.value.add(tag=key, simple_value=int(dictionary[key]))
elif isinstance(dictionary[key], six.binary_type):
try:
summ = summary_pb2.Summary.FromString(dictionary[key])
for i, img_bytes in enumerate(summ.value):
summ.value[i].tag = '%s/%d' % (key, i)
summary_proto.value.extend(summ.value)
except message.DecodeError:
logging.warn('Skipping summary for %s, cannot parse string to Summary.', key)
continue
elif isinstance(dictionary[key], np.ndarray):
value = summary_proto.value.add()
value.tag = key
value.node_name = key
array = dictionary[key]
# my change begins
if array.ndim == 2:
buffer = dump_as_image(array)
value.image.encoded_image_string = buffer
# my change ends
else:
tensor_proto = tensor_util.make_tensor_proto(array)
value.tensor.CopyFrom(tensor_proto)
logging.info(
'Summary for np.ndarray is not visible in Tensorboard by default. '
'Consider using a Tensorboard plugin for visualization (see '
'https://github.com/tensorflow/tensorboard-plugin-example/blob/master/README.md'
' for more information).')
else:
logging.warn(
'Skipping summary for %s, must be a float, np.float32, np.int64, '
'np.int32 or int or np.ndarray or a serialized string of Summary.',
key)
summary_writer.add_summary(summary_proto, current_global_step)
summary_writer.flush()
class ImageMonitoringEstimator(tf.estimator.Estimator):
def __init__(self, *args, **kwargs):
tf.estimator.Estimator._assert_members_are_not_overridden = lambda self: None
super(ImageMonitoringEstimator, self).__init__(*args, **kwargs)
def _evaluate_run(self, checkpoint_path, scaffold, update_op, eval_dict, all_hooks, output_dir):
eval_results = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
master=self._config.evaluation_master,
scaffold=scaffold,
eval_ops=update_op,
final_ops=eval_dict,
hooks=all_hooks,
config=self._session_config)
current_global_step = eval_results[ops.GraphKeys.GLOBAL_STEP]
_write_dict_to_summary(
output_dir=output_dir,
dictionary=eval_results,
current_global_step=current_global_step)
if checkpoint_path:
_write_checkpoint_path_to_summary(
output_dir=output_dir,
checkpoint_path=checkpoint_path,
current_global_step=current_global_step)
return eval_results
the model function is like --
def model_func(features, labels, mode):
# calculate network_output
if mode == tf.estimator.ModeKeys.TRAIN:
# training
elif mode == tf.estimator.ModeKeys.EVAL:
# make_image consists of slicing and concatenations
images = tf.map_fn(make_image, (features, network_output, labels), dtype=features.dtype)
eval_metric_ops = images, tf.no_op() # not working
return tf.estimator.EstimatorSpec(mode, loss=loss)
eval_metric_ops={'images': eval_metric_ops})
else:
# prediction
And the main part --
# mon_features and mon_labels are np.ndarray
estimator = ImageMonitoringEstimator(model_fn=model_func,...)
mon_input_func = tf.estimator.inputs.numpy_input_fn(mon_features,
mon_labels,
shuffle=False,
num_epochs=num_epochs,
batch_size=len(mon_features))
for _ in range(num_epochs):
estimator.train(...)
estimator.evaluate(input_fn=mon_input_func)
The code above will give a warning (later an error):
WARNING:tensorflow:An OutOfRangeError or StopIteration exception is
raised by the code in FinalOpsHook. This typically means the Ops
running by the FinalOpsHook have a dependency back to some input
source, which should not happen. For example, for metrics in
tf.estimator.Estimator, all metrics functions return two Ops:
value_op and update_op. Estimator.evaluate calls the update_op
for each batch of the data in input source and, once it is exhausted,
it call the value_op to get the metric values. The value_op here
should have dependency back to variables reading only, rather than
reading another batch from input. Otherwise, the value_op, executed
by FinalOpsHook, triggers another data reading, which ends
OutOfRangeError/StopIteration. Please fix that.
Looks like I didn't set the eval_metric_ops correctly. I guess tf.map_fn touches another batch as the warning message hints; maybe I need some stacking operation as the update_op to build the images used for monitoring incrementally? But I am not sure how to do that. So how to add an image to summary during evaluation when using Estimator?
The way I make it work is by passing a tf.train.SummarySaverHook under the evaluation mode and then declaring it to the tf.estimator.EstimatorSpec at evaluation_hooks=.
images is a list of the desired tf.summary.image you want to print during evaluation.
example:
eval_summary_hook = tf.train.SummarySaverHook(output_dir=params['eval_save_path'], summary_op=images, save_secs=120)
spec = tf.estimator.EstimatorSpec(mode=mode, predictions=y_pred, loss=loss, eval_metric_ops=eval_metric_ops,
evaluation_hooks=[eval_summary_hook])
I am trying to get the associated user data of a point (which is a SpotItem instance) in a scatter plot when clicked on it. While methods listed in the documentation (like pos() or size()) seem to work fine, I recieve a NoneType object when I apply the data() method. I actually expected it to return my user data, but it doesn't.
So, how can I retrieve my associated original data that I put in?
What I actually need is something like an index i of the original input lists for a clicked point that would allow me to track back the corresponding x[i] y[i] set.
Here is my code sample:
import pyqtgraph as pg
#some dummy data
x=[0,1,2,3,4,5,3.5,3.4]
y=[5,4,3,2,1,0,3.4,3.5]
win=pg.GraphicsWindow()
p1=win.addPlot(row=1, col=1)
my_data=pg.ScatterPlotItem(x,y,symbol='o',size=30)
p1.addItem(my_data)
def clicked(items,points):
print("point data: ",points[0].data())
my_data.sigClicked.connect(clicked)
I am using Python 3.6 (with Spyder 3.1.4), Qt 5.6 and PyQt 5
sigClicked gave us the item (ScatterPlotItem) that has been pressed and the points (SpotItem) where they have been pressed, with the seconds we can obtain the element Point() that gives us the position, and this has the methods x() y y() that return the coordinates. From item we can get all the x and y that you have initially placed through data['x'] and data['y'], respectively, then we have the points pressed and all the points possible so to find the indexes we use np.argwhere() and then we intersect the values with np.intersect1d(), at the end we eliminate the repeated points with set.
import numpy as np
from pyqtgraph.Qt import QtGui, QtCore
import pyqtgraph as pg
app = QtGui.QApplication([])
x=[0,1,2,3,4,5,3.5,3.4, 3.4]
y=[5,4,3,2,1,0,3.4,3.5, 3.5]
win=pg.GraphicsWindow()
p1=win.addPlot(row=1, col=1)
my_data=pg.ScatterPlotItem(x,y,symbol='o',size=30)
p1.addItem(my_data)
def clicked(item, points):
indexes = []
for p in points:
p = p.pos()
x, y = p.x(), p.y()
lx = np.argwhere(item.data['x'] == x)
ly = np.argwhere(item.data['y'] == y)
i = np.intersect1d(lx, ly).tolist()
indexes += i
indexes = list(set(indexes))
print(indexes)
my_data.sigClicked.connect(clicked)
if __name__ == '__main__':
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()
like this:
def clicked(items,obj, points):
now_point = point[0].pos()
x = now_point[0]
y = now_point[1]
Then you can get what you want.