Related
I am training a neural networks with three different output prediction. For computing the loss of one output I need one of the input that is passed into the network. I am not able to access it as the training data is feed into the network by a keras data generator object. Is there any workaround for this problem.
This is the Generator class that feds data into the model
class DataGenerator(tf.keras.utils.Sequence):
def __init__(self,list_ID,centers,sizes,batch_size=2,dims=(512,512),n_channels=3,n_classes=10,shuffle=True) -> None:
assert len(list_ID) == len(centers)
self.dims = dims
self.batch_size = batch_size
self.list_ID = list_ID
self.centers = centers
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.sizes = sizes
self.on_epoch_end()
self.mask = None
def __len__(self):
return int(np.floor(len(self.list_ID) / self.batch_size))
def on_epoch_end(self):
self.indexes = np.arange(len(self.list_ID))
if self.shuffle:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_ID_temp = [self.list_ID[k] for k in indexes]
centers_temp = [self.centers[k] for k in indexes]
sizes_temp = [self.sizes[k] for k in indexes]
X, y = self.__datageneration(list_ID_temp, centers_temp,sizes_temp)
return X, y
def __datageneration(self, list_ID_temp,centers_temp,sizes_temp):
X = np.empty((self.batch_size,*self.dims,self.n_channels))
Y_center = np.empty((self.batch_size,128,128,1))
Y_dimension = np.empty((self.batch_size,128,128,2))
Y_offset = np.empty((self.batch_size,128,128,2))
self.mask = np.empty((self.batch_size,128,128,1))
for i,ID in enumerate(list_ID_temp):
image = cv2.imread(path+'/'+ID) / 255.0
heat_center, self.mask[i,] = gaussian_2d(centers_temp[i],image.shape)
'''Here I tried to save mask which is what I need,
as an attribute to data generator but when accessed by loss function
the value is just None which is what I initialized it as in init method'''
heat_size,heat_off = size_off_heatmap(sizes_temp[i], centers_temp[i],image.shape)
image = cv2.resize(image,(512,512))
X[i,] = image
Y_center[i,] = heat_center
Y_dimension[i,] = heat_size
Y_offset[i,] = heat_off
return (X,{'center_output':Y_center,'size_output':Y_dimension,'offset_output':Y_offset})
This is the generator class I implemented and I needed the mask , which I tried to write as an attribute of data generator object(I have commented the code. For reference I will also include the function that will return the mask and the error function that requires the mask.
Function returning mask
def gaussian_2d(centers, img_shape):
heatmap = []
y_index = np.tile(np.arange(128), (128, 1))
mask = np.zeros((128,128,1))
width = img_shape[1]
height = img_shape[0]
for x_o, y_o in centers:
x = int(x_o / width * 128)
y = int(y_o / height * 128)
mask[y,x] = 1
gauss = np.exp(-((y_index.T - y) ** 2 + (y_index - x) ** 2) / 2 * 0.2 ** 2)
heatmap.append(gauss)
if len(heatmap) > 1:
heatmap = np.stack(heatmap)
heatmap = np.max(heatmap, axis=0)
else:
heatmap = np.array(heatmap)
heatmap = heatmap.reshape((128, 128,1))
return heatmap,mask
Loss function
def final_loss(mask):
def l1_loss(y_true, y_pred):
y_true = tf.cast(y_true, tf.float32)
y_pred = tf.cast(y_pred, tf.float32)
n = tf.reduce_sum(tf.cast(tf.equal(mask, 1.0),dtype=tf.float32))
tot_loss = tf.reduce_sum(tf.abs(y_pred - y_true))
if tf.greater(n,0):
loss = tot_loss / (n)
else:
loss = tot_loss
return loss
return l1_loss
The error show is as below
Epoch 1/10
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-27-74a28b075f52> in <module>()
----> 1 model.fit(gen,epochs=10,verbose=1,callbacks=Callback(patience=4))
9 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
975 except Exception as e: # pylint:disable=broad-except
976 if hasattr(e, "ag_error_metadata"):
--> 977 raise e.ag_error_metadata.to_exception(e)
978 else:
979 raise
ValueError: in user code:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
<ipython-input-24-c45fe131feb7>:5 l1_loss *
n = tf.reduce_sum(tf.cast(tf.equal(mask, 1.0),dtype=tf.float32))
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py:201 wrapper **
return target(*args, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:1679 equal
return gen_math_ops.equal(x, y, name=name)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_math_ops.py:3179 equal
name=name)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:540 _apply_op_helper
(input_name, err))
ValueError: Tried to convert 'x' to a tensor and failed. Error: None values not supported.
'''
I am trying to learn how to group functions by class. As an example, I tried to code a generalized least squares method to find the equation of a best-fitting line between a set of (x,y) coordinates. For my particular case, I chose a simple line y = x + 5, so slope should be close to 1 and y-intercept should be close to 5. Running my attempt at a coded solution below produces the error TypeError: set_x() takes 1 positional argument but 2 were given, though I am trying to pass an array of x-points. How can I circumvent this error?
import numpy as np
from scipy.optimize import minimize
class GeneralizedLeastSquares:
def __init__(self, residuals=None, parameters=None, x=None, y_true=None, y_fit=None, weights=None, method=None):
self.residuals = residuals
self.parameters = parameters
self.x = x
self.y_true = y_true
self.y_fit = y_fit
self.weights = weights
self.method = method
def set_residuals(self, residuals):
self.residuals = residuals
def set_parameters(self, parameters):
self.parameters = parameters
def set_x(self, x):
self.x = x
def set_y_true(self, y_true):
self.y_true = y_true
def set_y_fit(self, y_fit):
self.y_fit = y_fit
def set_weights(self, weights):
self.weights = weights
def set_method(self, method):
self.method = method
def get_residuals(self):
return [(self.y_true[idx] - self.y_fit[idx])**2 for idx in range(len(self.y_true)) if len(self.y_true) == len(self.y_fit) ]
def get_parameters(self):
return self.parameters
def get_x(self):
return self.x
def get_y_true(self):
return self.y_true
def get_y_fit(self):
return [self.parameters[0] * self.x[idx] + self.parameters[1] for idx in range(len(self.x))]
def get_weights(self):
return self.weights
def update_weights(self):
inverse_residuals = [1/self.residuals[idx] for idx in range(len(residuals))]
inverse_residuals_abs = [abs(inverse_residual) for inverse_residual in inverse_residuals]
residual_abs_total = sum(inverse_residuals_abs)
return [inverse_residuals_abs[idx]/residual_abs_total for idx in range(len(inverse_residuals_abs))]
def get_method(self):
return self.method
def get_error_by_residuals(self):
return sum([self.weights[idx] * self.residuals[idx] for idx in range(len(self.residuals))])
def get_error_by_std_mean(self):
return np.std(self.y_true)/np.sqrt(len(self.y_true))
def get_linear_fit(self):
"""
"""
if self.parameters == 'estimate':
slope_init = (self.y_true[-1] - self.y_true[0]) / (self.x[-1] - self.x[0])
b_init = np.mean([self.y_true[-1] - slope_init * self.x[-1], self.y_true[0] - slope_init * self.x[0]])
self.parameters = [slope_init, b_init]
elif not isinstance(self.parameters, (list, np.ndarray)):
raise ValueError("parameters = 'estimate' or [slope, y-intercept]")
meths = ['residuals', 'std of mean']
funcs = [get_error_by_residuals, get_error_by_std_mean]
func = dict(zip(meths, funcs))[self.method]
res = minimize(func, x0=self.parameters, args=(self,), method='Nelder-Mead')
self.parameters = [res.x[0], res.x[1]]
self.y_fit = get_y_fit(self)
self.residuals = get_residuals(self)
self.weights = update_weights(self)
return self.parameters, self.y_fit, self.residuals, self.weights
x = np.linspace(0, 4, 5)
y_true = np.linspace(5, 9, 5) ## using slope=1, y-intercept=5
y_actual = np.array([4.8, 6.2, 7, 8.1, 8.9]) ## test data
GLS = GeneralizedLeastSquares()
GLS.set_x(x)
GLS.set_y_true(y_actual)
GLS.set_weights(np.ones(len(x)))
GLS.set_parameters('estimate')
# GLS.set_parameters([1.2, 4.9])
GLS.set_method('residuals')
results = GLS.get_linear_fit()
print(results)
Your method is not taking an argument. It should be:
def set_x(self, x):
self.x = x
Wrapping properties in get/set methods is a very Java / outdated way of doing things. It is much easier to access the underlying property outside of your class. I.e. rather than: GLS.set_x(12), consider the more Pythonic: GLS.x = 12. This way you don't have to write a get and set method for each property.
Also, it might make more sense for the heavy lifting method of your object, get_linear_fit to be put in the __call__ method. This way, you can run the regression using by just typing GLS() rather than GLS.get_linear_fit()
I want to make 2 instances with same name like,
a = SomeClass(someAttr1)
a = SomeClass(someAttr2)
so that the new one should overwrite the previous one.
I also tried this:
a = SomeClass(someAttr1)
a = None
a = SomeClass(someAttr2)
I tried this but it doesn't overwrite the previous instance and adds it in itself, is there any way to do it?
Here is the code:
### Do not change the Location or Campus classes. ###
### Location class is the same as in lecture. ###
class Location(object):
def __init__(self, x, y):
self.x = x
self.y = y
def move(self, deltaX, deltaY):
return Location(self.x + deltaX, self.y + deltaY)
def getX(self):
return self.x
def getY(self):
return self.y
def dist_from(self, other):
xDist = self.x - other.x
yDist = self.y - other.y
return (xDist ** 2 + yDist ** 2) ** 0.5
def __eq__(self, other):
return (self.x == other.x and self.y == other.y)
def __str__(self):
return '<' + str(self.x) + ',' + str(self.y) + '>'
class Campus(object):
def __init__(self, center_loc):
self.center_loc = center_loc
def __str__(self):
return str(self.center_loc)
class MITCampus(Campus):
""" A MITCampus is a Campus that contains tents """
tents_list = []
def __init__(self, center_loc, tent_loc=Location(0, 0)):
""" Assumes center_loc and tent_loc are Location objects
Initializes a new Campus centered at location center_loc
with a tent at location tent_loc """
# Your code here
Campus.__init__(self, center_loc)
self.tent_loc = tent_loc
self.tents_list.append(self.tent_loc)
def add_tent(self, new_tent_loc):
""" Assumes new_tent_loc is a Location
Adds new_tent_loc to the campus only if the tent is at least 0.5 distance
away from all other tents already there. Campus is unchanged otherwise.
Returns True if it could add the tent, False otherwise. """
# Your code here
new_tent_flag = True
for loc in self.tents_list:
if loc == new_tent_loc or new_tent_loc.dist_from(loc) < 0.5:
new_tent_flag = False
if new_tent_flag:
self.tents_list.append(new_tent_loc)
return True
else:
return False
def get_tents(self):
""" Returns a list of all tents on the campus. The list should contain
the string representation of the Location of a tent. The list should
be sorted by the x coordinate of the location. """
# Your code here
new_list_sorted = sorted(self.tents_list, key=lambda tent: tent.getX())
str_list = []
for x in new_list_sorted:
str_list.append(x.__str__())
return str_list
Test Cases:
Test: 0
c = MITCampus(Location(1,2))
print(c.add_tent(Location(1,2)))
print(c.add_tent(Location(0,0)))
print(c.add_tent(Location(2,3)))
print(c.add_tent(Location(2,3)))
print(c.get_tents())
Output:
True
False
True
False
['<0,0>', '<1,2>', '<2,3>']
Test: 1
init campus with default tent loc
c = MITCampus(Location(-1,-2))
print(sorted(c.get_tents()))
Output:
['<0,0>','<0,0>', '<1,2>', '<2,3>']
Expected Output:
['<0,0>']
As can be seen that the second instance should overwrite the previous one but instead it is adding it. Is the problem in the code? and how to solve it?
Ok,
Your tents_list attribute is a class atribute, so even when your c object is overwrited the tents_list attribute stays the same.
It's better to make your tents_list an object argument so the tents_list attribute is overwrited too.
def __init__(self, center_loc, tent_loc=Location(0, 0)):
Campus.__init__(self, center_loc)
self.tents_list = [] # <--- Add this
self.tent_loc = tent_loc
self.tents_list.append(self.tent_loc)
The object is overwritten but you defined tents_list as a class variable so all of the instances of MITCampus share this list.
So any new instance will add to that list and that seems to you as not "overwritten".
If you want that "overwriting" behavior, move the tents_list into the __init__ method as self.tents_list. Only then it will be unique for each instance.
This is an assignment in Udacity linear algebra refresher course and I have solved it in my way although there is a solution available for this assignment I am trying to solve it in my way. But I am getting error while finding angle between 2 vector that is vectore v1 and v2 .
There is 2 problems asked 2 solve to this program. But even if there is same. but to programs is giving to diffrent out put even if there "angle" function is same.
import math
from decimal import Decimal,getcontext
getcontext().prec = 30
class Vector(object):
def __init__(self,coordinates):
try:
if not coordinates:
raise ValueError
#if coordinates is not passed then it will rise Value Error
self.coordinates = tuple([Decimal(x)for x in coordinates])
#Outside Class :-Vector.coordinates will give print vectors in tuple form
#Inside Class :- self.coordinates will print vectors in tuple form
self.dimension = len(coordinates)
#Outside Class :-Vector.dimension will print vectors dimension/size
#Inside Class :- self.dimension will print vectors dimension/size
except ValueError:
raise ValueError('The coordinates must be non empty')
except TypeError:
raise TypeError('The coordinates must be itterable')
def __str__(self):
return 'Vector:{}'.format(self,coordinates)
def __eq__(self,v):
return self.coordinates == v.coordinates
def add (self,v):
coordinates=[]
for i in range(0,self.dimension):
i=self.coordinates[i]+v.coordinates[i]
coordinates.append(i)
return coordinates
def mul(self,v):
coordinates=[]
for i in range(0,self.dimension):
i=self.coordinates[i]*v.coordinates[i]
coordinates.append(i)
return coordinates
def sub (self,v):
coordinates=[]
for i in range(0,self.dimension):
i=self.coordinates[i]-v.coordinates[i]
coordinates.append(i)
return coordinates
def scal_mul(self,s):
coordinates=[]
for i in self.coordinates:
i=i*Decimal(s)
coordinates.append(i)
return coordinates
def magnitude(self):
mag = 0
for i in self.coordinates:
i=i*i
mag =mag+i
return math.sqrt(mag)
def magnitude1(self):
mag = 0
coordinate_squre=[i*i for i in self.coordinates]
return math.sqrt(sum(coordinate_squre))
# def normalize(self):
# try:
# recip = Decimal(1)/self.magnitude()
# return Vector(self.scal_mul(recip))
# except ZeroDivisionError:
# raise Exception("Can not Normalize Zero Vector")
def normalize(self):
try:
recip = 1/self.magnitude()
return self.scal_mul(recip)
except ZeroDivisionError:
raise Exception("Can not Normalize Zero Vector")
def dot_product(self,v):
mul = self.mul(v)
return sum(mul)
def dot_product2(self,v):
self.mul = [x*y for x,y in zip(self.coordinates,v.coordinates)]
return sum(self.mul)
#-----------------angle functionn is giving wrong answer-------------
def angle_rad(self,v):
norm = self.normalize()
angle = 1/ math.cos(norm.dot_product2(v))
return angle
def angle(self,v,in_degree=False):
nrm_self = self.normalize()
nrm_v = v.normalize()
angle_rad = math.acos(nrm_self.dot_product2(nrm_v))
if in_degree:
angle_in_degree = angle_rad * 180./math.pi
return angle_in_degree
else:
return angle_rad
v1 = Vector([7.887,4.138])
v2 = Vector([-8.802,6.776])
print(v1.angle(v2))
v1 = Vector([-7.579,-7.88])
v2 = Vector([22.737,23.64])
v2.angle(v1)
This Code is giving Error as following
AttributeError Traceback (most recent call last)
<ipython-input-44-2087e4f0ca26> in <module>()
101 v1 = Vector([7.887,4.138])
102 v2 = Vector([-8.802,6.776])
--> 103 print(v1.angle(v2))
104 v1 = Vector([-7.579,-7.88])
105 v2 = Vector([22.737,23.64])
<ipython-input-44-2087e4f0ca26> in angle(self, v, in_degree)
92 nrm_self = self.normalize()
93 nrm_v = v.normalize()
---> 94 angle_rad = math.acos(nrm_self.dot_product2(nrm_v))
95 if in_degree:
96 angle_in_degree = angle_rad * 180./math.pi
AttributeError: 'list' object has no attribute 'dot_product2'
And another program with exactly same angle function is
import math
class Vector(object):
def __init__(self,coordinates):
try:
if not coordinates:
raise ValueError
#if coordinates is not passed then it will rise Value Error
self.coordinates = tuple(coordinates)
#Outside Class :-Vector.coordinates will give print vectors in tuple form
#Inside Class :- self.coordinates will print vectors in tuple form
self.dimension = len(coordinates)
#Outside Class :-Vector.dimension will print vectors dimension/size
#Inside Class :- self.dimension will print vectors dimension/size
except ValueError:
raise ValueError('The coordinates must be non empty')
except TypeError:
raise TypeError('The coordinates must be itterable')
def __str__(self):
return 'Vector:{}'.format(self,coordinates)
def __eq__(self,v):
return self.coordinates == v.coordinates
def add (self,v):
coordinates=[]
for i in range(0,self.dimension):
i=self.coordinates[i]+v.coordinates[i]
coordinates.append(i)
return coordinates
def mul(self,v):
coordinates=[]
for i in range(0,self.dimension):
i=self.coordinates[i]*v.coordinates[i]
coordinates.append(i)
return coordinates
def sub (self,v):
coordinates=[]
for i in range(0,self.dimension):
i=self.coordinates[i]-v.coordinates[i]
coordinates.append(i)
return coordinates
def scal_mul(self,s):
coordinates=[]
for i in self.coordinates:
i=i*s
coordinates.append(i)
return coordinates
def magnitude(self):
mag = 0
for i in self.coordinates:
i=i*i
mag =mag+i
return math.sqrt(mag)
def magnitude1(self):
mag = 0
coordinate_squre=[i*i for i in self.coordinates]
return math.sqrt(sum(coordinate_squre))
def normalize(self):
try:
recip = 1/self.magnitude()
return Vector(self.scal_mul(recip))
except ZeroDivisionError:
raise Exception("Can not Normalize Zero Vector")
def dot_product(self,v):
mul = self.mul(v)
return sum(mul)
def dot_product2(self,v):
self.mul = [x*y for x,y in zip(self.coordinates,v.coordinates)]
return sum(self.mul)
#-----------------angle functionn is giving wrong answer-------------
def angle_rad(self,v):
norm = self.normalize()
angle = 1/ math.cos(norm.dot_product2(v))
return angle
def angle(self,v,in_degree=False):
nrm_self = self.normalize()
nrm_v = v.normalize()
angle_rad = math.acos(nrm_self.dot_product2(nrm_v))
if in_degree:
angle_in_degree = angle_rad * 180./math.pi
return angle_in_degree
else:
return angle_rad
v1 = Vector([7.887,4.138])
v2 = Vector([-8.802,6.776])
print(v1.angle(v2))
v1 = Vector([-7.579,-7.88])
v2 = Vector([22.737,23.64])
v2.angle(v1)
it is giving Error as following
2.0023426999774925
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-214-2e3bee12967a> in <module>()
95 v1 = Vector([-7.579,-7.88])
96 v2 = Vector([22.737,23.64])
---> 97 v2.angle(v1)
98
<ipython-input-214-2e3bee12967a> in angle(self, v, in_degree)
82 nrm_self = self.normalize()
83 nrm_v = v.normalize()
---> 84 angle_rad = math.acos(nrm_self.dot_product2(nrm_v))
85 if in_degree:
86 angle_in_degree = angle_rad * 180./math.pi
ValueError: math domain error
We can see they are having same angle function
The problem with your first program is that the normalize method returns a list of coordinates, return self.scal_mul(recip) but doesn't convert that list into a Vector object like the second program, which instead return Vector(self.scal_mul(recip)). When you call nrm_self.dot_product2(nrm_v), the object nrm_self is a list, not a Vector, and it doesn't have a dot_product2 method. You need to add an explicit call to the constructor Vector(), like in the second program, to be able to invoke methods from the norm_self object.
The math domain error is being thrown because dot_product2 is returning a value larger than 1 (unfortunately, 1.0000000000000002), and the inverse cosine acos(x) is only defined for values between -1 and 1. This is due to numerical error in the normalization, and can be corrected by always rounding the reciprocal slightly up.
I am fairly new to the concepts of caching & memoization. I've read some other discussions & resources here, here, and here, but haven't been able to follow them all that well.
Say that I have two member functions within a class. (Simplified example below.) Pretend that the first function total is computationally expensive. The second function subtotal is computationally simple, except that it uses the return from the first function, and so also becomes computationally expensive because of this, in that it currently needs to re-call total to get its returned result.
I want to cache the results of the first function and use this as the input to the second, if the input y to subtotal shares the input x to a recent call of total. That is:
If calling subtotal() where y is equal to the value of x in a
previous call of total, then use that cached result instead of
re-calling total.
Otherwise, just call total() using x = y.
Example:
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
def total(self, x):
return (self.a + self.b) * x # some time-expensive calculation
def subtotal(self, y, z):
return self.total(x=y) + z # Don't want to have to re-run total() here
# IF y == x from a recent call of total(),
# otherwise, call total().
With Python3.2 or newer, you could use functools.lru_cache.
If you were to decorate the total with functools.lru_cache directly, then the lru_cache would cache the return values of total based on the value of both arguments, self and x. Since lru_cache's internal dict stores a reference to self, applying #lru_cache directly to a class method creates a circular reference to self which makes instances of the class un-dereferencable (hence a memory leak).
Here is a workaround which allows you to use lru_cache with class methods -- it caches results based on all arguments other than the first one, self, and uses a weakref to avoid the circular reference problem:
import functools
import weakref
def memoized_method(*lru_args, **lru_kwargs):
"""
https://stackoverflow.com/a/33672499/190597 (orly)
"""
def decorator(func):
#functools.wraps(func)
def wrapped_func(self, *args, **kwargs):
# We're storing the wrapped method inside the instance. If we had
# a strong reference to self the instance would never die.
self_weak = weakref.ref(self)
#functools.wraps(func)
#functools.lru_cache(*lru_args, **lru_kwargs)
def cached_method(*args, **kwargs):
return func(self_weak(), *args, **kwargs)
setattr(self, func.__name__, cached_method)
return cached_method(*args, **kwargs)
return wrapped_func
return decorator
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
#memoized_method()
def total(self, x):
print('Calling total (x={})'.format(x))
return (self.a + self.b) * x
def subtotal(self, y, z):
return self.total(x=y) + z
mobj = MyObject(1,2)
mobj.subtotal(10, 20)
mobj.subtotal(10, 30)
prints
Calling total (x=10)
only once.
Alternatively, this is how you could roll your own cache using a dict:
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
self._total = dict()
def total(self, x):
print('Calling total (x={})'.format(x))
self._total[x] = t = (self.a + self.b) * x
return t
def subtotal(self, y, z):
t = self._total[y] if y in self._total else self.total(y)
return t + z
mobj = MyObject(1,2)
mobj.subtotal(10, 20)
mobj.subtotal(10, 30)
One advantage of lru_cache over this dict-based cache is that the lru_cache
is thread-safe. The lru_cache also has a maxsize parameter which can help
protect against memory usage growing without bound (for example, due to a
long-running process calling total many times with different values of x).
Thank you all for the responses, it was helpful just to read them and see what's going on under the hood. As #Tadhg McDonald-Jensen said, it seems like I didn't need anything more here than #functools.lru_cache. (I'm in Python 3.5.) Regarding #unutbu's comment, I'm not getting an error from decorating total() with #lru_cache. Let me correct my own example, I'll keep this up here for other beginners:
from functools import lru_cache
from datetime import datetime as dt
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
#lru_cache(maxsize=None)
def total(self, x):
lst = []
for i in range(int(1e7)):
val = self.a + self.b + x # time-expensive loop
lst.append(val)
return np.array(lst)
def subtotal(self, y, z):
return self.total(x=y) + z # if y==x from a previous call of
# total(), used cached result.
myobj = MyObject(1, 2)
# Call total() with x=20
a = dt.now()
myobj.total(x=20)
b = dt.now()
c = (b - a).total_seconds()
# Call subtotal() with y=21
a2 = dt.now()
myobj.subtotal(y=21, z=1)
b2 = dt.now()
c2 = (b2 - a2).total_seconds()
# Call subtotal() with y=20 - should take substantially less time
# with x=20 used in previous call of total().
a3 = dt.now()
myobj.subtotal(y=20, z=1)
b3 = dt.now()
c3 = (b3 - a3).total_seconds()
print('c: {}, c2: {}, c3: {}'.format(c, c2, c3))
c: 2.469753, c2: 2.355764, c3: 0.016998
In this case I would do something simple, maybe is not the most elegant way, but works for the problem:
class MyObject(object):
param_values = {}
def __init__(self, a, b):
self.a, self.b = a, b
def total(self, x):
if x not in MyObject.param_values:
MyObject.param_values[x] = (self.a + self.b) * x
print(str(x) + " was never called before")
return MyObject.param_values[x]
def subtotal(self, y, z):
if y in MyObject.param_values:
return MyObject.param_values[y] + z
else:
return self.total(y) + z