I am following category theory for programmers and trying to implement a compose function in Python. What I have currently is
def compose(f, g):
def result(*args):
return f(g(*args))
return result
def square(x):
return x * x
def add(x, y):
return x + y
def f(x, y):
return min(x, y)
def g(x, y):
return x + y, x * y
result = compose(square, add)(1, 2)
print(result)
result = compose(f, g)(1, 2)
print(result)
But this will not work because I don't know whether g(*args) in compose will return an object which is an iterator or just a single return value. Is there a way to write compose in Python without using isinstance??
One option is to always return a list; then you can unpack g without checking:
def compose(f, g):
def result(*args):
return f(*g(*args)) # we can unpack g here if we know every function returns a list
return result
def square(x):
return [x * x]
def add(x, y):
return [x + y]
def f(x, y):
return [min(x, y)]
def g(x, y):
return x + y, x * y
result = compose(square, add)(1, 2)
print(result)
result = compose(f, g)(1, 2)
print(result)
Related
I have simple function for which I want to calculate the minimum value
from scipy import optimize
def f(x):
return x**2
optimize.fminbound(f, -1, 2)
This works fine. Now I modify above function which now returns a dictionary
def f1(x):
y = x ** 2
return_list = {}
return_list['x'] = x
return_list['y'] = y
return return_list
While it is returning multiple objects x and y, I want to apply fminbound only on y, the other object x is just for informational purpose for other use of this function.
How can I use fminbound for this setup?
You need a simple wrapper function that extracts y:
def f1(x):
y = x ** 2
return_list = {}
return_list['x'] = x
return_list['y'] = y
return return_list
optimize.fminbound(lambda x: f1(x)['y'], -1, 2)
In Pytorch, how can I make the gradient of a parameter a function itself?
Here is a simple code snippet:
import torch
def fun(q):
def result(w):
l = w * q
l.backward()
return w.grad
return result
w = torch.tensor((2.), requires_grad=True)
q = torch.tensor((3.), requires_grad=True)
f = fun(q)
print(f(w))
In the code above, how can I make f(w) have gradient with respect to q?
EDIT: based on the accepted answer I was able to write a code that works. Essentially I am alternating between 2 optimization steps. For dim == 1 it works and for dim == 2 it does not. I get the error "RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time."
import torch
class f_class():
def __init__(self, dim):
self.dim = dim
if self.dim == 1:
self.w = torch.tensor((3.), requires_grad=True)
elif self.dim == 2:
self.w = [torch.tensor((3.), requires_grad=True), torch.tensor((5.), requires_grad=True)]
else:
raise ValueError("dim 1 or 2")
def forward(self, x):
if self.dim == 1:
return torch.mul(self.w, x)
elif self.dim == 2:
return torch.mul(torch.mul(self.w[0], self.w[1]), x)
def set_w(self, w):
self.w = w
def get_w(self):
return self.w
class g_class():
def __init__(self):
self.q = torch.tensor((4.), requires_grad=True)
def forward(self, f):
return torch.mul(self.q, f)
def set_q(self, q):
self.q = q
def get_q(self):
return self.q
def w_new(f, g, dim):
loss_g = g.forward(f.forward(xd))
if dim == 1:
grads = torch.autograd.grad(loss_g, f.get_w(), create_graph=True, only_inputs=True)[0]
temp = f.get_w().detach() + grads
else:
grads = torch.autograd.grad(loss_g, f.get_w(), create_graph=True, only_inputs=True)
temp = [wi.detach() + gi for wi, gi in zip(f.get_w(), grads)]
return temp
def q_new(f, g):
loss_f = 2 * f.forward(xd)
loss_f.backward()
temp = g.get_q().detach() + g.get_q().grad
temp.requires_grad = True
return temp
dim = 1
xd = torch.tensor((2.))
f = f_class(dim)
g = g_class()
for _ in range(3):
print(f.get_w(), g.get_q())
wnew = w_new(f, g, dim)
f.set_w(wnew)
print(f.get_w(), g.get_q())
qnew = q_new(f, g)
g.set_q(qnew)
print(f.get_w(), g.get_q())
When computing gradients, if you want to construct a computation graph for the gradient itself you need to specify create_graph=True to autograd.
A potential source of error in your code is using Tensor.backward within f. The problem here is that w.grad and q.grad will be populated with the gradient of l. This means that when you call f(w).backward(), the gradients of both f and l will be added to w.grad and q.grad. In effect you will end up with w.grad being equal to dl/dw + df/dw and similarly for q.grad. One way to get around this is to zero the gradients after f(w) but before .backward(). A better way is to use torch.autograd.grad within f. Using the latter approach, the grad attribute of w and q will not be populated when calling f, only when calling .backward(). This leaves room for things like gradient accumulation during training.
import torch
def fun(q):
def result(w):
l = w * q
return torch.autograd.grad(l, w, only_inputs=True, retain_graph=True)[0]
return result
w = torch.tensor((2.), requires_grad=True)
q = torch.tensor((3.), requires_grad=True)
f = fun(q)
f(w).backward()
print('w.grad:', w.grad)
print('q.grad:', q.grad)
which results in
w.grad: None
q.grad: tensor(1.)
Note that w.grad was not populated. This is because f(w) = dl/dw = q is not a function of w, and therefore w is not part of the computation graph. If you're using a standard pytorch optimizer this is fine since None gradients are implicitly assumed to be zero.
If l were instead a non-linear function of w, then w.grad would have been populated after f(w).backward(). For example
import torch
def fun(q):
def result(w):
# now dl/dw = 2 * w * q
l = w**2 * q
return torch.autograd.grad(l, w, only_inputs=True, create_graph=True)[0]
return result
w = torch.tensor((2.), requires_grad=True)
q = torch.tensor((3.), requires_grad=True)
f = fun(q)
f(w).backward()
print('w.grad:', w.grad)
print('q.grad:', q.grad)
which results in
w.grad: tensor(6.)
q.grad: tensor(4.)
guys how can I make it so that calling make_repeater(square, 0)(5) return 5 instead of 25? I'm guessing I would need to change the line "function_successor = h" because then I'm just getting square(5) but not sure what I need to change it to...
square = lambda x: x * x
def compose1(h, g):
"""Return a function f, such that f(x) = h(g(x))."""
def f(x):
return h(g(x))
return f
def make_repeater(h, n):
iterations = 1
function_successor = h
while iterations < n:
function_successor = compose1(h, function_successor)
iterations += 1
return function_successor
it needs to satisfy a bunch of other requirements like:
make_repeater(square, 2)(5) = square(square(5)) = 625
make_repeater(square, 4)(5) = square(square(square(square(5)))) = 152587890625
To achieve that, you have to use the identity function (f(x) = x) as the initial value for function_successor:
def compose1(h, g):
"""Return a function f, such that f(x) = h(g(x))."""
def f(x):
return h(g(x))
return f
IDENTITY_FUNCTION = lambda x: x
def make_repeater(function, n):
function_successor = IDENTITY_FUNCTION
# simplified loop
for i in range(n):
function_successor = compose1(function, function_successor)
return function_successor
if __name__ == "__main__":
square = lambda x: x * x
print(make_repeater(square, 0)(5))
print(make_repeater(square, 2)(5))
print(make_repeater(square, 4)(5))
and the output is
5
625
152587890625
This isn't most optimal for performance though since the identity function (which doesn't do anything useful) is always part of the composed function, so an optimized version would look like this:
def make_repeater(function, n):
if n <= 0:
return IDENTITY_FUNCTION
function_successor = function
for i in range(n - 1):
function_successor = compose1(function, function_successor)
return function_successor
I am trying to learn how to group functions by class. As an example, I tried to code a generalized least squares method to find the equation of a best-fitting line between a set of (x,y) coordinates. For my particular case, I chose a simple line y = x + 5, so slope should be close to 1 and y-intercept should be close to 5. Running my attempt at a coded solution below produces the error TypeError: set_x() takes 1 positional argument but 2 were given, though I am trying to pass an array of x-points. How can I circumvent this error?
import numpy as np
from scipy.optimize import minimize
class GeneralizedLeastSquares:
def __init__(self, residuals=None, parameters=None, x=None, y_true=None, y_fit=None, weights=None, method=None):
self.residuals = residuals
self.parameters = parameters
self.x = x
self.y_true = y_true
self.y_fit = y_fit
self.weights = weights
self.method = method
def set_residuals(self, residuals):
self.residuals = residuals
def set_parameters(self, parameters):
self.parameters = parameters
def set_x(self, x):
self.x = x
def set_y_true(self, y_true):
self.y_true = y_true
def set_y_fit(self, y_fit):
self.y_fit = y_fit
def set_weights(self, weights):
self.weights = weights
def set_method(self, method):
self.method = method
def get_residuals(self):
return [(self.y_true[idx] - self.y_fit[idx])**2 for idx in range(len(self.y_true)) if len(self.y_true) == len(self.y_fit) ]
def get_parameters(self):
return self.parameters
def get_x(self):
return self.x
def get_y_true(self):
return self.y_true
def get_y_fit(self):
return [self.parameters[0] * self.x[idx] + self.parameters[1] for idx in range(len(self.x))]
def get_weights(self):
return self.weights
def update_weights(self):
inverse_residuals = [1/self.residuals[idx] for idx in range(len(residuals))]
inverse_residuals_abs = [abs(inverse_residual) for inverse_residual in inverse_residuals]
residual_abs_total = sum(inverse_residuals_abs)
return [inverse_residuals_abs[idx]/residual_abs_total for idx in range(len(inverse_residuals_abs))]
def get_method(self):
return self.method
def get_error_by_residuals(self):
return sum([self.weights[idx] * self.residuals[idx] for idx in range(len(self.residuals))])
def get_error_by_std_mean(self):
return np.std(self.y_true)/np.sqrt(len(self.y_true))
def get_linear_fit(self):
"""
"""
if self.parameters == 'estimate':
slope_init = (self.y_true[-1] - self.y_true[0]) / (self.x[-1] - self.x[0])
b_init = np.mean([self.y_true[-1] - slope_init * self.x[-1], self.y_true[0] - slope_init * self.x[0]])
self.parameters = [slope_init, b_init]
elif not isinstance(self.parameters, (list, np.ndarray)):
raise ValueError("parameters = 'estimate' or [slope, y-intercept]")
meths = ['residuals', 'std of mean']
funcs = [get_error_by_residuals, get_error_by_std_mean]
func = dict(zip(meths, funcs))[self.method]
res = minimize(func, x0=self.parameters, args=(self,), method='Nelder-Mead')
self.parameters = [res.x[0], res.x[1]]
self.y_fit = get_y_fit(self)
self.residuals = get_residuals(self)
self.weights = update_weights(self)
return self.parameters, self.y_fit, self.residuals, self.weights
x = np.linspace(0, 4, 5)
y_true = np.linspace(5, 9, 5) ## using slope=1, y-intercept=5
y_actual = np.array([4.8, 6.2, 7, 8.1, 8.9]) ## test data
GLS = GeneralizedLeastSquares()
GLS.set_x(x)
GLS.set_y_true(y_actual)
GLS.set_weights(np.ones(len(x)))
GLS.set_parameters('estimate')
# GLS.set_parameters([1.2, 4.9])
GLS.set_method('residuals')
results = GLS.get_linear_fit()
print(results)
Your method is not taking an argument. It should be:
def set_x(self, x):
self.x = x
Wrapping properties in get/set methods is a very Java / outdated way of doing things. It is much easier to access the underlying property outside of your class. I.e. rather than: GLS.set_x(12), consider the more Pythonic: GLS.x = 12. This way you don't have to write a get and set method for each property.
Also, it might make more sense for the heavy lifting method of your object, get_linear_fit to be put in the __call__ method. This way, you can run the regression using by just typing GLS() rather than GLS.get_linear_fit()
I am fairly new to the concepts of caching & memoization. I've read some other discussions & resources here, here, and here, but haven't been able to follow them all that well.
Say that I have two member functions within a class. (Simplified example below.) Pretend that the first function total is computationally expensive. The second function subtotal is computationally simple, except that it uses the return from the first function, and so also becomes computationally expensive because of this, in that it currently needs to re-call total to get its returned result.
I want to cache the results of the first function and use this as the input to the second, if the input y to subtotal shares the input x to a recent call of total. That is:
If calling subtotal() where y is equal to the value of x in a
previous call of total, then use that cached result instead of
re-calling total.
Otherwise, just call total() using x = y.
Example:
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
def total(self, x):
return (self.a + self.b) * x # some time-expensive calculation
def subtotal(self, y, z):
return self.total(x=y) + z # Don't want to have to re-run total() here
# IF y == x from a recent call of total(),
# otherwise, call total().
With Python3.2 or newer, you could use functools.lru_cache.
If you were to decorate the total with functools.lru_cache directly, then the lru_cache would cache the return values of total based on the value of both arguments, self and x. Since lru_cache's internal dict stores a reference to self, applying #lru_cache directly to a class method creates a circular reference to self which makes instances of the class un-dereferencable (hence a memory leak).
Here is a workaround which allows you to use lru_cache with class methods -- it caches results based on all arguments other than the first one, self, and uses a weakref to avoid the circular reference problem:
import functools
import weakref
def memoized_method(*lru_args, **lru_kwargs):
"""
https://stackoverflow.com/a/33672499/190597 (orly)
"""
def decorator(func):
#functools.wraps(func)
def wrapped_func(self, *args, **kwargs):
# We're storing the wrapped method inside the instance. If we had
# a strong reference to self the instance would never die.
self_weak = weakref.ref(self)
#functools.wraps(func)
#functools.lru_cache(*lru_args, **lru_kwargs)
def cached_method(*args, **kwargs):
return func(self_weak(), *args, **kwargs)
setattr(self, func.__name__, cached_method)
return cached_method(*args, **kwargs)
return wrapped_func
return decorator
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
#memoized_method()
def total(self, x):
print('Calling total (x={})'.format(x))
return (self.a + self.b) * x
def subtotal(self, y, z):
return self.total(x=y) + z
mobj = MyObject(1,2)
mobj.subtotal(10, 20)
mobj.subtotal(10, 30)
prints
Calling total (x=10)
only once.
Alternatively, this is how you could roll your own cache using a dict:
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
self._total = dict()
def total(self, x):
print('Calling total (x={})'.format(x))
self._total[x] = t = (self.a + self.b) * x
return t
def subtotal(self, y, z):
t = self._total[y] if y in self._total else self.total(y)
return t + z
mobj = MyObject(1,2)
mobj.subtotal(10, 20)
mobj.subtotal(10, 30)
One advantage of lru_cache over this dict-based cache is that the lru_cache
is thread-safe. The lru_cache also has a maxsize parameter which can help
protect against memory usage growing without bound (for example, due to a
long-running process calling total many times with different values of x).
Thank you all for the responses, it was helpful just to read them and see what's going on under the hood. As #Tadhg McDonald-Jensen said, it seems like I didn't need anything more here than #functools.lru_cache. (I'm in Python 3.5.) Regarding #unutbu's comment, I'm not getting an error from decorating total() with #lru_cache. Let me correct my own example, I'll keep this up here for other beginners:
from functools import lru_cache
from datetime import datetime as dt
class MyObject(object):
def __init__(self, a, b):
self.a, self.b = a, b
#lru_cache(maxsize=None)
def total(self, x):
lst = []
for i in range(int(1e7)):
val = self.a + self.b + x # time-expensive loop
lst.append(val)
return np.array(lst)
def subtotal(self, y, z):
return self.total(x=y) + z # if y==x from a previous call of
# total(), used cached result.
myobj = MyObject(1, 2)
# Call total() with x=20
a = dt.now()
myobj.total(x=20)
b = dt.now()
c = (b - a).total_seconds()
# Call subtotal() with y=21
a2 = dt.now()
myobj.subtotal(y=21, z=1)
b2 = dt.now()
c2 = (b2 - a2).total_seconds()
# Call subtotal() with y=20 - should take substantially less time
# with x=20 used in previous call of total().
a3 = dt.now()
myobj.subtotal(y=20, z=1)
b3 = dt.now()
c3 = (b3 - a3).total_seconds()
print('c: {}, c2: {}, c3: {}'.format(c, c2, c3))
c: 2.469753, c2: 2.355764, c3: 0.016998
In this case I would do something simple, maybe is not the most elegant way, but works for the problem:
class MyObject(object):
param_values = {}
def __init__(self, a, b):
self.a, self.b = a, b
def total(self, x):
if x not in MyObject.param_values:
MyObject.param_values[x] = (self.a + self.b) * x
print(str(x) + " was never called before")
return MyObject.param_values[x]
def subtotal(self, y, z):
if y in MyObject.param_values:
return MyObject.param_values[y] + z
else:
return self.total(y) + z