I have an object with a dict of functions that I need to store (for example, in JSON). Such functions are not bound to an object nor decorated and are defined stand-alone in different modules. Is there a way to programatically get the module where the function is defined?
What I expect:
import importlib
class FunctionBag(object):
#property
def functions(self)
return self._functions
def __init__(self, functions):
self._functions = functions
def to_dict_for_json(self):
result = {}
for key, fn, in self._functions.items():
name = fn.__name__
module = MAGIC(fn)
result[key] = (module, name,)
#classmethod
def FromDictOfJson(cls, dct):
functions = {}
for key, value, in dct.items():
module,name, = value
module = importlib.import_module(module)
functions[key] = getattr(module, name)
return cls(functions)
Is there a way to do what MAGIC() does?
>>> def foo():
... pass
...
>>> foo.__module__
'__main__'
>>> sys.modules[foo.__module__]
<module '__main__' (built-in)>
Related
I am trying to assign dictionary keys to object functions but for some reason it won't work inside of decorators. When I try to call a.run(), self doesn't seem to be passed into the dictionary func. I also don't have access to f.self in decorator so I know it has to be something wrong in there. I have written a simple example of my code. I want it to be something similar to app.route in flask being that it init the mapping between endpoints and functions.
ERROR:
Traceback (most recent call last):
File "main.py", line 27, in <module>
a.run()
File "main.py", line 14, in run
self.rmap[k](data)
TypeError: one_way() missing 1 required positional argument: 'data'
CODE:
class A (object):
def __init__(self):
self.rmap = {}
def route(self, r):
def decorator(f):
self.rmap[r] = f
return f
return decorator
def run(self):
data = [1,2,3]
for k in self.rmap.keys():
self.rmap[k](data)
a = A()
class B (object):
def __init__(self):
pass
#a.route('/one/way')
def one_way (self, data):
print('A WAY:{}'.format(self))
b = B()
a.run()
At the time it's being decorated, one_way() is a plain function, not a method - it only becomes a method when looked up on a B instance. IOW, you need to explicitely provide a B instance when calling it from A().run() (the fact you have a global b instance in your code is irrelevant - the function object stored in a.rmap knows absolutely nothing about it, nor even about the B class FWIW.
To make a long story short, your current design cannot work as is. If you only ever intend to decorate methods (well, functions) from one single class and call them on one single instance of this class, you could pass an instance of this class to a.run() ie:
class A():
# ...
def run(self, obj):
data = [1,2,3]
for k in self.rmap.keys():
self.rmap[k](obj, data)
b = B()
a.run(b)
but this would be of very limited use.
Or you could just use the decorator to "mark" functions to be used for routing (together with the effective route), add some register() methdo to A and explicitely pass B or whatever else instance to this method ie
def route(r):
def decorator(f):
f._A_route = r
return f
return decorator
class A (object):
def __init__(self):
self.rmap = {}
def register(self, *objects):
for obj in objects:
self._register(obj)
def _register(self, obj):
for name in dir(obj):
if name.startswith("_"):
continue
attr = getattr(obj, name)
if callable(attr) and hasattr(attr, "_A_route"):
self.rmap[attr._A_route] = attr
def run(self):
data = [1,2,3]
for k in self.rmap.keys():
self.rmap[k](data)
class B (object):
def __init__(self):
pass
#route('/one/way')
def one_way (self, data):
print('A WAY:{}'.format(self))
if __name__ == "__main__":
a = A()
b = B()
a.register(b)
a.run()
Now there might be better solutions for your concrete use case, but it's impossible to tell without knowing about the whole context etc.
When calling self.rmap[k](data) you are not passing in the self parameter. This has to be an instance of class B in order to work.
Normally you'd just pass on the parameters with which the decorated function was called, but you seem to want to use your decorated function differently. In your case what would work is:
def run(self):
data = [1,2,3]
b = B()
for k in self.rmap.keys():
self.rmap[k](b, data)
You could of course also instantiate the B instance somewhere else if you want to reuse it between calls.
While testing the create_response method, I cannot seem to mock the return value of the get_external_response method.
/foo/response
from abc import ABCMeta, abstractmethod
def create_response(url, type):
query = create_query(url, type)
external_response = get_external_response(query) <-- what I want to mock
return external_response
def create_query(url, type):
cc = MyFactory
return cc.get_concrete_class(url, type)
def get_external_response(cc):
return cc.make_query()
class MyAbstractClass(metaclass=ABCMeta):
def __init__(self, url, type):
self.url = url
self.type = type
self.query = self.make_query()
#abstractmethod
def make_query(self):
pass
class MyFactory:
#staticmethod
def get_concrete_class(url, type):
if type == 'A':
return MyClass(url, type)
else:
print("not valid type")
class MyClass(MyAbstractClass):
def __init__(self, url, type):
super().__init__(url, type)
def make_query(self):
return self.url + self.type
if __name__ == '__main__':
result = create_response('www.stackoverflow.com', 'A')
print(result)
If I run the above, I get the expected www.stackoverflow.comA.
But if try to mock the return value of get_external_response, it does not seem to do anything: it still returns www.stackoverflow.comA and the assertion below fails.
/foo/test_response
from foo.response import create_response
import pytest
from unittest import mock
def test_create_response():
mock_external_response = mock.Mock()
mock_external_response.create_flask_response.return_value = 'www'
result = create_response('www.stackoverflow.com', 'A')
assert result == 'www'
I do not understand why the return value is not set since when the create_response is called, it will eventually reach the point of calling the create_flask_response which, if I am not mistaken, should return www given that I have mocked it.
Can someone explain what I am doing wrong?
I noticed that you are creating a Mock object inside of a function, but are not actually using the Mock.
It looks like you need to patch the function where it is used to use the Mock.
/foo/test_response
#mock.patch('foo.response.get_external_response')
def test_create_response(mock_get_external_reponse):
mock_get_external_response.return_value = 'www' # This is a Mock object, but will be used as a patch over the actual function where it is used
result = create_response('www.stackoverflow.com', 'A')
assert result == 'www'
Quick links to relevant documentation sections for convenience:
Intro to mocking and patching: https://docs.python.org/3/library/unittest.mock.html#quick-guide
Patch specifically here: https://docs.python.org/3/library/unittest.mock.html#unittest.mock.patch
I have 2 classes:
class A:
name = 'test'
def __init__(self):
pass
#staticmethod
def from_json(json: dict) -> object:
obj = A()
obj.name = json["name"]
return obj
class B(A):
description = "desc"
def __init__(self):
super().__init__(self) # I was originally doing: A.__init__(self) but online said to use super.
#staticnmethod
def from_json(json: dict) -> object:
obj = A.from_json(json) # As seen above, A.from_json, this returns an instance of A.
obj.description = json["description"]
return obj
I know there isnt really any casting, but I want the returned class to be of type B, so it gains all the other new properties / methods.
How to i have B::from_json return type B? I was thinking there was a way to create something like:
b = B()
and then through some python magic pass all properties from A into B and then return b, but i wasnt sure if that is the right solution.
Here is now a functional test of the flaw:
x = A.from_json({'name': 'foo'})
z = B.from_json({ 'name': 'thor', 'description': 'god of thunder'})
type(x) == A # <class '__main__.A'>
type(z) == B # <class '__main__.A'>
You should use classmethod here, not staticmethod. Then you can remove all the hardcoded classes references
class A:
name = 'test'
def __init__(self):
pass
#classmethod
def from_json(cls, json: dict) -> object:
obj = cls()
obj.name = json["name"]
return obj
class B(A):
description = "desc"
def __init__(self):
super().__init__()
#classmethod
def from_json(cls, json: dict) -> object:
obj = super().from_json(json)
obj.description = json["description"]
return obj
print(type(B.from_json({'name': 'name', 'description': 'description'})))
Outputs
<class '__main__.B'>
And your tests:
x = A.from_json({'name': 'foo'})
z = B.from_json({ 'name': 'thor', 'description': 'god of thunder'})
print(type(x) == A)
print(type(z) == B)
Outputs
True
True
Using classmethod is actually the recommended way in the official Python docs to create alternative "constructors" (which is what from_json essentially is). Otherwise, you don't have any access to the correct class (as you found out).
This works because (quoted from the docs):
If a class method is called for a derived class, the derived class
object is passed as the implied first argument.
I have written a custom logging class for module logging that I called call. With this class I hope to place it in any function/method and it logs the function name with its arguments and all values the function was called with.
This works fine for class methods
Foo.bar(self, a=1, b=2, c=3, *args=(), **kwargs={'something': 4})
using this minimal example
import logging
import inspect
def call(logger):
fname = [] # Function name including module and class
fargs = [] # Arguments of function including positional and named arguments
parentframe = inspect.stack()[1][0]
module = inspect.getmodule(parentframe)
if module and module.__name__ != "__main__":
fname.append(module.__name__)
codename = parentframe.f_code.co_name
if "self" in parentframe.f_locals:
fname.append(parentframe.f_locals["self"].__class__.__name__)
fobj = getattr(parentframe.f_locals["self"].__class__, codename)
if codename != "<module>":
fname.append(codename)
argspec = inspect.formatargspec(*inspect.getfullargspec(fobj))
args = argspec[1:-1].split(",")
for arg in args:
argkey = arg.strip().replace("*", "").split("=")[0]
if arg == "self":
fargs.append("self")
else:
fargs.append(arg.split("=")[0] + "=" + str(parentframe.f_locals[argkey]))
del parentframe
msg = ".".join(fname) + "(" + ",".join(fargs) + ")"
if logger.isEnabledFor(30):
logger.log(30, msg)
class Foo:
def __init__(self, l):
self.logger = l
def bar(self, a, b, c=3, *args, **kwargs):
call(self.logger)
if __name__ == "__main__":
logging.addLevelName(30, "CALL")
logger = logging.getLogger('blub')
logger.level = 20
f = Foo(logger)
f.bar(1, 2, something=4)
print("done...")
My problem is when I use the same functionality on static methods or simple functions. It fails at the line where I get the function object (fobj = getattr(parentframe.f_locals["self"].__class__, codename)
) using self.
parentframe is the Frame object of the function in questions I presume. I have not yet found a way to get the function object from that object. Is there a way?
Use getattr(module, codename) to get the function-object of functions that are not contained in classes.
Here the full code:
import logging
import inspect
def call(logger):
fname = [] # Function name including module and class
fargs = [] # Arguments of function including positional and named arguments
parentframe = inspect.stack()[1][0]
module = inspect.getmodule(parentframe)
if module and module.__name__ != "__main__":
fname.append(module.__name__)
codename = parentframe.f_code.co_name
if "self" in parentframe.f_locals:
fname.append(parentframe.f_locals["self"].__class__.__name__)
fobj = getattr(parentframe.f_locals["self"].__class__, codename)
else:
fobj = getattr(module, codename)
if codename != "<module>":
fname.append(codename)
argspec = inspect.formatargspec(*inspect.getfullargspec(fobj))
args = argspec[1:-1].split(",")
for arg in args:
argkey = arg.strip().replace("*", "").split("=")[0]
if arg == "self":
fargs.append("self")
else:
fargs.append(arg.split("=")[0] + "=" + str(parentframe.f_locals[argkey]))
del parentframe
msg = ".".join(fname) + "(" + ",".join(fargs) + ")"
if logger.isEnabledFor(30):
logger.log(30, msg)
class Foo:
def __init__(self, l):
self.logger = l
def bar(self, a, b, c=3, *args, **kwargs):
call(self.logger)
def boo(a, b, c=3, *args, **kwargs):
call(logger)
if __name__ == "__main__":
logging.addLevelName(30, "CALL")
logger = logging.getLogger('blub')
logger.level = 20
f = Foo(logger)
f.bar(1, 2, something=4)
boo(1, 2, something=4)
print("done...")
Taking ideas from both, I wrote this function to find the function object from a frame. I'm sure there's some edge cases around inherited staticmethods, and obviously any code not using the cls and self conventions for param names. This also doesn't work for lambdas... but you shouldn't be logging anything out in a lamba anyway :-P
def _get_func_obj(f):
"""
Get function object from a frame. If it can't find it, return None
"""
codename = f.f_code.co_name
fobj = None
try:
if "self" in f.f_locals: # regular method
fobj = getattr(f.f_locals["self"].__class__, codename)
elif "cls" in f.f_locals: # class method
fobj = getattr(f.f_locals["cls"], codename)
else:
module = inspect.getmodule(f) # only fetch module if we need it
if hasattr(module, codename): # regular module level function
fobj = getattr(module, codename)
else: # static method
classes = [
getattr(module, name)
for name in dir(module)
if inspect.isclass(getattr(module, name))
]
for cls in classes:
if (
hasattr(cls, codename)
and getattr(cls, codename).__code__ == f.f_code
):
fobj = getattr(cls, codename)
break
if fobj is None:
"""it's likely some nested function/method or a lambda, who logs in a lambda?"""
return fobj
except Exception:
"""never break logging"""
I have a method decorator looking like
def debug_run(fn):
from functools import wraps
#wraps(fn)
def wrapper(self, *args, **kw):
# log some stuff
# timeit fn
res = fn(self, *args, **kw)
return wrapper
Right now I used to use it apply on each method that I want to debug. Now i'm trying to apply to all class method using a class decorator looking like.
Rather doing
class A():
#debug_run
def f(self):
pass
I do
#decallmethods(debug_run)
class A():
def f(self):
pass
def decallmethods(decorator):
def dectheclass(cls):
for name, m in inspect.getmembers(cls, inspect.ismethod):
if name in getattr(cls, 'METHODS_TO_INSPECT', []):
setattr(cls, name, decorator(m))
return cls
return dectheclass
Trying to apply to decorator to the base class, not working as expected. no log to the console. Now i wonder if this approach is the good or I should used something else (apply the debug decorator to selected method from base class to all sub classes).
[EDIT]
Finally found why no logs were printed
Why is there a difference between inspect.ismethod and inspect.isfunction from python 2 -> 3?
Here a complete example reflecting my code
import inspect
import time
import logging as logger
from functools import wraps
logger.basicConfig(format='LOGGER - %(asctime)s %(message)s', level=logger.DEBUG)
def debug_run(fn):
#wraps(fn)
def wrapper(self, *args, **kw):
logger.debug(
"call method %s of instance %s with %r and %s "
% (fn.__name__, self, args, kw))
time1 = time.time()
res = fn(self, *args, **kw)
time2 = time.time()
logger.debug(
"%s function %0.3f ms" % (fn, (time2-time1)*1000.0))
return res
return wrapper
def decallmethods(decorator):
def dectheclass(cls):
for name, m in inspect.getmembers(
cls, predicate=lambda x: inspect.isfunction(x) or inspect.ismethod(x)):
methods_to_inspect = getattr(cls, 'METHODS_TO_INSPECT', [])
if name in methods_to_inspect:
setattr(cls, name, decorator(m))
return cls
return dectheclass
class B(object):
METHODS_TO_INSPECT = ["bfoo1", "bfoo2", "foo"]
def __str__(self):
return "%s:%s" % (repr(self), id(self))
def bfoo1(self):
pass
def bfoo2(self):
pass
def foo(self):
pass
def run(self):
print("print - Base run doing nothing")
class C(object):
pass
#decallmethods(debug_run)
class A(B, C):
METHODS_TO_INSPECT = ["bfoo1", "bfoo2", "foo", "run"]
def foo(self):
print("print - A foo")
def run(self):
self.bfoo1()
self.bfoo2()
self.foo()
a = A()
b = B()
a.run()
b.run()
In this case applying decallmethods to B, will not affect the A so i must to apply to both A and B thus to all sub classes of B.
It is possible to have such mechanism that permit to apply decallmethods to all sub classes methods ?
look at this:
How can I decorate all functions of a class without typing it over and over for each method added? Python
delnan has a good answer,
only add this rule to his answer
if name in getattr(cls, 'METHODS_TO_INSPECT', []):