I came across this link but am still struggling to construct an answer.
This is what one of the complex structs that I have looks like. This is actually a deep nested struct within other structs :)
/*
* A domain consists of a variable length array of 32-bit unsigned integers.
* The domain_val member of the structure below is the variable length array.
* The domain_count is the number of elements in the domain_val array.
*/
typedef struct domain {
uint32_t domain_count;
uint32_t *domain_val;
} domain_t;
The test code in C is doing something like this:
uint32_t domain_seg[4] = { 1, 9, 34, 99 };
domain_val = domain_seg;
The struct defined in python is
class struct_domain(ctypes.Structure):
_pack_ = True # source:False
_fields_ = [
('domain_count', ctypes.c_uint32),
('PADDING_0', ctypes.c_ubyte * 4),
('domain_val', POINTER_T(ctypes.c_uint32)),
]
How to populate the domain_val in that struct ? Can I use a python list ?
I am thinking something along dom_val = c.create_string_buffer(c.sizeof(c.c_uint32) * domain_count) but then how to iterate through the buffer to populate or read the values ?
Will dom_val[0], dom_val[1] be able to iterate through the buffer with the correct length ? Maybe I need some typecast while iterating to write/read the correct number of bytes
Here's one way to go about it:
import ctypes as ct
class Domain(ct.Structure):
_fields_ = (('domain_count', ct.c_uint32),
('domain_val', ct.POINTER(ct.c_uint32)))
def __init__(self, data):
size = len(data)
# Create array of fixed size, initialized with the data
self.domain_val = (ct.c_uint32 * size)(*data)
self.domain_count = size
# Note you can slice the pointer to the correct length to retrieve the data.
def __repr__(self):
return f'Domain({self.domain_val[:self.domain_count]})'
x = Domain([1, 9, 34, 99])
print(x)
# Just like in C, you can iterate beyond the end
# of the array and create undefined behavior,
# so make sure to index only within the bounds of the array.
for i in range(x.domain_count):
print(x.domain_val[i])
Output:
Domain([1, 9, 34, 99])
1
9
34
99
To make it safer, you could add a property that casts the pointer to single element to a pointer to sized-array of elements so length checking happens:
import ctypes as ct
class Domain(ct.Structure):
_fields_ = (('_domain_count', ct.c_uint32),
('_domain_val', ct.POINTER(ct.c_uint32)))
def __init__(self,data):
size = len(data)
self._domain_val = (ct.c_uint32 * size)(*data)
self._domain_count = size
def __repr__(self):
return f'Domain({self._domain_val[:self._domain_count]})'
#property
def domain(self):
return ct.cast(self._domain_val, ct.POINTER(ct.c_uint32 * self._domain_count)).contents
x = Domain([1, 9, 34, 99])
print(x)
for i in x.domain: # now knows the size
print(i)
x.domain[2] = 44 # Can mutate the array,
print(x) # and it reflects in the data.
x.domain[4] = 5 # IndexError!
Output:
Domain([1, 9, 34, 99])
1
9
34
99
Domain([1, 9, 44, 99])
Traceback (most recent call last):
File "C:\demo\test.py", line 27, in <module>
x.domain[4] = 5
IndexError: invalid index
I am trying access public attributes on RPyC call by following this document but don't see it to be working as mentioned in document.
Documentation says if you don't specify protocol_config={'allow_public_attrs': True,} , public attributes , even of builtin data types won't be accessible. However even if we specify this, public attributes of nested data structure is not accessible ?
RPyC Server code.
import pickle
import rpyc
class MyService(rpyc.Service):
def on_connect(self, conn):
# code that runs when a connection is created
# (to init the service, if needed)
pass
def on_disconnect(self, conn):
# code that runs after the connection has already closed
# (to finalize the service, if needed)
pass
def exposed_get_answer(self): # this is an exposed method
return 42
exposed_the_real_answer_though = 43 # an exposed attribute
def get_question(self): # while this method is not exposed
return "what is the airspeed velocity of an unladen swallow?"
def exposed_hello(self, collection):
print ("Collection is ", collection)
print ("Collection type is ", type(collection).__name__)
for item in collection:
print ("Item type is ", type(item).__name__)
print(item)
def exposed_hello2(self, collection):
for item in collection:
for key, val in item.items():
print (key, val)
def exposed_hello_json(self, collection):
for item in collection:
item = json.loads(item)
for key, val in item.items():
print (key, val)
if __name__ == "__main__":
from rpyc.utils.server import ThreadedServer
t = ThreadedServer(
MyService(),
port=3655,
protocol_config={'allow_public_attrs': True,}
)
t.start()
Client Side Calls
>>> import rpyc
>>> rpyc.__version__
(4, 0, 2)
>>> c = rpyc.connect('a.b.c.d', 3655) ; client=c.root
# Case 1
If data is in nested structure (using builtin data types) , it doesn't work.
>>> data
[{'a': [1, 2], 'b': 'asa'}]
>>> client.hello2(data)
...
AttributeError: cannot access 'items'
========= Remote Traceback (2) =========
Traceback (most recent call last):
File "/root/lydian.egg/rpyc/core/protocol.py", line 329, in _dispatch_request
res = self._HANDLERS[handler](self, *args)
File "/root/lydian.egg/rpyc/core/protocol.py", line 590, in _handle_call
return obj(*args, **dict(kwargs))
File "sample.py", line 33, in exposed_hello2
for key, val in item.items():
File "/root/lydian.egg/rpyc/core/netref.py", line 159, in __getattr__
return syncreq(self, consts.HANDLE_GETATTR, name)
File "/root/lydian.egg/rpyc/core/netref.py", line 75, in syncreq
return conn.sync_request(handler, proxy, *args)
File "/root/lydian.egg/rpyc/core/protocol.py", line 471, in sync_request
return self.async_request(handler, *args, timeout=timeout).value
File "/root/lydian.egg/rpyc/core/async_.py", line 97, in value
raise self._obj
_get_exception_class.<locals>.Derived: cannot access 'items'
Case 2 : Workaround, Pass nested data as string using json (poor man's pickle) and decode at server end.
>>> jdata = [json.dumps({'a': [1,2], 'b':"asa"})].
>>> client.hello_json(jdata) # Prints following at remote endpoint.
a [1, 2]
b asa
Case 3 :
Interestingly, at first level builtin items are accessible as in case of
hello method. But calling that on nested data is giving error.
>>> client.hello([1,2,3,4]) # Prints following at remote endpoint.
Collection is [1, 2, 3, 4]
Collection type is list
Item type is int
1
Item type is int
2
Item type is int
3
Item type is int
4
I have workaround / solution to the problem (case 2 above) but looking for explanation on why is this not allowed or if it is a bug. Thanks for inputs.
The issue is not related to nested data.
Your problem is that you are not allowing public attributes in the client side.
The solution is simple:
c = rpyc.connect('a.b.c.d', 3655, config={'allow_public_attrs': True})
Keep in mind that rpyc is a symmetric protocol (see https://rpyc.readthedocs.io/en/latest/docs/services.html#decoupled-services).
In your case, the server tries to access the client's object, so allow_public_attrs must be set in the client side.
Actually for your specific example, there is no need to set allow_public_attrs in the server side.
Regarding case 3:
In the line for item in collection:, the server tries to access two fields: collection.__iter__ and collection.__next__.
Both of these fields are considered by default as "safe attributes", and this is why you didn't get error there.
To inspect the default configuration dictionary in rpyc:
>>> import rpyc
>>> rpyc.core.protocol.DEFAULT_CONFIG
d =dict(input('Enter a dictionary'))
sum = 0
for i in d.values():
sum +=i
print(sum)
outputs: Enter a dictionary{'a': 100, 'b':200, 'c':300}
this is the problem arises:
Traceback (most recent call last):
File "G:/DurgaSoftPython/smath.py", line 2, in <module>
d =dict(input('Enter a dictionary'))
ValueError: dictionary update sequence element #0 has length 1; 2 is required
You can't create a dict from a string using the dict constructor, but you can use ast.literal_eval:
from ast import literal_eval
d = literal_eval(input('Enter a dictionary'))
s = 0 # don't name your variable `sum` (which is a built-in Python function
# you could've used to solve this problem)
for i in d.values():
s +=i
print(s)
Output:
Enter a dictionary{'a': 100, 'b':200, 'c':300}
600
Using sum:
d = literal_eval(input('Enter a dictionary'))
s = sum(d.values())
print(s)
import json
inp = input('Enter a dictionary')
inp = dict(json.loads(inp))
sum = sum(inp.values())
print(sum)
input Enter a dictionary{"a": 100, "b":200, "c":300}
output 600
Actually the return of input function is a string. So, in order to have a valid python dict you need to evaluate the input string and convert it into dict.
One way to do this can be done using literal_eval from ast package.
Here is an example:
from ast import literal_eval as le
d = le(input('Enter a dictionary: '))
_sum = 0
for i in d.values():
_sum +=i
print(_sum)
Demo:
Enter a dictionary: {'a': 100, 'b':200, 'c':300}
600
PS: Another way can be done using eval but it's not recommended.
In a python project, my class has several properties that I need to be of specific type. Users of the class must have the ability to set the property.
What is the best way to do this? Two solutions come to my mind:
1. Have test routines in each setter function.
2. Use decorators for attributes
My current solution is 1 but I am not happy with it due to the code duplication. It looks like this:
class MyClass(object):
#property
def x(self):
return self._x
#x.setter
def x(self, val):
if not isinstance(self, int):
raise Exception("Value must be of type int")
self._x = val
#property
def y(self):
return self._y
#x.setter
def y(self, val):
if not isinstance(self, (tuple, set, list)):
raise Exception("Value must be of type tuple or set or list")
self._y = val
From what I know of decorators, it should be possible to have a decorator before def x(self) handle this job. Alas I fail miserably at this, as all examples I found (like this or this) are not targeted at what I want.
The first question is thus: Is it better to use a decorator to check property types? If yes, the next question is: What is wrong with below decorator (I want to be able write #accepts(int)?
def accepts(types):
"""Decorator to check types of property."""
def outer_wrapper(func):
def check_accepts(prop):
getter = prop.fget
if not isinstance(self[0], types):
msg = "Wrong type."
raise ValueError(msg)
return self
return check_accepts
return outer_wrapper
Appetizer
Callables
This is likely beyond your needs, since it sounds like you're dealing with end-user input, but I figured it may be helpful for others.
Callables include functions defined with def, built-in functions/methods such as open(), lambda expressions, callable classes, and many more. Obviously, if you only want to allow a certain type(s) of callables, you can still use isinstance() with types.FunctionType, types.BuiltinFunctionType, types.LambdaType, etc. But if this is not the case, the best solution to this that I am aware of is demonstrated by the MyDecoratedClass.z property using isinstance() with collections.abc.Callable. It's not perfect, and will return false positives in extraordinary cases (for example, if a class defines a __call__ function that doesn't actually make the class callable). The callable(obj) built-in is the only foolproof check function to my knowledge. The MyClass.z the use property demonstrates this function, but you'd have to write another/modify the existing decorator function in MyDecoratedClass in order to support the use of check functions other than isinstance().
Iterables (and Sequences and Sets)
The y property in the code you provided is supposed to be restricted to tuples, sets, and lists, so the following may be of some use to you.
Instead of checking if arguments are of individual types, you might want to consider using Iterable, Sequence, and Set from the collections.abc module. Please use caution though, as these types are far less restrictive than simply passing (tuple, set, list) as you have. abc.Iterable (as well as the others) work near-perfectly with isinstance(), although it does sometimes return false positives as well (e.g. a class defines an __iter__ function but doesn't actually return an iterator -- who hurt you?). The only foolproof method of determining whether or not an argument is iterable is by calling the iter(obj) built-in and letting it raise a TypeError if it's not iterable, which could work in your case. I don't know of any built-in alternatives to abc.Sequence and abc.Set, but almost every sequence/set object is also iterable as of Python 3, if that helps. The MyClass.y2 property implements iter() as a demonstration, however the decorator function in MyDecoratedClass does not (currently) support functions other than isinstance(); as such, MyDecoratedClass.y2 uses abc.Iterable instead.
For the completeness' sake, here is a quick comparison of their differences:
>>> from collections.abc import Iterable, Sequence, Set
>>> def test(x):
... print((isinstance(x, Iterable),
... isinstance(x, Sequence),
... isinstance(x, Set)))
...
>>> test(123) # int
False, False, False
>>> test("1, 2, 3") # str
True, True, False
>>> test([1, 2, 3]) # list
(True, True, False)
>>> test(range(3)) # range
(True, True, False)
>>> test((1, 2, 3)) # tuple
(True, True, False)
>>> test({1, 2, 3}) # set
(True, False, True)
>>> import numpy as np
>>> test(numpy.arange(3)) # numpy.ndarray
(True, False, False)
>>> test(zip([1, 2, 3],[4, 5, 6])) # zip
(True, False, False)
>>> test({1: 4, 2: 5, 3: 6}) # dict
(True, False, False)
>>> test({1: 4, 2: 5, 3: 6}.keys()) # dict_keys
(True, False, True)
>>> test({1: 4, 2: 5, 3: 6}.values()) # dict_values
(True, False, False)
>>> test({1: 4, 2: 5, 3: 6}.items()) # dict_items
(True, False, True)
Other Restrictions
Virtually all other argument type restrictions that I can think of must use hasattr(), which I'm not going to get into here.
Main Course
This is the part that actually answers your question. assert is definitely the simplest solution, but it has its limits.
class MyClass:
#property
def x(self):
return self._x
#x.setter
def x(self, val):
assert isinstance(val, int) # raises AssertionError if val is not of type 'int'
self._x = val
#property
def y(self):
return self._y
#y.setter
def y(self, val):
assert isinstance(val, (list, set, tuple)) # raises AssertionError if val is not of type 'list', 'set', or 'tuple'
self._y = val
#property
def y2(self):
return self._y2
#y2.setter
def y2(self, val):
iter(val) # raises TypeError if val is not iterable
self._y2 = val
#property
def z(self):
return self._z
#z.setter
def z(self, val):
assert callable(val) # raises AssertionError if val is not callable
self._z = val
def multi_arg_example_fn(self, a, b, c, d, e, f, g):
assert isinstance(a, int)
assert isinstance(b, int)
# let's say 'c' is unrestricted
assert isinstance(d, int)
assert isinstance(e, int)
assert isinstance(f, int)
assert isinstance(g, int)
this._a = a
this._b = b
this._c = c
this._d = d
this._e = e
this._f = f
this._g = g
return a + b * d - e // f + g
Pretty clean overall, besides the multi-argument function I threw in there at the end, demonstrating that asserts can get tedious. However, I'd argue that the biggest drawback here is the lack of Exception messages/variables. If the end-user sees an AssertionError, it has no message and is therefore mostly useless. If you write intermediate code that could except these errors, that code will have no variables/data to be able to explain to the user what went wrong. Enter the decorator function...
from collections.abc import Callable, Iterable
class MyDecoratedClass:
def isinstance_decorator(*classinfo_args, **classinfo_kwargs):
'''
Usage:
Always remember that each classinfo can be a type OR tuple of types.
If the decorated function takes, for example, two positional arguments...
* You only need to provide positional arguments up to the last positional argument that you want to restrict the type of. Take a look:
1. Restrict the type of only the first argument with '#isinstance_decorator(<classinfo_of_arg_1>)'
* Notice that a second positional argument is not required
* Although if you'd like to be explicit for clarity (in exchange for a small amount of efficiency), use '#isinstance_decorator(<classinfo_of_arg_1>, object)'
* Every object in Python must be of type 'object', so restricting the argument to type 'object' is equivalent to no restriction whatsoever
2. Restrict the types of both arguments with '#isinstance_decorator(<classinfo_of_arg_1>, <classinfo_of_arg_2>)'
3. Restrict the type of only the second argument with '#isinstance_decorator(object, <classinfo_of_arg_2>)'
* Every object in Python must be of type 'object', so restricting the argument to type 'object' is equivalent to no restriction whatsoever
Keyword arguments are simpler: #isinstance_decorator(<a_keyword> = <classinfo_of_the_kwarg>, <another_keyword> = <classinfo_of_the_other_kwarg>, ...etc)
* Remember that you only need to include the kwargs that you actually want to restrict the type of (no using 'object' as a keyword argument!)
* Using kwargs is probably more efficient than using example 3 above; I would avoid having to use 'object' as a positional argument as much as possible
Programming-Related Errors:
Raises IndexError if given more positional arguments than decorated function
Raises KeyError if given keyword argument that decorated function isn't expecting
Raises TypeError if given argument that is not of type 'type'
* Raised by 'isinstance()' when fed improper 2nd argument, like 'isinstance(foo, 123)'
* Virtually all UN-instantiated objects are of type 'type'
Examples:
example_instance = ExampleClass(*args)
# Neither 'example_instance' nor 'ExampleClass(*args)' is of type 'type', but 'ExampleClass' itself is
example_int = 100
# Neither 'example_int' nor '100' are of type 'type', but 'int' itself is
def example_fn: pass
# 'example_fn' is not of type 'type'.
print(type(example_fn).__name__) # function
print(type(isinstance).__name__) # builtin_function_or_method
# As you can see, there are also several types of callable objects
# If needed, you can retrieve most function/method/etc. types from the built-in 'types' module
Functional/Intended Errors:
Raises TypeError if a decorated function argument is not an instance of the type(s) specified by the corresponding decorator argument
'''
def isinstance_decorator_wrapper(old_fn):
def new_fn(self, *args, **kwargs):
for i in range(len(classinfo_args)):
classinfo = classinfo_args[i]
arg = args[i]
if not isinstance(arg, classinfo):
raise TypeError("%s() argument %s takes argument of type%s' but argument of type '%s' was given" %
(old_fn.__name__, i,
"s '" + "', '".join([x.__name__ for x in classinfo]) if isinstance(classinfo, tuple) else " '" + classinfo.__name__,
type(arg).__name__))
for k, classinfo in classinfo_kwargs.items():
kwarg = kwargs[k]
if not isinstance(kwarg, classinfo):
raise TypeError("%s() keyword argument '%s' takes argument of type%s' but argument of type '%s' was given" %
(old_fn.__name__, k,
"s '" + "', '".join([x.__name__ for x in classinfo]) if isinstance(classinfo, tuple) else " '" + classinfo.__name__,
type(kwarg).__name__))
return old_fn(self, *args, **kwargs)
return new_fn
return isinstance_decorator_wrapper
#property
def x(self):
return self._x
#x.setter
#isinstance_decorator(int)
def x(self, val):
self._x = val
#property
def y(self):
return self._y
#y.setter
#isinstance_decorator((list, set, tuple))
def y(self, val):
self._y = val
#property
def y2(self):
return self._y2
#y2.setter
#isinstance_decorator(Iterable)
def y2(self, val):
self._y2 = val
#property
def z(self):
return self._z
#z.setter
#isinstance_decorator(Callable)
def z(self, val):
self._z = val
#isinstance_decorator(int, int, e = int, f = int, g = int, d = (int, float, str))
def multi_arg_example_fn(self, a, b, c, d, e, f, g):
# Identical to assertions in MyClass.multi_arg_example_fn
self._a = a
self._b = b
self._c = c
self._d = d
return a + b * e - f // g
Clearly, multi_example_fn is one place where this decorator really shines. The clutter made by assertions has been reduced to a single line. Let's take a look at some example error messages:
>>> test = MyClass()
>>> dtest = MyDecoratedClass()
>>> test.x = 10
>>> dtest.x = 10
>>> print(test.x == dtest.x)
True
>>> test.x = 'Hello'
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 7, in x
AssertionError
>>> dtest.x = 'Hello'
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 100, in new_fn
TypeError: x() argument 0 takes argument of type 'int' but argument of type 'str' was given
>>> test.y = 1
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 15, in y
AssertionError
>>> test.y2 = 1
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 23, in y2
TypeError: 'int' object is not iterable
>>> dtest.y = 1
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 100, in new_fn
TypeError: y() argument 0 takes argument of types 'list', 'set', 'tuple' but argument of type 'int' was given
>>> dtest.y2 = 1
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 100, in new_fn
TypeError: y2() argument 0 takes argument of type 'Iterable' but argument of type 'int' was given
>>> test.z = open
>>> dtest.z = open
>>> test.z = None
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 31, in z
AssertionError
>>> dtest.z = None
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 100, in new_fn
TypeError: z() argument 0 takes argument of type 'Callable' but argument of type 'NoneType' was given
Far superior in my opinion. Everything looks good except...
>>> test.multi_arg_example_fn(9,4,[1,2],'hi', g=2,e=1,f=4)
11
>>> dtest.multi_arg_example_fn(9,4,[1,2],'hi', g=2,e=1,f=4)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 102, in new_fn
KeyError: 'd'
>>> print('I forgot that you have to merge args and kwargs in order for the decorator to work properly with both but I dont have time to fix it right now. Absolutely safe for properties for the time being though!')
I forgot that you have to merge args and kwargs in order for the decorator to work properly with both but I dont have time to fix it right now. Absolutely safe for properties for the time being though!
Edit Notice: My previous answer was completely incorrect. I was suggesting the use of type hints, forgetting that they aren't actually ensured in any way. They are strictly a development/IDE tool. They still are insanely helpful though; I recommend looking into using them.
I am getting a syntax error when trying to do the following MCVE in Python 3.
HEIGHT = 26
WIDTH = 26
OTHERVAR = 5
class Foo():
def __init__(self, OTHERVAR, HEIGHT*WIDTH):
print (str(OTHERVAR + HEIGHT*WIDTH))
foo_inst = Foo()
Below is the error
File "a.py", line 6
def __init__(self, OTHERVAR, HEIGHT*WIDTH):
^
SyntaxError: invalid syntax
I'm wondering why the multiplication * operator is invalid syntax in this scenario.
If someone could explain why this is bad syntax and offer a potential workaround, that would be great. Thank you.
A function parameter supposes to be a variable, your HEIGHT*WIDTH produces a value, not a variable.
Are you probably looking for this (default value)?
>>> a = 1
>>> b = 2
>>> def test(c=a*b):
... print(c)
...
>>> test()
2
>>> def test(c=a*b, d):
... print(c, d)
...
File "<stdin>", line 1
SyntaxError: non-default argument follows default argument
>>> def test(d, c=a*b):
... print(d, c)
...
>>> test(10)
(10, 2)
And called by named parameters
>>> def test(d, c=a*b, e=20):
... print(d, c, e)
...
>>> test(10, e=30)
(10, 2, 30)