I am attempting to write a function which will async create Indexes for each of my collections dynamically. The struggle I am having is accessing the class instance of each of my collections, and then calling the object to create the index.
I have tried:
getattr
ast.literal_eval
NOTE: I am unable to use eval() for security reasons
Example of how my class instance looks:
import ast
from models.database_models import IndexedKeyModels
class Mongo: # pylint: disable=too-many-instance-attributes
def __init__(self):
self.connection = self.get_set_connection() # Function to create MongoDB connection
self.db = self.connection.pal_db
self.test1 = self.db.test1
self.test2 = self.db.test2
async def set_indexes(self) -> None:
test_string: str = ""
if self.dev == "True":
test_string += "test_"
for name in IndexedKeyModels():
await ast.literal_eval(
f"self.{test_string}{name[0]}.create_index([('{name[1]}', {ASCENDING})], background=True)"
)
The IndexedKeyModels looks like this:
class IndexedKeyModels(BaseModel):
"""
MongoDB Indexed Key names
"""
test1: str = "Index1"
test2: str = "Index2"
I would like to be able to dynamically create each Index for each collection without having to define each create_index for each collection we have.
If you want to create an index on a specific collection and field, you can do this programatically with this construct:
collectionname = 'coll1'
fieldname = 'field1'
db[collectionname].create_index([(fieldname , pymongo.ASCENDING)])
Related
I have two factory classes, the other is linked to the one through foreign key relationships, and was kinda hoping to achieve some similarities with the attributes. To start with, the model looks something like this:
class Track(models.Model):
response = models.ForeignKey('Response')
def __str__(self):
return str(self.response)
class Response(models.Model):
title = models.CharField(max_length=640)
def __str__(self):
return self.title
I should be able to access these classes as I have done below
r = Response(title='foo')
r.save()
t = Track(response=r)
t.save()
# with this I wanted to test that str(t) == t.response
The factory classes look like this:
class ResponseFactory(factory.django.DjangoModelFactory):
class Meta:
model = Response
title = factory.Faker('text')
class TrackFactory(factory.django.DjangoModelFactory):
class Meta:
model = Track
response = factory.SubFactory(ResponseFactory)
Below is how I have accessed these factory classes to test for str equality
track = TrackFactory() # generates random string e.g `foo`
a = str(track) # == foo
b = track.response # == foo
# however I get an assertion error with the statement below
assert a == b
Could you point out where I've gone wrong, thank you.
How can I share variables across different modules of my Python project if I need these variables to have setter and getter methods.
The reason I need setter\getter methods is because when getting and setting the variables I need to have backwards compatibility with code that stored these variable as environment variables. So I need to write and read using os.environ too.
Usually all I need to do is create a class with class-level variables, import the class in each Module and access the module as follows:
datastore.py/
class DataStore:
target_server_ip: str = '10.10.10.100'
consumer.py/
from project.datastore import DataStore
def print_target_server_ip():
print(DataStore.target_server_ip)
This doesn't work (at least not in Python 3.6.5) if the variables require property getter and setter methods.
The reason is that I cannot define a class level method as a property. The following code just isn't possible:
datastore.py/
class DataStore:
target_server_ip: str = '10.10.10.100'
#classmethod
#property
def target_server_ip(cls):
return cls.target_server_ip
#classmethod
#target_server_ip.setter
def target_server_ip(cls, value):
cls.target_server_ip = value
To solve this issue I propose the following code section. It is split into two classes.
The first class is working at the class level and maintains a 2 level nested dictionary that contains the name of the datastore and the variable name.
The second class is the datastore itself. It has the minimum required code to keep it visually simple.
This specific implementation has one known error prone limitation. If you declare two or more variables with the same name in different datastore classes, i.d. you define class FrameworkDatastore and another class SecondDatastore with the same variable in both, the environment will have only one of them.
import inspect
import logging
import os
from typing import Any, Dict, Type
logger = logging.getLogger(__name__)
class _BaseDataStoreWithEnvironSupport:
"""
The class support global storing of variables in a class level dictionary, allowing all instances of the
datastore to access the same values.
This class is backward compatible to store the global variables as os.environ, but also
"""
_members: Dict[str, Dict[str, Any]] = {} # holds all the members of the datastore
#classmethod
def get_value(cls) -> Any:
datastore_name: str = cls.__name__
member_name: str = inspect.stack()[1][3]
env_value: str = os.environ.get(member_name)
ds_value: Any = cls._members[datastore_name][member_name]
if env_value:
type_ds_value: Type = type(ds_value)
if type_ds_value is bool:
value: bool = (env_value == True.__str__())
else:
value: Any = type(ds_value)(env_value)
if value != ds_value:
logger.warning('Environment stored value is different from Datastore value. Check your implementation')
else:
value: Any = ds_value
return value
#classmethod
def set_value(cls, value: Any) -> None:
datastore_name: str = cls.__name__
name: str = inspect.stack()[1][3]
if datastore_name not in cls._members.keys():
cls._members[datastore_name] = {}
cls._members[datastore_name][name] = value
os.environ[name] = str(value)
def validate_datastore(self):
members = set([attr for attr in dir(self) if not callable(getattr(self, attr)) and not attr.startswith("_")])
if members.__len__() == 0:
raise RuntimeError(f'There are no members in the datastore or the validation runs at the start of __init__')
datastore_name: str = self.__class__.__name__
dict_keys: set = set(self._members[datastore_name].keys())
if members != dict_keys:
missing_members: set = members - dict_keys
raise NotImplementedError(f'Datastore is missing get and set methods for members: {missing_members}')
class FrameworkDatastore(_BaseDataStoreWithEnvironSupport):
"""
This class is storing all variables that are currently saved as global or os.environ variables
If the data stored here becomes irrelevant after the code change or is seldom used, remove it and merge its
functionality into other sections
"""
def __init__(self):
"""
predefine all the members of the datastore.
Members which dont implement get/set methods will be flagged by the validate_datastore check
"""
self.run_traffic_validations: bool = True # Should Ixia traffic validations run in the current suite
# The validation of the datastore must come at the end of the __init__ method
self.validate_datastore()
#property
def run_traffic_validations(self):
return self.get_value()
#run_traffic_validations.setter
def run_traffic_validations(self, value: Any):
self.set_value(value)
if __name__ == '__main__':
# This tests the datastore code
fd1 = FrameworkDatastore()
fd2 = FrameworkDatastore()
print(fd1.run_traffic_validations)
print(fd2.run_traffic_validations)
fd1.run_traffic_validations = False
print(fd1.run_traffic_validations)
print(fd2.run_traffic_validations)
fd2.run_traffic_validations = True
print(fd1.run_traffic_validations)
print(fd2.run_traffic_validations)
I'm writing a class to manage a collection of objects that I'd like to "load" only if actually used (immagine that each object is an heavy document). Also I want to refer to each object both with a numeric key and a string name.
I decided to create a class that inherits from OrderedDict:
from collections import OrderedDict
class MyClass:
def load_me(self, key):
print(f"Object {key} loaded")
class MyClassColl(OrderedDict):
def __getitem__(self, key):
if isinstance(key, int):
key = list(self.keys())[key]
res = super().get(key).load_me(key)
return res
When I initialise the collection and retrieve a single object everything works well and:
my_coll = MyClassColl([('Obj1', MyClass()), ('Obj2', MyClass()), ('Obj3', MyClass())])
my_obj = my_coll['Obj2'] # or my_obj = my_coll[1]
prints:
Object Obj2 loaded
But using a loop, the objects are not properly loaded so:
for key, item in my_coll.items():
obj = item
has not output.
This is because the __getitem__ method is not getting called when you loop through the dictionary like that. It is only called when you use an index operator (as far as I know). So, a super easy fix would be to do your for loop like this:
for key in my_coll:
item = my_coll[key]
Alternatively you could try playing around with the __iter__ method but I think the way you've done it is probably ideal.
Hi I am writing unittest using pytest. But I am not able to mock few db functions. We are using psycopg2 for db connections and executions. Response of query returned from psycopg2 is of the type DictRow which can be accessed either by key or by index.
Ex:
response = ['prajwal', '23', 'engineer'] #Response of a query "select name, age , job from users"
>>>response[0]
'prajwal'
>>>response['name']
'prajwal'
I want to know is there any way by which we can covert dict/list to above mentioned type.
Looking at the source for psycopg2, creating a DictRow requires passing in a DictCursor object. However the only thing it uses from DictCursor appears to be an index and description attribute.
# found in lib\site-packages\psycopg2.extras.py
class DictRow(list):
"""A row object that allow by-column-name access to data."""
__slots__ = ('_index',)
def __init__(self, cursor):
self._index = cursor.index
self[:] = [None] * len(cursor.description)
The index looks like a dict with a mapping a key to an index. e.g.response['name'] = 0
The description looks like your dict that you want to convert.
If you're feeling hacky you could take advantage of duck typing and pretend you're passing in a cursor when you're just satisfying the requirements.
The only caveat is after we instantiate the DictRow, we need to populate it. Our fake cursor hack will take care of the rest.
from psycopg2.extras import DictRow
class DictRowHack:
def __init__(self, my_dict):
# we need to set these 2 attributes so that
# it auto populates our indexes
self.index = {key: i for i, key in enumerate(my_dict)}
self.description = my_dict
def dictrow_from_dict(my_dict):
# this is just a little helper function
# so you don't always need to go through
# the steps to recreate a DictRow
fake_cursor = DictRowHack(my_dict)
my_dictrow = DictRow(fake_cursor)
for k, v in my_dict.items():
my_dictrow[k] = v
return my_dictrow
response = {'name': 'prajwal', 'age': '23', 'job': 'engineer'}
my_dictrow = dictrow_from_dict(response)
print(my_dictrow[1])
print(my_dictrow['name'])
print(type(my_dictrow))
I am trying to Link a flask server to a Peewee database. I have a Rest GET request that passes data of the form
{'a':1,'b':2, 'filter':{'name':'Foo', 'count':3}}
I want to write a method that converts my filters into a database query and execute it to return their resource:
import datetime
import peewee as pw
import uuid
DATABASE = "Resources.db"
database = pw.SqliteDatabase(DATABASE)
class BaseModel(pw.Model):
class Meta:
database = database
class Resource(BaseModel):
name = pw.CharField(unique=True)
current_count = pw.IntegerField(default=1)
total_count = pw.IntegerField(default=1)
flavor = pw.CharField(default="undefined")
users = pw.TextField()
metadata = pw.TextField(default="")
is_avalible = pw.BooleanField(default=True)
uuid = pw.UUIDField(primary_key=True, default=uuid.uuid4)
max_reservation_time = pw.IntegerField(default=10)
def __str__(self):
return f"My name is {self.name} {vars(self)}"
This is kinda what my resource looks like. Here is what I am trying to do... (not a working full example)
def filter(filters):
for i,j in filters.items():
dq = Resource.select().where(getattr(Resource, i) == j)
for resource in dq:
print(resource)
if __name__ == "__main__":
try:
database.connect()
except pw.OperationalError:
print("Open Connection")
try:
create_tables()
except pw.OperationalError:
print("Resource table already exists!")
with database.atomic():
reso = Resource.create(name="Burns", current_count=4, total_count=5, users="Bar", blah=2)
filter({'name':"Burns","total_count":5})
Here I would expect to get back: My name is Burns {'__data__': {'uuid': UUID('80808e3a-4b10-47a5-9d4f-ff9ff9ca6f5c'), 'name': 'Burns', 'current_count': 4, 'total_count': 5, 'flavor': 'undefined', 'users': 'Grant', 'metadata': '', 'is_avalible': True, 'max_reservation_time': 10}, '_dirty': set(), '__rel__': {}}I believe I might be able to create individual peewee.expressions and join them some how, I just am not sure how.
Since peewee expressions can be arbitrarily combined using the builtin & and | operators, we'll use the reduce() function to combine the list using the given operand:
def filter(filters):
expression_list = [getattr(Resource, field) == value
for field, value in filters.items()]
# To combine all expressions with "AND":
anded_expr = reduce(operator.and_, expression_list)
# To combine all expressions with "OR":
ored_expr = reduce(operator.or_, expression_list)
# Then:
return Resource.select().where(anded_expr) # or, ored_expr
Thanks to #coleifer for the reminder. Here was my solution:
OP_MAP = {
"==": pw.OP.EQ,
"!=": pw.OP.NE,
">": pw.OP.GT,
"<": pw.OP.LT,
">=": pw.OP.GTE,
"<=": pw.OP.LTE,
}
def _generate_expressions(model, query_filter):
expressions = []
for expression in query_filter:
expressions.append(
pw.Expression(
getattr(model, expression["attr"]), OP_MAP[expression["op"]], expression["value"]
)
)
return expressions
def generate_query(model, query_data):
if query_data.get("filters") is None:
database_query = model.select()
else:
database_query = model.select().where(
*(_generate_expressions(model, query_data["filters"]))
)
return database_query
I pass the type of object I want to create an expression for and operator in the filter data. Iterating over the filters I can build the expressions and combine them.