Goal:
I want a scalable way to add factories that represent my models so I can manipulate them at run time and have a test DB from test set up to tear down.
Issue:
cls = <class 'tests.factories.analysis.Analysis'>
model_class = <class 'src.api.someModel'>, session = <function dbsession at 0xffffb8e6a5e0>, args = ()
kwargs = {'analysis_id': 365, 'created_by': 619, 'file_extension': 'Stop.', 'original_file_name': 'Share.', ...}
#classmethod
def _save(cls, model_class, session, args, kwargs):
session_persistence = cls._meta.sqlalchemy_session_persistence
obj = model_class(*args, **kwargs)
> session.add(obj)
E AttributeError: 'function' object has no attribute 'add'
What this means to me ^ my Session object is not behaving the way it should, why i dont know.
conftest.py ( got this code snippet from here https://gist.github.com/kissgyorgy/e2365f25a213de44b9a2?permalink_comment_id=3496042 )
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from domain.declarative import Base
from sqlalchemy.orm import sessionmaker
TEST_DATABASE = "postgresql://postgres:postgres#localhost:5432/testDB"
#pytest.fixture(scope="session")
def engine():
return create_engine(TEST_DATABASE)
#pytest.fixture(scope="session")
def tables(engine):
Base.metadata.create_all(engine)
yield
Base.metadata.drop_all(engine)
#pytest.fixture
def dbsession(engine, tables):
"""Returns an sqlalchemy session, and after the test tears down everything properly."""
connection = engine.connect()
# begin the nested transaction
transaction = connection.begin()
# use the connection with the already started transaction
session = Session(bind=connection)
yield session
session.close()
# roll back the broader transaction
transaction.rollback()
# release connection
connection.close()
Base_factory.py
import factory.alchemy
from sqlalchemy.orm import scoped_session, sessionmaker
from the.code.above import engine, dbsession
session = scoped_session(sessionmaker(bind=engine))
class BaseFactory(factory.alchemy.SQLAlchemyModelFactory):
class Meta:
abstract = True
sqlalchemy_session = dbsession
sqlalchemy_session_persistence = "commit"
Child Factory
import random
from datetime import datetime
from faker import Faker
from domain.declarative import TheModel
from tests.factories.base_factory import BaseFactory
faker = Faker()
class ChildFactory(BaseFactory):
class Meta:
model = TheModel
analysis_id = random.randrange(1, 1000)
created_by = random.randrange(1, 1000)
file_extension = faker.text(max_nb_chars=10)
original_file_name = faker.text(max_nb_chars=10)
upload_date = faker.date_between_dates(
date_start=datetime(2000, 1, 1), date_end=datetime(2019, 12, 31)
)
some test
class TestSimple(unittest.TestCase):
def test_important_stuff(self):
f = ChildFactory()
So this looks like to me the Session object is not being instantiated properly and is of a type function? it's calling add here at somepoint and we get an atrribute error because of that where am I going wrong?
session.add(obj)
E AttributeError: 'function' object has no attribute 'add'
Related
Please help.
I try to use SQLAlchemy with dataclasses.
Here is the code.
from dataclasses import fields
from sqlalchemy import Column
from sqlalchemy import Integer, Boolean, String
from sqlalchemy import MetaData
from sqlalchemy import Table
from sqlalchemy.orm import registry
from models import Project, Model, Test
TABLES: dict = {}
#dataclass
class Test:
identifier: int = field(init=False)
name: str = None
def create_table(
sql_mapper_registry: registry, sql_metadata: MetaData, model: type[Test]
) -> None:
table_name: str = model.__name__.lower() + "_table"
TABLES[table_name] = Table(table_name, sql_metadata)
for i in range(len(fields(model))):
if fields(model)[i].type == type(bool()):
type_var = Boolean
elif fields(model)[i].type == type(int()):
type_var = Integer
elif fields(model)[i].type == type(str()):
type_var = String(60)
else:
raise TypeError
TABLES[table_name].append_column(
Column(
fields(model)[i].name,
type_var,
primary_key=(fields(model)[i].name == "identifier"),
)
)
sql_mapper_registry.map_imperatively(
model,
TABLES[table_name],
)
return None
if __name__ == '__main__':
engine = create_engine("mariadb+mariadbconnector://root:Password#127.0.0.1:3306/company")
metadata_obj.create_all(engine)
session = orm.sessionmaker(bind=engine)()
create_table(mapper_registry, metadata_obj, Test)
test = Test()
session.add(test)
session.commit()
But this doen't work and i don`t know why.
I've got this error:
sqlalchemy.exc.OperationalError: (mariadb.OperationalError)
Unknown prepared statement handler (4294967295) given to mysqld_stmt_execute
[SQL: INSERT INTO test_table (name) VALUES (?)]
[parameters: (None,)]
I want to update the table with data of class Test.
I have an old question looking for a fresh answer. I've tried the recipes presented in the similar but somewhat aged question "Start a flask application in a seperate thread", and some other similar solutions found in other posts.
The long and short of it is, I need to start a flask application in a 'background' thread, such that a wxPython GUI can run in the foreground. The solutions presented here seem to no longer have the desired effect. The flask app starts and the GUI never runs.
My suspicion is, the existing answers are out of date. That said, I'm open to the possibility that I've mangled something else that's hosing it up, please have a peek and advise accordingly.
Thanks for your eyeballs and brain cycles :)
My code follows.
#!/usr/bin/env python
"""
integrator.py (the app)
"""
import wx
from pubsub import pub
from flask import Flask
from flask_graphql import GraphQLView
from models import db_session
from schema import schema
from models import engine, db_session, Base, Idiom
flaskapp = Flask(__name__)
flaskapp.debug = True
flaskapp.add_url_rule(
'/graphql',
view_func=GraphQLView.as_view(
'graphql',
schema=schema,
graphiql=True
)
)
flaskapp.run(threaded=True,use_reloader=False)
#flaskapp.teardown_appcontext
def shutdown_session(exception=None):
db_session.remove()
class IntegratorTarget(wx.TextDropTarget):
def __init__(self, object):
wx.DropTarget.__init__(self)
self.object = object
def OnDropText(self, x, y, data):
print(f"<publish>{data}</publish>")
pub.sendMessage('default', arg1=data)
return True
class IntegratorFrame(wx.Frame):
def __init__(self, parent, title):
super(IntegratorFrame, self).__init__(parent, title = title,size = wx.DisplaySize())
self.panel = wx.Panel(self)
box = wx.BoxSizer(wx.HORIZONTAL)
dropTarget = IntegratorTarget(self.panel)
self.panel.SetDropTarget(dropTarget)
pub.subscribe(self.catcher, 'default')
self.panel.SetSizer(box)
self.panel.Fit()
self.Centre()
self.Show(True)
def catcher(self,arg1):
data = arg1
print(f"<subscribed>{data}</subscribed>\n\n")
return
ex = wx.App()
Base.metadata.create_all(bind=engine)
IntegratorFrame(None,'Praxis:Integrator')
ex.MainLoop()
-- eof --
""" models.py """
from sqlalchemy import *
from sqlalchemy.orm import (scoped_session, sessionmaker, relationship, backref)
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine('sqlite:///.praxis/lexicon/unbound.db3', convert_unicode=True)
db_session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
# We will need this for querying
Base.query = db_session.query_property()
class Idiom(Base):
__tablename__ = "idiomae"
id = Column(Integer, primary_key=True)
src = Column(String) # the text of the drag/paste operation
taxonomy = Column(String) # the type of resource referenced in the drag/paste operation
localblob = Column(String) # local path to media referenced in 'src'
timestamp = Column(DateTime) # date and time of capture
-- eof --
""" schema.py """
import graphene
from graphene import relay
from graphene_sqlalchemy import SQLAlchemyObjectType, SQLAlchemyConnectionField
from models import db_session, Idiom as IdiomaticModel
class Idiom(SQLAlchemyObjectType):
class Meta:
model = IdiomaticModel
interfaces = (relay.Node, )
class Query(graphene.ObjectType):
node = relay.Node.Field()
# Allows sorting over multiple columns, by default over the primary key
all_idioms = SQLAlchemyConnectionField(Idiom.connection)
# Disable sorting over this field
# all_departments = SQLAlchemyConnectionField(Department.connection, sort=None)
schema = graphene.Schema(query=Query)
I see where you tell Flask to be multi-threaded, but I don't see where you're starting up the Flask app in a thread. I expected to see something like
app = Flask(__name__)
# any extra configuration
def webserver():
app.run(use_reloader=False)
web_thread = threading.Thread(target=webserver)
web_thread.start()
... continue on with the main thread
I have a working example you can crib from here. Note the need to use appropriate locking of any data structures shared between the primary thread and the threads running Flask.
I have used flask-sqlalchemy to create a mixin in a file called itemAbstract.py, to be shared by two model classes: ItemModel and ItemHistoryModelrespectively. Below is the code I have written in the itemAbstract.py
from databaseHandler import databaseHandler
from sqlalchemy.ext.declarative import declared_attr
# pylint: disable=maybe-no-member
class Item(databaseHandler.Model):
__abstract__ = True
itemName = databaseHandler.Column(databaseHandler.String(80), nullable = False)
price = databaseHandler.Column(databaseHandler.Numeric, nullable = False)
itemImage = databaseHandler.Column(databaseHandler.String(1000), nullable = False)
#classmethod
#declared_attr
def restaurantId(cls):
return databaseHandler.Column(
databaseHandler.Integer, databaseHandler.ForeignKey("restaurant.restaurantId"))
#classmethod
#declared_attr
def restaurant(cls):
return databaseHandler.relationship(
"RestaurantModel", backref=databaseHandler.backref('items', lazy=True))
#classmethod
#declared_attr
def productTypeId(cls):
return databaseHandler.Column(
databaseHandler.Integer, databaseHandler.ForeignKey("product_type.productTypeId"))
#classmethod
#declared_attr
def productType(cls):
return databaseHandler.relationship(
"ProductTypeModel", backref=databaseHandler.backref('items', lazy=True))
And I have inherited it in the itemModel.py and itemHistoryModel.py like so:
from databaseHandler import databaseHandler
from sqlalchemy import and_, or_
from abstracts.itemAbstract import Item
# pylint: disable=maybe-no-member
class ItemModel(Item):
__tablename__ = 'item'
itemId = databaseHandler.Column(databaseHandler.Integer, primary_key = True)
And
from databaseHandler import databaseHandler
from sqlalchemy import and_, or_
from abstracts.itemAbstract import Item
# pylint: disable=maybe-no-member
class ItemHistoryModel(Item):
__tablename__ = 'item_history'
historyId = databaseHandler.Column(databaseHandler.Integer, primary_key = True)
I have a class method in both files that is supposed to help me get a list of items a restaurant sells by passing in the restaurantId as parameter
#classmethod
def findItemsByRestaurant(cls, param):
return cls.query.filter_by(restaurantId = param)
However, anytime I execute this method it returns a query string in the resultset instead of a list of items. Here is a sample resultset:
SELECT item_history.`itemName` AS `item_history_itemName`, item_history.price AS item_history_price, item_history.`itemImage` AS `item_history_itemImage`, item_history.`historyId` AS `item_history_historyId`
FROM item_history
WHERE false = 1
Somehow, SQLAlchemy makes my parameter false and assigns a value of 1 to it meanwhile the actual ID of the restaurant is 10. What am I doing wrong?
This is the databaseHandler.py file:
from flask_sqlalchemy import SQLAlchemy
databaseHandler = SQLAlchemy()
The Query object has a number of API methods for getting pythonic objects rather than amending the query:
get
all
from_statement
first
one_or_none
one
scalar (as_scalar to be depreciated)
count
I have a model Prescription.
from datetime import timedelta
from sqlalchemy.ext.hybrid import hybrid_property
class Prescription(db.Model):
""" docstring """
ID = db.column(db.Integer, primary_key=True)
date = db.Column(db.DateTime)
duration = db.Column(db.SmallInteger)
#hybrid_property
def expiration_date(self):
# return self.date + timedelta(days=self.duration)
return self.date + timedelta(days=7)
#classmethod
def actual(cls ):
""" docstring """
today = datetime.today()
return cls.query.filter(Prescription.expiration_date>=today)
I want to get only actual prescriptions in my actual method, and when I specify
#hybrid_property
def expiration_date(self):
return self.date + timedelta(days=7)
everything works like a charm.
But every prescription has a different duration, and when I specify
#hybrid_property
def expiration_date(self):
return self.date + timedelta(days=self.duration)
I've got an error
TypeError: unsupported type for timedelta days component: InstrumentedAttribute
I tried to make a little hack like this
#property
def days(self):
return int(self.duration)
but no luck.
Can anyone tell some workaround, or creating some lazy object for duration field or maybe another way get actual prescriptions, filtering by calculated expiration_date?
You might be trying to calculate a DATEDIFF:
Calculate DATEDIFF in POSTGRES using SQLAlchemy
Here, is_expired will generate a SQL query part which calculates difference in days between the start date and utcnow(), and compares the result with self.duration
This works in PostgreSQL, but I have not tested on other RDBMS.
from datetime import datetime
import sqlalchemy as sa
class Prescription:
ID = db.column(db.Integer, primary_key=True)
date = db.Column(db.DateTime)
duration = db.Column(db.SmallInteger)
#hybrid_property
def is_expired(self):
days_since_published = sa.func.trunc((
sa.extract('epoch', datetime.utcnow()) -
sa.extract('epoch', self.date)
) / 3600 / 24)
return days_since_published >= self.duration
#classmethod
def active(cls):
return cls.query.filter(is_expired=False)
In your #classmethod annotated actual() method, you are trying to access a non-static property(expiration_date) which is raising the error.
As per my understanding of what your code is doing, I have changed the code as follow:
from datetime import datetime,timedelta
from sqlalchemy.ext.hybrid import hybrid_property
from flask_sqlalchemy import SQLAlchemy
from flask import Flask
app = Flask(__name__)
db = SQLAlchemy(app)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SQLALCHEMY_DATABASE_URI'] ='sqlite:///'
class Prescription(db.Model):
""" docstring """
def mydefault(context):
return context.get_current_parameters()['date'] + timedelta(days=context.get_current_parameters()['duration'])
ID = db.Column(db.Integer,primary_key= True)
date = db.Column(db.Date)
duration = db.Column(db.SmallInteger)
expiration_date = db.Column(db.Date,default = mydefault)
#classmethod
def actual(cls ):
""" docstring """
today = datetime.today().date()
return cls.query.filter(Prescription.expiration_date>=today).all()
def __repr__(self):
return "Id:{} , date:{}, duration:{}, expiration_date:{}".format(self.ID,self.date,self.duration,self.expiration_date)
db.create_all()
q=Prescription(ID=1,date=datetime(2019,5,26).date(),duration = 1) #Expired
r=Prescription(ID=2,date=datetime(2019,5,20).date(),duration = 3) #Expired
s=Prescription(ID=3,date=datetime(2019,5,27).date(),duration = 5) #Not Expired
t = Prescription(ID=4,date=datetime.now().date(),duration = 1) #Not Expired
db.session.add(q)
db.session.add(r)
db.session.add(s)
db.session.add(t)
db.session.commit()
list_obj = Prescription.query.all()
print("All Objects in DB:-")
for l in list_obj:
print(l)
print()
print("Valid Prescription:")
print(Prescription.actual())
Output:
One thing I can suggest to you is since the model does not need the time of expiration.
So you could change
date = db.Column(db.DateTime)
to
date = db.Column(db.Date)
and it will ease your task :)
I am trying to execute multi-process to pull the data from Cassandra. But, I'm facing the issue.I want to pull it for single key or multiple keys using the multi-process provided my Cassandra
My cassandra_db class
from cassandra.cluster import Cluster
import cassandra
import pandas as pd
import numpy as np
from datetime import datetime
import sys
import os
from threading import Event
import itertools
from multiprocessing import Pool
from cassandra.concurrent import execute_concurrent_with_args
from cassandra.query import tuple_factory
ip_address = '127.0.0.1'
class cassandra_db(object):
concurrency = 2 # chosen to match the default in execute_concurrent_with_args
def __init__(self,process_count=None):
self.pool = Pool(processes=process_count, initializer=self._setup)
#classmethod
def _setup(cls):
cls.session = Cluster([ip_address]).connect(keyspace='test')
cls.session.row_factory = pandas_factory
cls.prepared = cls.session.prepare('SELECT * FROM tr_test WHERE key=?')
def close_pool(self):
self.pool.close()
self.pool.join()
def get_results(self, params):
try:
xrange
except NameError:
xrange = range
params = list(params)
print("-----> ",params)
print("-----+>",self.concurrency)
self.pool.map(_multiprocess_get, (params[n:n + self.concurrency] for n in xrange(0, len(params), self.concurrency)))
#classmethod
def _results_from_concurrent(cls, params):
return execute_concurrent_with_args(cls.session, cls.prepared, params)
def _multiprocess_get(params):
return cassandra_db._results_from_concurrent(params)
My calling class
import os
import pandas as pd
import sys
relative_path='/home/anji'
sys.path.append(os.path.join(relative_path ,'commons','Database Operations'))
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
from cassandra_db import cassandra_db
from cassandra.policies import ConstantReconnectionPolicy
processes =2
con_db = cassandra_db(processes)
keys=[(1,),(2,)]
df = con_db.get_results(keys)
print("Result",df.head())
Error:
multiprocessing.pool.MaybeEncodingError: Error sending result: '[[ExecutionResult(success=True, result_or_exc=<cassandra.cluster.ResultSet object at 0x7fa93658bbe0>), ExecutionResult(success=True, result_or_exc=<cassandra.cluster.ResultSet object at 0x7fa936a2e0f0>)]]'. Reason: 'PicklingError("Can't pickle <class 'importlib._bootstrap.ExecutionResult'>: attribute lookup ExecutionResult on importlib._bootstrap failed",)'
My trying to execute for 2 keys but facing the issue. Can any help me to solve this issue