I want to use python-socketio and I want to query my database from socketio methods. My db settings are saved in the pyramid request. But I don't understand how to get these settings without http-requests.
#sio.event
def connect(sid, environ):
print('connect ', sid)
#sio.event
def message(sid, data):
# I want to query my database at this location
sio.send(data)
print('Server send', data)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
I was thinking of using ' pyramid.threadlocal.get_current_request ' but this method returns None.
I add database settings to the pyramid request by this code.
__init__.py
def db(request):
session = session_maker(request)
return session
...
config.add_request_method(db, reify=True)
session_maker
def session_returner(request, connect_line):
engine = create_engine(connect_line, echo=debug)
Base.metadata.bind = engine
Session = orm.sessionmaker(bind=engine)
session = Session()
def cleanup(request):
if request.exception is not None:
session.rollback()
session.close()
request.add_finished_callback(cleanup)
return session
def session_maker(request, settings=None):
if settings is None:
settings = get_settings(request)
connect_line = 'postgresql://{user}:{password}#{postgre_server}:{bd_port}/{bd_name}'.format(user=settings['bd_user'], postgre_server=settings['postgre_server'], bd_port=settings['bd_port'], password=settings['bd_password'], bd_name=settings['bd_name'])
return session_returner(request, connect_line)
You can make a request object, but obviously there isn't one by default when not serving a request. This can be done via:
request = pyramid.request.Request.blank('/')
request.registry = registry
pyramid.request.apply_request_extensions(request)
request.db.query(...)
This assumes you have access to the registry which was defined by the wsgi app. Something like config.registry or app = config.make_wsgi_app(); registry = app.registry or request.registry from another request.
Related
I'm trying to implement an async RPC client within a Flask server.
The idea is that each request spawn a thread with an uuid, and each request is going to wait until there is a response in the RpcClient queue attribute object with the correct uuid.
The problem is that one request out of two fails. I think that might be a problem with multi-threading, but I don't see where it comes from.
Bug can be seen here.
Using debug print, it seems that the message with the correct uuid is received in the _on_response callback and update the queue attribute in this instance correctly, but the queue attribute within the /rpc_call/<payload> endpoint doesn't synchronize (so queue[uuid] has a value of response in the RpcClient callback but still None in the scope of the endpoint).
My code:
from flask import Flask, jsonif
from gevent.pywsgi import WSGIServer
import sys
import os
import pika
import uuid
import time
import threading
class RpcClient(object):
"""Asynchronous Rpc client."""
internal_lock = threading.Lock()
queue = {}
def __init__(self):
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host='rabbitmq'))
self.channel = self.connection.channel()
self.channel.basic_qos(prefetch_count=1)
self.channel.exchange_declare(exchange='kaldi_expe', exchange_type='topic')
# Create all the queue and bind them to the corresponding routing key
self.channel.queue_declare('request', durable=True)
result = self.channel.queue_declare('answer', durable=True)
self.channel.queue_bind(exchange='kaldi_expe', queue='request', routing_key='kaldi_expe.web.request')
self.channel.queue_bind(exchange='kaldi_expe', queue='answer', routing_key='kaldi_expe.kaldi.answer')
self.callback_queue = result.method.queue
.
thread = threading.Thread(target=self._process_data_events)
thread.setDaemon(True)
thread.start()
def _process_data_events(self):
self.channel.basic_consume(self.callback_queue, self._on_response, auto_ack=True)
while True:
with self.internal_lock:
self.connection.process_data_events()
time.sleep(0.1)
def _on_response(self, ch, method, props, body):
"""On response we simply store the result in a local dictionary."""
self.queue[props.correlation_id] = body
def send_request(self, payload):
corr_id = str(uuid.uuid4())
self.queue[corr_id] = None
with self.internal_lock:
self.channel.basic_publish(exchange='kaldi_expe',
routing_key="kaldi_expe.web.request",
properties=pika.BasicProperties(
reply_to=self.callback_queue,
correlation_id=corr_id,
),
body=payload)
return corr_id
def flask_app():
app = Flask("kaldi")
#app.route('/', methods=['GET'])
def server_is_up():
return 'server is up', 200
#app.route('/rpc_call/<payload>')
def rpc_call(payload):
"""Simple Flask implementation for making asynchronous Rpc calls. """
corr_id = app.config['RPCclient'].send_request(payload)
while app.config['RPCclient'].queue[corr_id] is None:
#print("queue server: " + str(app.config['RPCclient'].queue))
time.sleep(0.1)
return app.config['RPCclient'].queue[corr_id]
if __name__ == '__main__':
while True:
try:
rpcClient = RpcClient()
app = flask_app()
app.config['RPCclient'] = rpcClient
print("Rabbit MQ is connected, starting server", file=sys.stderr)
app.run(debug=True, threaded=True, host='0.0.0.0')
except pika.exceptions.AMQPConnectionError as e:
print("Waiting for RabbitMq startup" + str(e), file=sys.stderr)
time.sleep(1)
except Exception as e:
worker.log.error(e)
exit(e)
I found where the bug came from:
Thedebug=True of the line app.run(debug=True, threaded=True, host='0.0.0.0') restart the server at the beginning.
The whole script is then restarted from the beginning. Because of it, another rpcClient is initialized and consume from the same queue. Problem is that the previous thread is also running. This cause two rpcClient to consume from the same thread, with one that is virtually useless.
I want to terminate rest request coming to server without further processing if input params are missing.
Currently this is the implementation, which I think is not very good for verify_required_params().
I want to terminate this request without returning any value from verify_required_params() in case of missing params. else flow should continue.
Running this on flask server and open to include any new package for best/ optimized approach.
Can please someone suggest an optimize way for this?
#app.route('/is_registered', methods=['POST'])
def is_registered():
_json = request.get_json()
keys = _json.keys()
customer = Customer()
if verify_required_params(['mobile'], keys) is True:
_mobile = _json['mobile']
validated = validate_mobile(_mobile)
registered = customer.find(_mobile)
if not validated:
response = get_response('MOBILE_NUMBER_NOT_VALID')
return jsonify(response)
if not registered:
response = get_response('MOBILE_NUMBER_NOT_REGISTERED')
return jsonify(response)
response = get_response('MOBILE_NUMBER_REGISTERED')
return jsonify(response)
else:
return verify_required_params(['mobile'], keys)
def verify_required_params(required, received):
required = set(required)
received = set(received)
missing = list(sorted(required - received))
data = {"missing_key(s)": missing}
# response = app.response_class(
# response=json.dumps(data),
# status=200,
# mimetype='application/json'
# )
if missing:
return jsonify(data)
return True
🎶 You say it works in a RESTful way, then your errors come back as 200 OK 🎶
In REST, your URL should encode all the information about your entity. In your case, you are identifying a client by their phone number, and you are getting rather than updating information about them, so your endpoint should look like GET /client/<phonenumber>/registered. That way, a request can't not provide this information without going to a different endpoint.
In short, your code will be replaced with:
#app.route('/client/<mobile>/registered', methods=['GET'])
def is_registered(mobile):
if not mobile.is_decimal():
return jsonify({'error': 'mobile is not number'}), 400 # Bad Request
customer = Customer()
registered = bool(customer.find(mobile))
# does it make sense to have a customer who is not registered yet?
# if not, use:
if not registered:
return jsonify({'error': 'client not found'}), 404 # Not Found
validated = validate_mobile(mobile)
return jsonify( {'validated': validated, 'registered': registered} )
In addition, it's better to have the validation function be a decorator. That way it gets called before the actual business logic of the function. For your example of checking whether request.get_json() contains the proper fields, this is how it would look like:
import functools
def requires_fields(fields):
required_fields = set(fields)
def wrapper(func):
#functools.wraps(decorated)
def decorated(*args, **kwargs):
current_fields = set(request.get_json().keys())
missing_fields = required_fields
if missing_fields:
return jsonify({'error': 'missing fields', 'fields': list(missing_fields)}), 400 # Bad Request
resp = func(*args, **kwargs)
return resp
return wrapper
# usage:
#app.route('/comment', methods=['POST'])
#requires_fields(['author', 'post_id', 'body'])
def create_comment():
data = request.get_json()
id = FoobarDB.insert('comment', author=data['author'], post_id=data['post_id'], body=data['body'])
return jsonify({'new_id': id}), 201 # Created
If you must leave it the way it is now, then in order to not return data from the validation function, you must raise an HTTPException. The default function to do it is flask.abort(code).
I'm developing a Flask app, with MySQL (flask-mysqldb) and MQTT (flask-mqtt) integrations. I can perform any DB operation from a Flask method (e.g. #app.route('/')), but if I try to do it from a MQTT method when I receive a message (e.g. #mqtt.on_message()) it does nothing. This last method works perfectly because it receives and shows in log the message received.
I have a method that performs DB operations, and depending on where I call it from, it works or not. I guess it should be because of the MySQL object, but I don't know exactly.
Here is the code I'm using (just the problem):
#mqtt.on_message()
def handle_mqtt_message(client, userdata, message):
print('New message {}'.format(message.payload.decode()))
storeDB('test') #Here it doesn't work
################## Methods ###########################
def storeDB(param_text):
cur = mysql.connection.cursor()
cur.execute(
'INSERT INTO contacts (fullname, phone, email) VALUES (%s,%s,%s)', (param_text, param_text, param_text))
mysql.connection.commit()
###################### FLASK #########################
#app.route('/')
def index():
storeDB('temp') #Here it works
return 'Hello World'
If I access to localhost it shows the "Hello World" text in browser and updates the DB; otherwise, if I receive a MQTT message, it is shown on terminal but not updated the DB.
Thanks.
This is how I have it working using the MySQLdbpackage:
import MySQLdb
try:
db = MySQLdb.connect("HOST", "USER", "PASS", "DB")
except:
print("Problem creating DB connection")
sys.exit()
cursor = db.cursor()
def storeDB(param_text1, param_text2, param_text3):
query = """INSERT INTO `DB`.`TABLE` (`fullname`, `phone`, `email`) VALUES ('""" + \
param_text1+"""','"""+param_text2+"""','"""+param_text3+"""');"""
try:
cursor.execute(query)
db.commit()
print('DB updated')
except:
db.rollback()
print("Problem updating DB :(")
I am following the tutorial by http://www.patricksoftwareblog.com/flask-tutorial/, which I believe is based on https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-i-hello-world. Great stuff for a beginner.
I am getting different results when testing my code through frontend manually (which works fine) v.s. through pytest.
My test tries to show the "groups" endpoint which requires a login (standard #login_required decorator).
I initially test the user getting a login page ("Knock knock") when trying to get the endpoint without a login. This works manually and through pytest.
I login a user. If I inspect the response from the login I can clearly see a "Welcome back Pete!" success message.
My second assert receives a response from URL /login?next=%2Fgroups indicating the /groups endpoint is called without a login/authentication preceding it and the assert fails. Testing this manually works as expected. Why is that single test not using the same user/session combination in the next step(s)?
Test with the problem is the first snippet below:
def test_groups(app):
assert b'Knock knock' in get(app, "/groups").data
login(app, "pete#testmail.com", "pete123")
assert b'Test group 1' in get(app, "/groups").data
My "get" function for reference:
def get(app, endpoint: str):
return app.test_client().get(endpoint, follow_redirects=True)
My "login" function for reference:
def login(app, email="testuser#testmail.com", password="testing"):
return app.test_client().post('/login', data=dict(email=email, password=password), follow_redirects=True)
The app (from a conftest fixture imported in the test module by #pytest.mark.usefixtures('app')) for reference:
#pytest.fixture
def app():
"""An application for the tests."""
_app = create_app(DevConfig)
ctx = _app.test_request_context()
ctx.push()
yield _app
ctx.pop()
The login route for reference:
#app.route('/login', methods=['GET', 'POST'])
def login():
form = LoginForm(request.form)
if request.method == 'POST':
if form.validate_on_submit():
user = User.query.filter_by(email=form.email.data).first()
if user is not None and user.is_correct_password(form.password.data):
user.authenticated = True
user.last_login = user.current_login
user.current_login = datetime.now()
user.insert_user()
login_user(user)
flash(f'Welcome back {user.name}!', 'success')
return redirect(url_for('our_awesome_group.index'))
else:
flash('Incorrect credentials! Did you already register?', 'error')
else:
flash_errors(form)
return render_template('login.html', form=form)
The groups route for reference:
#app.route('/groups')
#login_required
def groups():
groups_and_users = dict()
my_group_uuids = Membership.list_groups_per_user(current_user)
my_groups = [Group.query.filter_by(uuid=group).first() for group in my_group_uuids]
for group in my_groups:
user_uuids_in_group = Membership.list_users_per_group(group)
users_in_group = [User.query.filter_by(uuid=user).first() for user in user_uuids_in_group]
groups_and_users[group] = users_in_group
return render_template('groups.html', groups_and_users=groups_and_users)
Im going to sum up the comments I made that gave the answer on how to solve this issue.
When creating a test app using Pytest and Flask there are a few different ways to go about it.
The suggested way to create a test client with proper app context is to use something like:
#pytest.fixture
def client():
""" Creates the app from testconfig, activates test client and context then makes the db and allows the test client
to be used """
app = create_app(TestConfig)
client = app.test_client()
ctx = app.app_context()
ctx.push()
db.create_all()
yield client
db.session.close()
db.drop_all()
ctx.pop()
That creates the client while pushing the app context so you can register things like your database and create the tables to the test client.
The second way is show in OP's question where use app.test_request context
#pytest.fixture
def app():
"""An application for the tests."""
_app = create_app(DevConfig)
ctx = _app.test_request_context()
ctx.push()
yield _app
ctx.pop()
and then create the test client in another pytest fixture
#pytest.fixture
def client(app):
return app.test_client()
Creating a test client allows you to use various testing features and gives access to flask requests with the proper app context.
I'm a Flask newbie trying to create a simple app. I'm currently stuck at user registration where I'm trying to save data in database but it's not happening. However, the logging I'm doing indicates that the operation was a success. Could someone tell me what I'm doing wrong?
Here are portions of code that'll help you understand what I'm trying to do:
from flask import Flask, request, session, g, redirect, url_for, abort, render_template, flash
from flask.ext.mysqldb import MySQL
# Configuration
MYSQL_HOST = 'localhost'
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'root'
MYSQL_DB = 'up2date'
DEBUG = True
SECRET_KEY =
'\xc6)\x0f\\\xc5\x86*\xd7[\x92\x89[\x95\xcfD\xfd\xc1\x18\x8e\xf1P\xf7_\r'
# Create the flask app
app = Flask(__name__)
app.config.from_object(__name__)
# Create instance for working with MySQL
mysql = MySQL(app)
# Function to connect to DB
def connect_db():
return mysql.connection.cursor()
# define functions that will make DB available automatically on each request
#app.before_request
def before_request():
g.db = connect_db()
#app.teardown_request
def teardown_request(exception):
g.db.close()
And finally, the code that performs user registration:
#app.route('/register', methods=['GET', 'POST'])
def register():
if request.method == 'POST':
email = request.form['email']
password = request.form['password']
result = g.db.execute('INSERT INTO users (email, password) VALUES (%s, %s)', [email, password])
print(email, password)
print(result, " rows affected")
flash('Registration successful! You may log in now.')
return redirect(url_for('show_home'))
The two print statements confirm that the email address and password were captured correctly, and the result variable contains 1, indicating 1 row affected. But still there's no row in the DB. I earlier thought this had something to do with committing, but g.db.commit() throws error: AttributeError: 'Cursor' object has no attribute 'commit'
I assume you use MySQL-python.
connect_db() returns the cursor and not the connection. The cursor does not have a commit() function, as the exception says, however the connection has the commit function you need. I think you need to do this:
def connect_db():
return mysql.connection
For more info you can take a look at the code.