Threading Class - 'int' not iterable | Flask - python-3.x

I'm current encountering an issue where I'm using threads to perform tasks in the background upon a POST request, but I'm having an issue with using an integer as a parameter; or any data type as a parameter for that matter.
What is supposed to happen is that it's supposed to perform everything inside of the POST request check, then perform something else after that has been completed.
Imports:
from flask import Flask, url_for, render_template, redirect, request
from flask.cli import with_appcontext, click
import sqlalchemy
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager, login_required, current_user
Thread Class:
#click.command(name='run')
#click.pass_context
class Compute(Thread):
def __init__(self, secs):
Thread.__init__(self)
self.secs = secs
def run(self):
print('Thread Opened')
sleep(self.secs)
db.session.query(Users).filter(Users.username == current_user.username).first().requests += 1
db.session.commit()
print('Thread Closed')
Flask Route:
#app.route('/index', methods=['GET', 'POST'])
#login_required
def index():
if request.method == 'POST':
time = '5'
thread = Compute(int(time))
thread.start()
return redirect(url_for('index'))
else:
return render_template('index.html')
The error I am receiving when the thread tries to start is this:
Traceback (most recent call last):
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask\app.py", line 1952, in full_dispatch_request rv = self.handle_user_exception(e)
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask\app.py", line 1821, in handle_user_exception reraise(exc_type, exc_value, tb)
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask\app.py", line 1950, in full_dispatch_request rv = self.dispatch_request()
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\flask_login\utils.py", line 272, in decorated_view return func(*args, **kwargs)
File "c:/Users/myuser/Desktop/Projects/Python/app.py", line 104, in layer4
thread = Compute(int(time))
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\click\core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "C:\Users\myuser\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.7_qbz5n2kfra8p0\LocalCache\local-packages\Python37\site-packages\click\core.py", line 767, in main
args = list(args)
TypeError: 'int' object is not iterable
I am not very experienced with threading, I've only dabbled with it in other projects so I'm not sure why I'm getting this error; any help is greatly appreciated.

You need to remove the #click.command and #click.pass_context decorators.
#click.command(name='run')
#click.pass_context
class Compute(Thread):
Those decorators are wrapping the class in a function that takes its own parameters. It is intended for running a function from the command line and parsing arguments. This is why the traceback shows args = list(args) as the line causing the error.
https://github.com/pallets/click/blob/master/src/click/core.py#L916-L917
:param args: the arguments that should be used for parsing. If not
provided, ``sys.argv[1:]`` is used.

Related

Flask-Restful Object not serializable errors on return code 200

Im currently getting a strange error whenever I try to return a response with the code 200.
Here's the snippet of the code I'm using:
from pathlib import Path
from flask_restful import Resource
from app.models.user import User
from app.models.workspace import Workspace
from app.models.file import File
from app.controllers.filesystem import FileSystem
from flask import request
import os
class Signup(Resource):
def post(self):
if not request.is_json:
return {"msg": "Missing JSON in request"}, 400
email = request.json.get('email', None)
password = request.json.get('password', None)
if not email:
return {"msg": "Missing username parameter"}, 400
if not password:
return {"msg": "Missing password parameter"}, 400
# TODO- Find a more elegant way to do this
try:
user = User.objects.get(email=email)
except User.DoesNotExist:
user = None
if user:
return {"msg": "User already exists"}, 400
new_user = User(email=email, password=password)
new_user.hash_password()
# Create a new workspace called "Microfluidics Examples"
new_workspace = Workspace(name="Microfluidics Examples")
# Step 1 - Go through every file in the examples directory
# Step 2 - Upload these files to s3 and get the file_id
# Step 3 - Create a new file object for each of the file_ids and add it to the workspace
# Go through every file in the examples directory
examples_directory = Path("examples")
for file_name in examples_directory.iterdir():
# Upload the file to s3 and get the file_id
s3_object_id = FileSystem.upload_file(file_name)
# Create a new file object for the file_id and add it to the workspace
new_file = File(file_name=str(file_name.name), s3_path=s3_object_id)
new_file.save()
new_workspace.design_files.append(new_file)
# Save the workspace
new_workspace.save()
# Add the workspace to the user
new_user.workspaces.append(new_workspace)
# Save the user
new_user.save()
# Return success
return {"mes":, "id": str(new_user.id)}, 200
From the error trace, it looks like something originating from the cors package might be interfering with this. Here's the trace:
127.0.0.1 - - [19/Nov/2022 14:18:02] "GET /api/v2/user HTTP/1.1" 500 -
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2548, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2528, in wsgi_app
response = self.handle_exception(e)
File "/usr/local/lib/python3.8/dist-packages/flask_restful/__init__.py", line 271, in error_router
return original_handler(e)
File "/usr/local/lib/python3.8/dist-packages/flask_cors/extension.py", line 165, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 2525, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1822, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.8/dist-packages/flask_restful/__init__.py", line 271, in error_router
return original_handler(e)
File "/usr/local/lib/python3.8/dist-packages/flask_cors/extension.py", line 165, in wrapped_function
return cors_after_request(app.make_response(f(*args, **kwargs)))
File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1820, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.8/dist-packages/flask/app.py", line 1796, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)
File "/usr/local/lib/python3.8/dist-packages/flask_restful/__init__.py", line 471, in wrapper
return self.make_response(data, code, headers=headers)
File "/usr/local/lib/python3.8/dist-packages/flask_restful/__init__.py", line 500, in make_response
resp = self.representations[mediatype](data, *args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/flask_restful/representations/json.py", line 21, in output_json
dumped = dumps(data, **settings) + "\n"
File "/usr/lib/python3.8/json/__init__.py", line 234, in dumps
return cls(
File "/usr/lib/python3.8/json/encoder.py", line 201, in encode
chunks = list(chunks)
File "/usr/lib/python3.8/json/encoder.py", line 438, in _iterencode
o = _default(o)
File "/usr/lib/python3.8/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type function is not JSON serializable
I can't figure out why this is happening; for instance, if I change the return code to 400, it just works fine. Can someone help me figure out what this error is?
Apologies to the community, I traced the error incorrectly. The error did not originate because of the given code snippet. I will update the question shortly to reflect the source of the error trace correctly and post the solution

What is the right way to initialize database using tortoise orm in flask app?

I learn flask, before I've complete chapter about tortoise ORM.
I try to combine flask and tortoise and I can't find proper way.
When I call async function I got error:
Traceback (most recent call last):
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 2450, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 1867, in handle_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\_compat.py", line 39, in reraise
raise value
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\flask\app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "C:\Users\Vladlen\python\pythonBasics\lesson 19\views\doc_income.py", line 19, in new
run_async(create_doc(request.values['account_id'], request.values['sum']))
File "C:\Users\Vladlen\python\pythonBasics\venv\lib\site-packages\tortoise\__init__.py", line 634, in run_async
loop = asyncio.get_event_loop()
File "C:\Users\Vladlen\AppData\Local\Programs\Python\Python37\lib\asyncio\events.py", line 644, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-7'.
piece of my code calling document creation:
#doc_income_app.route("new", methods=['GET', 'POST'])
def new():
if request.method == 'GET':
return render_template('doc_income/new.html')
else:
run_async(create_doc(request.values['account_id'], request.values['sum']))
return jsonify(ok=True)
async def create_doc(account_id, sum):
await db.init()
doc = db.DocIncome(
account_id=account_id,
sum=sum
)
await doc.save()
My question is how to call async db manipulation code in flask application?
How to call async code from sync code?
Create an event loop, and run code in there. Problem is the event loop has to clear out, so any scheduled tasks will make it never end. (A risk of doing run_until_complete)
Basically:
asyncio.get_event_loop().run_until_complete(create_doc(...))
The run_async helper that tortoise provides also enforces that DB is cleaned up, so I'm uncertain it's what you want.
Apologies, but you are essentially calling from an incompatible colour and is trailblazing, there isn't a texbook answer I can give.
My original answer:
Flask is a synchronous Python framework, it doesn't use an event loop.
Consider using an asyncio-based framework, such as Quart/aiohttp/starlette, both that and Tortoise-ORM the uses the same concurrency method and it should work.

Tornado simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

I have a TornadoAPI for keras model, is it possible to check concurrency using python multiprocessing module, I tried with below code, but it throws error
from multiprocessing import Pool
import requests, json
url = 'http://localhost:8888/index/predict'
payload = { "colA":"some1", "colB":"some2",...….)
pl = json.dumps(payload)
def callAPI(x):
session = requests.Session()
r = session.post(url, json=json.loads(pl))
response = r.json()
return response
if __name__ == '__main__':
Pool(processes=15).map(callAPI, range(5))
Error
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "C:\Models\CallThreadsCheck.py", line 40, in callAPI
response = r.json()
File "C:\Users\HZ\Anaconda3\lib\site-packages\requests\models.py", line 897, in json
return complexjson.loads(self.text, **kwargs)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\__init__.py", line 525, in loads
return _default_decoder.decode(s)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "CallThreadsCheck.py", line 49, in <module>
p.map(callAPI, range(calls))
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 657, in get
raise self._value
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
May I know what's this error about pls.

Random connection errors with aio_pika after 2 days of running

I have asyncio script which connects to rabbitmq with aio_pika library every 40 seconds and checks if there are any messages and prints them out which then repeats forever. However, usually, after 2 or so days of running, I will start receiving endless connection exception errors which will only be solved by restarting the script. Perhaps there are some obvious mistakes in the logic of my asyncio script which I am missing?
#!/usr/bin/python3
import time
import async_timeout
import asyncio
import aio_pika
async def got_message(message: aio_pika.IncomingMessage):
with message.process():
print(message.body.decode())
async def main(loop):
try:
with async_timeout.timeout(10):
connection = await aio_pika.connect_robust(
host='#',
virtualhost='#',
login='#',
password='#',
port=5671,
loop=loop,
ssl=True
)
channel = await connection.channel()
await channel.set_qos(prefetch_count=100)
queue_name='mm_message'
queue = await channel.declare_queue(auto_delete=False, name=queue_name)
routing_key='mm_msg'
await queue.bind("amq.topic", routing_key)
que_len = queue.declaration_result.message_count
if(que_len > 0):
await queue.consume(got_message)
except:
print("connection problems..")
if __name__ == "__main__":
loop = asyncio.get_event_loop()
while(True):
time.sleep(40)
loop.run_until_complete(main(loop))
This is the error I endlessly receive after some time:
Traceback (most recent call last):
File "/usr/lib/python3.5/asyncio/events.py", line 125, in _run
self._callback(*self._args)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/adapters/base_connection.py", line 364, in _handle_events
self._handle_read()
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/adapters/base_connection.py", line 415, in _handle_read
self._on_data_available(data)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1347, in _on_data_available
self._process_frame(frame_value)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1414, in _process_frame
if self._process_callbacks(frame_value):
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1384, in _process_callbacks
frame_value) # Args
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/callback.py", line 60, in wrapper
return function(*tuple(args), **kwargs)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/callback.py", line 92, in wrapper
return function(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/callback.py", line 236, in process
callback(*args, **keywords)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1332, in _on_connection_tune
self._send_connection_open()
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1517, in _send_connection_open
self._on_connection_open, [spec.Connection.OpenOk])
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1501, in _rpc
self._send_method(channel_number, method_frame)
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1569, in _send_method
self._send_frame(frame.Method(channel_number, method_frame))
File "/usr/local/lib/python3.5/dist-packages/aio_pika/pika/connection.py", line 1548, in _send_frame
raise exceptions.ConnectionClosed
aio_pika.pika.exceptions.ConnectionClosed
except:
print("connection problems..")
This will catch service Exceptions like KeyboardInterrupt, SystemExit. You should never do such thing if you're not going to reraise exception. At very lest you should write:
except Exception:
print("connection problems..")
However in context of asyncio snippet above will break mechanism of cancellation. To avoid it as explained here you should write:
try:
await operation
except asyncio.CancelledError:
raise
except Exception:
log.log('an error has occurred')
Not less important is to understand that connection should not only be opened, but closed also (regardless of what happened between opening and closing). To achieve that people usually use context managers (and in asyncio - asynchronous context managers).
aio_pika doesn't seem to be exception. As example shows you should use async with when dealing with connection:
connection = await aio_pika.connect_robust(
"amqp://guest:guest#127.0.0.1/", loop=loop
)
async with connection:
# ...

Why is BeautifulSoup related to 'Task exception was never retrieved'?

I want to use the coroutine to crawl and parse webpages. I write a sample and test. The program could run well in python 3.5 in ubuntu 16.04 and it will quit when all the works have been done. The source code is below.
import aiohttp
import asyncio
from bs4 import BeautifulSoup
async def coro():
coro_loop = asyncio.get_event_loop()
url = u'https://www.python.org/'
for _ in range(4):
async with aiohttp.ClientSession(loop=coro_loop) as coro_session:
with aiohttp.Timeout(30, loop=coro_session.loop):
async with coro_session.get(url) as resp:
print('get response from url: %s' % url)
source_code = await resp.read()
soup = BeautifulSoup(source_code, 'lxml')
def main():
loop = asyncio.get_event_loop()
worker = loop.create_task(coro())
try:
loop.run_until_complete(worker)
except KeyboardInterrupt:
print ('keyboard interrupt')
worker.cancel()
finally:
loop.stop()
loop.run_forever()
loop.close()
if __name__ == '__main__':
main()
While testing, I find when I shut down the program by 'Ctrl+C', there will be a error 'Task exception was never retrieved'.
^Ckeyboard interrupt
Task exception was never retrieved
future: <Task finished coro=<coro() done, defined at ./test.py:8> exception=KeyboardInterrupt()>
Traceback (most recent call last):
File "./test.py", line 23, in main
loop.run_until_complete(worker)
File "/usr/lib/python3.5/asyncio/base_events.py", line 375, in run_until_complete
self.run_forever()
File "/usr/lib/python3.5/asyncio/base_events.py", line 345, in run_forever
self._run_once()
File "/usr/lib/python3.5/asyncio/base_events.py", line 1312, in _run_once
handle._run()
File "/usr/lib/python3.5/asyncio/events.py", line 125, in _run
self._callback(*self._args)
File "/usr/lib/python3.5/asyncio/tasks.py", line 307, in _wakeup
self._step()
File "/usr/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "./test.py", line 17, in coro
soup = BeautifulSoup(source_code, 'lxml')
File "/usr/lib/python3/dist-packages/bs4/__init__.py", line 215, in __init__
self._feed()
File "/usr/lib/python3/dist-packages/bs4/__init__.py", line 239, in _feed
self.builder.feed(self.markup)
File "/usr/lib/python3/dist-packages/bs4/builder/_lxml.py", line 240, in feed
self.parser.feed(markup)
File "src/lxml/parser.pxi", line 1194, in lxml.etree._FeedParser.feed (src/lxml/lxml.etree.c:119773)
File "src/lxml/parser.pxi", line 1316, in lxml.etree._FeedParser.feed (src/lxml/lxml.etree.c:119644)
File "src/lxml/parsertarget.pxi", line 141, in lxml.etree._TargetParserContext._handleParseResult (src/lxml/lxml.etree.c:137264)
File "src/lxml/parsertarget.pxi", line 135, in lxml.etree._TargetParserContext._handleParseResult (src/lxml/lxml.etree.c:137128)
File "src/lxml/lxml.etree.pyx", line 324, in lxml.etree._ExceptionContext._raise_if_stored (src/lxml/lxml.etree.c:11090)
File "src/lxml/saxparser.pxi", line 499, in lxml.etree._handleSaxData (src/lxml/lxml.etree.c:131013)
File "src/lxml/parsertarget.pxi", line 88, in lxml.etree._PythonSaxParserTarget._handleSaxData (src/lxml/lxml.etree.c:136397)
File "/usr/lib/python3/dist-packages/bs4/builder/_lxml.py", line 206, in data
def data(self, content):
KeyboardInterrupt
I looked through the offical docs of python but haven't got a clue. I try to capture the Keyboard Interrupt in coro().
try:
soup = BeautifulSoup(source_code, 'lxml')
except KeyboardInterrupt:
print ('capture exception')
raise
Everytime the 'try/except' around BeautifulSoup() capture the KeyboardInterrupt, the error will occur. It seems that BeautifulSoup contribute to the error. But how to tackle it?
When you call task.cancel() this function doesn't actually cancel task, it just "mark" task to be cancelled. Actual process of cancelling task would be started when task will resume it's execution. asyncio.CancelledError will be immediately raised inside task forcing it to be actually cancelled. Task will finish it's execution with this exception.
On the other hand asyncio warns you if some of your tasks finished with exception silently (if you didn't check result of task execution).
To avoid problems you should await task cancellation receiving asyncio.CancelledError (and probably suppressing since you don't need it then):
import asyncio
from contextlib import suppress
async def coro():
# ...
def main():
loop = asyncio.get_event_loop()
worker = asyncio.ensure_future(coro())
try:
loop.run_until_complete(worker)
except KeyboardInterrupt:
print('keyboard interrupt')
worker.cancel()
with suppress(asyncio.CancelledError):
loop.run_until_complete(worker) # await task cancellation.
finally:
loop.close()
if __name__ == '__main__':
main()

Resources