Related
How Do I Delete Local Files on Session Close?
My app allows users to upload a file and request some server-side stuff be done to it. Awesome. However, I don't want these files to stay there. Looking at the documentation, something like this should work (sanitized to protect the innocent)...
from shiny import App, reactive, render, ui
import os
def server(input, output, session):
def delete_uploaded_file():
try:
os.remove('file.xlsx')))
except:
# If there isn't a file, it's not really a problem
pass
# This triggers at the end of a session
session.on_ended(delete_uploaded_file())
# this closes the session when the user exits out on their browser
#reactive.Effect
#reactive.event(input.close)
async def _():
await session.close()
app = App(app_ui, server)
but it yields an 'Exception in ASGI application' and a 'TypeError,' all without triggering any of the above code. Everything in the traceback seems to point to shiny, so what am I doing wrong here?
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\uvicorn\protocols\websockets\websockets_impl.py", line 238, in run_asgi
result = await self.app(self.scope, self.asgi_receive, self.asgi_send)
File "C:\Users\e365627\AppData\Roaming\Python\Python39\site-packages\uvicorn\middleware\proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\shiny\_app.py", line 220, in __call__
await self.starlette_app(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\applications.py", line 124, in __call__
await self.middleware_stack(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\middleware\errors.py", line 149, in __call__
await self.app(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\shiny\_autoreload.py", line 114, in __call__
return await self.app(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\middleware\exceptions.py", line 79, in __call__
raise exc
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\middleware\exceptions.py", line 68, in __call__
await self.app(scope, receive, sender)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\routing.py", line 706, in __call__
await route.handle(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\routing.py", line 341, in handle
await self.app(scope, receive, send)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\starlette\routing.py", line 82, in app
await func(session)
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\shiny\_app.py", line 294, in _on_connect_cb
await session._run()
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\shiny\session\_session.py", line 326, in _run
self._run_session_end_tasks()
File "C:\Users\username\AppData\Roaming\Python\Python39\site-packages\shiny\session\_session.py", line 228, in _run_session_end_tasks
self._on_ended_callbacks.invoke()
File "C:\Users\username\AppData\Python\Python39\site-packages\shiny\_utils.py", line 335, in invoke
fn()
TypeError: 'NoneType' object is not callable
INFO: connection closed
Good afternoon friends.
I am having issues with FastApi's FileResponse and catching FileNotFoundError (or any, really) exceptions.
Here's my script:
import os
import sys
from fastapi import FastAPI, HTTPException, File, UploadFile
from fastapi.responses import FileResponse, JSONResponse
from fastapi.encoders import jsonable_encoder
#app.get("/download/{namespace}/{file_name}", response_class=FileResponse)
async def processed_file(namespace: str, file_name: str):
try:
logger.info("About to return file")
return FileResponse(
path=f'{data/{namespace}/{file_name}',
media_type="application/pdf",
filename=file_name,
)
except:
logger.info("Should log this, it doesn't")
raise HTTPException(status_code=404, detail="File not found")
If I call the API endpoint with a path to a non-existing file (which the user can do just by changing one letter in the name of the file, hence the need for this exception),
I can read in the console the following stack trace error:
[2022-10-17 21:00:24 +0000] [8] [ERROR] Exception in ASGI application
Traceback (most recent call last):
File "/opt/venv/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 404, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/opt/venv/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/fastapi/applications.py", line 199, in __call__
await super().__call__(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/starlette.py", line 293, in _sentry_patched_asgi_app
return await middleware(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/asgi.py", line 138, in _run_asgi3
return await self._run_app(scope, lambda: self.app(scope, receive, send))
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/asgi.py", line 187, in _run_app
raise exc from None
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/asgi.py", line 182, in _run_app
return await callback()
File "/opt/venv/lib/python3.9/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/starlette.py", line 98, in _create_span_call
await old_call(*args, **kwargs)
File "/opt/venv/lib/python3.9/site-packages/starlette/middleware/errors.py", line 181, in __call__
raise exc from None
File "/opt/venv/lib/python3.9/site-packages/starlette/middleware/errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/asgi.py", line 138, in _run_asgi3
return await self._run_app(scope, lambda: self.app(scope, receive, send))
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/asgi.py", line 148, in _run_app
raise exc from None
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/asgi.py", line 145, in _run_app
return await callback()
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/starlette.py", line 191, in _sentry_exceptionmiddleware_call
await old_call(self, scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/sentry_sdk/integrations/starlette.py", line 98, in _create_span_call
await old_call(*args, **kwargs)
File "/opt/venv/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__
raise exc from None
File "/opt/venv/lib/python3.9/site-packages/starlette/exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "/opt/venv/lib/python3.9/site-packages/starlette/routing.py", line 580, in __call__
await route.handle(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/starlette/routing.py", line 241, in handle
await self.app(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/starlette/routing.py", line 55, in app
await response(scope, receive, send)
File "/opt/venv/lib/python3.9/site-packages/starlette/responses.py", line 286, in __call__
raise RuntimeError(f"File at path {self.path} does not exist.")
RuntimeError: File at path /app/src/../data/new-namespace/wrong.pdf does not exist.
Note: I originally had except FileNotFoundError: instead of the general exception, still nothing works.
The log file correctly shows the "About to return file" entry, it never reaches the exception with the "should log this...". It also, obviously, does not return the 404 HTTPException. The stack trace error never gets to "my own" file, they are exceptions that are all within the site-packages folder.
I am baffled by this as I would have hoped that my solution would catch any exception. Still, the process is not doing what I was expecting. I have a hunch that I'm missing something obvious.
Thanks for any help, best regards,
Rafa.
The exception is raised after the return - i.e. the exception doesn't get raised when you create FileResponse - it gets raised when Starlette (the library under FastAPI) tries to read the file and return it.
Since it never gets raised in your code, there is nothing for you to catch.
Instead you could validate that whether the requested file is valid or not yourself - Starlette calls os.stat on the file and checks whether the result is None - you could do that (it's probably a good idea to validate that the filename is among the allowed files instead of blindly trusting information from a user) - or you could wrap the existing FileResponse and handle the exception there (in your own implementation of __call__ that calls into the parent class).
So far in my project I was prototyping and used sqlite with sqlalchemy.
I have a couple of tables with oneToMany relationships, all works nicely.
Now, after switching to postgres, I get the asyncpg.exceptions.ForeignKeyViolationError for the tables where I defined ForeignKeys.
Here is my sqlalchemy parent and child model:
from sqlalchemy.orm import relationship
from .database import Base
from fastapi_users_db_sqlalchemy import SQLAlchemyBaseUserTable
class Company(Base):
__tablename__ = "company"
id = Column(Integer, primary_key=True, index=True)
name = Column(String, unique=True)
type = Column(
String(length=10),
server_default="No type given",
nullable=False,
)
users = relationship("User", back_populates="company")
class User(Base, SQLAlchemyBaseUserTable):
first_name = Column(
String(length=50),
index=True,
server_default="No name given",
nullable=False,
)
company_id = Column(Integer, ForeignKey("company.id"), nullable=False)
company = relationship("Company", back_populates="users")
When trying to register a user, I get:
INFO: 172.17.0.1:63534 - "POST /auth/register HTTP/1.1" 500 Internal Server Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/usr/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 373, in run_asgi
result = await app(self.scope, self.receive, self.send)
File "/usr/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 75, in __call__
return await self.app(scope, receive, send)
File "/usr/lib/python3.9/site-packages/fastapi/applications.py", line 208, in __call__
await super().__call__(scope, receive, send)
File "/usr/lib/python3.9/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/usr/lib/python3.9/site-packages/starlette/middleware/errors.py", line 181, in __call__
raise exc
File "/usr/lib/python3.9/site-packages/starlette/middleware/errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "/usr/lib/python3.9/site-packages/starlette/middleware/cors.py", line 92, in __call__
await self.simple_response(scope, receive, send, request_headers=headers)
File "/usr/lib/python3.9/site-packages/starlette/middleware/cors.py", line 147, in simple_response
await self.app(scope, receive, send)
File "/usr/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__
raise exc
File "/usr/lib/python3.9/site-packages/starlette/exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "/usr/lib/python3.9/site-packages/starlette/routing.py", line 656, in __call__
await route.handle(scope, receive, send)
File "/usr/lib/python3.9/site-packages/starlette/routing.py", line 259, in handle
await self.app(scope, receive, send)
File "/usr/lib/python3.9/site-packages/starlette/routing.py", line 61, in app
response = await func(request)
File "/usr/lib/python3.9/site-packages/fastapi/routing.py", line 226, in app
raw_response = await run_endpoint_function(
File "/usr/lib/python3.9/site-packages/fastapi/routing.py", line 159, in run_endpoint_function
return await dependant.call(**values)
File "/usr/lib/python3.9/site-packages/fastapi_users/router/register.py", line 32, in register
created_user = await user_manager.create(user, safe=True, request=request)
File "/usr/lib/python3.9/site-packages/fastapi_users/manager.py", line 153, in create
created_user = await self.user_db.create(db_user)
File "/usr/lib/python3.9/site-packages/fastapi_users_db_sqlalchemy/__init__.py", line 159, in create
await self.database.execute(query, user_dict)
File "/usr/lib/python3.9/site-packages/databases/core.py", line 169, in execute
return await connection.execute(query, values)
File "/usr/lib/python3.9/site-packages/databases/core.py", line 295, in execute
return await self._connection.execute(built_query)
File "/usr/lib/python3.9/site-packages/databases/backends/postgres.py", line 210, in execute
return await self._connection.fetchval(query_str, *args)
File "/usr/lib/python3.9/site-packages/asyncpg/connection.py", line 645, in fetchval
data = await self._execute(query, args, 1, timeout)
File "/usr/lib/python3.9/site-packages/asyncpg/connection.py", line 1659, in _execute
result, _ = await self.__execute(
File "/usr/lib/python3.9/site-packages/asyncpg/connection.py", line 1684, in __execute
return await self._do_execute(
File "/usr/lib/python3.9/site-packages/asyncpg/connection.py", line 1731, in _do_execute
result = await executor(stmt, None)
File "asyncpg/protocol/protocol.pyx", line 201, in bind_execute
asyncpg.exceptions.ForeignKeyViolationError: insert or update on table "user" violates foreign key constraint "user_company_id_fkey"
DETAIL: Key (company_id)=(1) is not present in table "company".
The Error is right, there is no column company_id in table company - only id.
I defined to use the value company.id for column company_id in the user-table.
Why is the program checking on company_id?
I followed this sqlalchemy documentation to define the relationships: https://docs.sqlalchemy.org/en/14/orm/basic_relationships.html
It works fine like this with sqlite.
I can see in the traceback, that it connects to the right backend (postgres), so does that mean, sqlalchemy is not translating the relationship in the right way for postgres?
Any ideas how to solve this?
Solved - for anyone who has this problem as well:
postgres does not seem to reset the id-counter like sqlite does. Since I created test data in a script, always starting from 1 up to the range limit, I used ids that did not exist anymore.
To put it into my example: The company_id I used to create a new user plainly did not exist anymore.
This, I discovered by accident. The error message misled me completely.
mistakes
mistakes 2
photo
https://github.com/mahenzon/aiogram-lessons/tree/master/lesson-02
I can't understand why my code isn't working. I took code from here and only change file config.py, change token and MY_ID.
executor.py [ LINE:362 ]# INFO [2021-08-12 14:43:46,938] Bot: lap156 [#lab156_bot]
dispatcher.py [ LINE:360 ]# INFO [2021-08-12 14:43:46,938] Start polling.
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:47:59,585] Task exception was never retrieved
future: <Task finished name='Task-24' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 55, in process_photo_command
await bot.send_photo(message.from_user.id, CAT_BIG_EYES,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 482, in send_photo
result = await self.request(api.Methods.SEND_PHOTO, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:48:03,624] Task exception was never retrieved
future: <Task finished name='Task-32' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 73, in process_note_command
await bot.send_video_note(message.from_user.id, VIDEO_NOTE)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 963, in send_video_note
result = await self.request(api.Methods.SEND_VIDEO_NOTE, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:48:06,943] Task exception was never retrieved
future: <Task finished name='Task-37' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 55, in process_photo_command
await bot.send_photo(message.from_user.id, CAT_BIG_EYES,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 482, in send_photo
result = await self.request(api.Methods.SEND_PHOTO, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:48:15,298] Task exception was never retrieved
future: <Task finished name='Task-41' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 73, in process_note_command
await bot.send_video_note(message.from_user.id, VIDEO_NOTE)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 963, in send_video_note
result = await self.request(api.Methods.SEND_VIDEO_NOTE, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
I guess the error it seems that one running terminal of the bot is already running when you run your code. So Terminate all running terminals/processes and try again.
So I asked this question and tried the ProcessPoolExecutor approach. I used the decorator suggested the following way:
Running Image Manipulation in run_in_executor. Adapting to multiprocessing
import asyncio
import functools
from concurrent import futures
from app.exceptions.errors import ManipulationError
_pool = futures.ProcessPoolExecutor()
def executor(function):
#functools.wraps(function)
def decorator(*args, **kwargs):
try:
partial = functools.partial(function, *args, **kwargs)
loop = asyncio.get_event_loop()
return loop.run_in_executor(_pool, partial)
except Exception as e:
raise ManipulationError(str(e))
return decorator
I then used it on a function like:
#executor
#pil
def blur(image):
frame = image.convert("RGBA")
return frame.filter(ImageFilter.BLUR)
Note the #pil is another decorator I made.
def pil(function):
#functools.wraps(function)
def wrapper(image, *args, **kwargs) -> BytesIO:
img = PILManip.pil_image(image)
if img.format == "GIF":
frames = []
for frame in ImageSequence.Iterator(img):
res_frame = function(frame, *args, **kwargs)
frames.append(res_frame)
return PILManip.pil_gif_save(frames), "gif"
elif img.format in ["PNG", "JPEG"]:
img = function(img, *args, **kwargs)
return PILManip.pil_image_save(img), "png"
else:
raise BadImage("Bad Format")
return wrapper
I called it in a FastApi route like so:
#router.get("/blur/", responses=normal_response)
async def blur_image(url: str):
byt = await Client.image_bytes(url)
img, image_format = await blur(byt)
return Response(img.read(), media_type=f"image/{image_format}")
I get some error about pickling.
500 Internal Server Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/uvicorn/protocols/http/httptools_impl.py", line 391, in run_asgi
result = await app(self.scope, self.receive, self.send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
return await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/fastapi/applications.py", line 199, in __call__
await super().__call__(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/applications.py", line 111, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 181, in __call__
raise exc from None
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 25, in __call__
response = await self.dispatch_func(request, self.call_next)
File "/home/codespace/workspace/dagpi-image/app/middleware/timer.py", line 8, in add_process_time_header
response = await call_next(request)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 45, in call_next
task.result()
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 38, in coro
await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 25, in __call__
response = await self.dispatch_func(request, self.call_next)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette_prometheus/middleware.py", line 56, in dispatch
raise e from None
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette_prometheus/middleware.py", line 52, in dispatch
response = await call_next(request)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 45, in call_next
task.result()
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 38, in coro
await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/exceptions.py", line 82, in __call__
raise exc from None
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/routing.py", line 566, in __call__
await route.handle(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/routing.py", line 227, in handle
await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/routing.py", line 41, in app
response = await func(request)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/fastapi/routing.py", line 201, in app
raw_response = await run_endpoint_function(
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/fastapi/routing.py", line 148, in run_endpoint_function
return await dependant.call(**values)
File "/home/codespace/workspace/dagpi-image/app/routes/image_routes.py", line 107, in blur_image
img, image_format = await blur(byt)
File "/opt/python/3.8.6/lib/python3.8/multiprocessing/queues.py", line 239, in _feed
obj = _ForkingPickler.dumps(obj)
File "/opt/python/3.8.6/lib/python3.8/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
_pickle.PicklingError: Can't pickle <function blur at 0x7f87524091f0>: it's not the same object as app.image.pil_manipulation.blur
Does someone know why this keeps happening?
I was told the objects have to be serializable, I believe BytesIO is the only in/out of the image. That should be serializable.
Decorators typically produce wrapped functions that aren't easy to pickle (serialize) because they contain hidden state. When dealing with multiprocessing, you should avoid decorators and send ordinary global functions to run_in_executor. For example, you could re-write your executor decorator into a utility function:
_pool = concurrent.futures.ProcessPoolExecutor()
async def exec_async(fn, *args):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(_pool, fn, *args)
Instead of decorating a function with executor, you can just await it using await exec_async(some_function, arg1, arg2, ...). Likewise, you can rewrite the pil decorator into another utility:
def pil(image, transform):
img = PILManip.pil_image(image)
if img.format == "GIF":
frames = []
for frame in ImageSequence.Iterator(img):
res_frame = transform(frame)
frames.append(res_frame)
return PILManip.pil_gif_save(frames), "gif"
elif img.format in ["PNG", "JPEG"]:
img = transform(img)
return PILManip.pil_image_save(img), "png"
else:
raise BadImage("Bad Format")
The implementation of blur now becomes an ordinary function which calls pil, and which can be safely passed to exec_async:
def blur(image):
def transform(frame):
frame = frame.convert("RGBA")
return frame.filter(ImageFilter.BLUR)
return pil(image, transform)
#router.get("/blur/", responses=normal_response)
async def blur_image(url: str):
byt = await Client.image_bytes(url)
img, image_format = await exec_async(blur, byt)
return Response(img.read(), media_type=f"image/{image_format}")
Note: the above code is untested.