Related
I have a problem with fsm using aiogram with Fastapi. I ran the code from aiogram_fsm_example, but changed the long-polling to the Fastapi implementation. Here's the code I've got:
import logging
from fastapi import FastAPI, Request
import aiogram.utils.markdown as md
from aiogram import Bot, Dispatcher, types
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters import Text
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.types import ParseMode
from aiogram.utils import executor
logging.basicConfig(level=logging.INFO)
API_TOKEN = "here's the bot token"
bot = Bot(token=API_TOKEN)
# For example use simple MemoryStorage for Dispatcher.
storage = MemoryStorage()
dp = Dispatcher(bot, storage=storage)
# States
class Form(StatesGroup):
name = State() # Will be represented in storage as 'Form:name'
age = State() # Will be represented in storage as 'Form:age'
gender = State() # Will be represented in storage as 'Form:gender'
#dp.message_handler(commands='start')
async def cmd_start(message: types.Message):
"""
Conversation's entry point
"""
# Set state
await Form.name.set()
await message.reply("Hi there! What's your name?")
#dp.message_handler(state=Form.name)
async def process_name(message: types.Message, state: FSMContext):
"""
Process user name
"""
async with state.proxy() as data:
data['name'] = message.text
await Form.next()
await message.reply("How old are you?")
# Check age. Age gotta be digit
#dp.message_handler(lambda message: not message.text.isdigit(), state=Form.age)
async def process_age_invalid(message: types.Message):
"""
If age is invalid
"""
return await message.reply("Age gotta be a number.\nHow old are you? (digits only)")
#dp.message_handler(lambda message: message.text.isdigit(), state=Form.age)
async def process_age(message: types.Message, state: FSMContext):
# Update state and data
await Form.next()
await state.update_data(age=int(message.text))
# Configure ReplyKeyboardMarkup
markup = types.ReplyKeyboardMarkup(resize_keyboard=True, selective=True)
markup.add("Male", "Female")
markup.add("Other")
await message.reply("What is your gender?", reply_markup=markup)
#dp.message_handler(lambda message: message.text not in ["Male", "Female", "Other"], state=Form.gender)
async def process_gender_invalid(message: types.Message):
"""
In this example gender has to be one of: Male, Female, Other.
"""
return await message.reply("Bad gender name. Choose your gender from the keyboard.")
#dp.message_handler(state=Form.gender)
async def process_gender(message: types.Message, state: FSMContext):
async with state.proxy() as data:
data['gender'] = message.text
# Remove keyboard
markup = types.ReplyKeyboardRemove()
# And send message
await bot.send_message(
message.chat.id,
md.text(
md.text('Hi! Nice to meet you,', md.bold(data['name'])),
md.text('Age:', md.code(data['age'])),
md.text('Gender:', data['gender']),
sep='\n',
),
reply_markup=markup,
parse_mode=ParseMode.MARKDOWN,
)
# Finish conversation
await state.finish()
# my changes
app = FastAPI()
#app.get("/")
async def root():
return "ok"
#app.post("/")
async def process_update(request: Request):
update = await request.json()
update = types.Update(**update)
print("incoming", update)
await dp.process_update(update)
But when I run that with uvicorn (uvicorn main:app) and send /start command to the bot, the backend throws this error:
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/uvicorn/protocols/http/h11_impl.py", line 373, in run_asgi
result = await app(self.scope, self.receive, self.send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 75, in __call__
return await self.app(scope, receive, send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/fastapi/applications.py", line 208, in __call__
await super().__call__(scope, receive, send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 181, in __call__
raise exc
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/exceptions.py", line 82, in __call__
raise exc
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/routing.py", line 656, in __call__
await route.handle(scope, receive, send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/routing.py", line 259, in handle
await self.app(scope, receive, send)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/starlette/routing.py", line 61, in app
response = await func(request)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/fastapi/routing.py", line 226, in app
raw_response = await run_endpoint_function(
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/fastapi/routing.py", line 159, in run_endpoint_function
return await dependant.call(**values)
File "/home/oleh/projects/tg_bot_test_fsm/./main.py", line 124, in process_update
await dp.process_update(update)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/aiogram/dispatcher/dispatcher.py", line 257, in process_update
return await self.message_handlers.notify(update.message)
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/aiogram/dispatcher/handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "/home/oleh/projects/tg_bot_test_fsm/./main.py", line 38, in cmd_start
await Form.name.set()
File "/home/oleh/projects/tg_bot_test_fsm/.venv/lib/python3.8/site-packages/aiogram/dispatcher/filters/state.py", line 56, in set
state = Dispatcher.get_current().current_state()
AttributeError: 'NoneType' object has no attribute 'current_state'
As far as I understood: there's a state in the dispatcher that is not created somewhy when I use the dp.process_update() function.
When I run that with long_polling - everything works fine, but I need so much to run it with Fastapi.
Is there a way to set up the state manually? Or I just do not process it correctly?
P.S. I run it in the WSL Ubuntu 20.04 LTS. Python version is 3.8.10, aiogram - 2.15, uvicorn - 0.15.0 and Fastapi - 0.70.0.
SOLVED: if you're using Fastapi with aiogram and trying FSM, you need to set state in another way, with state.set_state(Form.name) function. My working code of start method looks like that:
#dp.message_handler(commands='start', state="*")
async def cmd_start(message: types.Message, state: FSMContext):
"""
Conversation's entry point
"""
# Set state
await state.set_state(Form.name)
await message.reply("Hi there! What's your name?")
It's enough to set current context for Dispatcher
dp = Dispatcher(bot, storage=storage)
Dispatcher.set_current(dp)
mistakes
mistakes 2
photo
https://github.com/mahenzon/aiogram-lessons/tree/master/lesson-02
I can't understand why my code isn't working. I took code from here and only change file config.py, change token and MY_ID.
executor.py [ LINE:362 ]# INFO [2021-08-12 14:43:46,938] Bot: lap156 [#lab156_bot]
dispatcher.py [ LINE:360 ]# INFO [2021-08-12 14:43:46,938] Start polling.
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:47:59,585] Task exception was never retrieved
future: <Task finished name='Task-24' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 55, in process_photo_command
await bot.send_photo(message.from_user.id, CAT_BIG_EYES,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 482, in send_photo
result = await self.request(api.Methods.SEND_PHOTO, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:48:03,624] Task exception was never retrieved
future: <Task finished name='Task-32' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 73, in process_note_command
await bot.send_video_note(message.from_user.id, VIDEO_NOTE)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 963, in send_video_note
result = await self.request(api.Methods.SEND_VIDEO_NOTE, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:48:06,943] Task exception was never retrieved
future: <Task finished name='Task-37' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 55, in process_photo_command
await bot.send_photo(message.from_user.id, CAT_BIG_EYES,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 482, in send_photo
result = await self.request(api.Methods.SEND_PHOTO, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
base_events.py [ LINE:1738 ]# ERROR [2021-08-12 14:48:15,298] Task exception was never retrieved
future: <Task finished name='Task-41' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py:409> exception=WrongFileIdentifier('Wrong file identifier/http url specified')>
Traceback (most recent call last):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 417, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 238, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 259, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\Zver\aiogram-lessons\lesson-02\bot.py", line 73, in process_note_command
await bot.send_video_note(message.from_user.id, VIDEO_NOTE)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\bot.py", line 963, in send_video_note
result = await self.request(api.Methods.SEND_VIDEO_NOTE, payload, files)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\base.py", line 208, in request
return await api.make_request(self.session, self.server, self.__token, method, data, files,
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\bot\api.py", line 115, in check_result
exceptions.BadRequest.detect(description)
File "C:\Users\Zver\AppData\Local\Programs\Python\Python39\lib\site-packages\aiogram\utils\exceptions.py", line 140, in detect
raise err(cls.text or description)
aiogram.utils.exceptions.WrongFileIdentifier: Wrong file identifier/http url specified
I guess the error it seems that one running terminal of the bot is already running when you run your code. So Terminate all running terminals/processes and try again.
#bot.command(cls = CommandWithCooldown, pass_context = True)
#commands.cooldown(1, 30, commands.BucketType.user)
async def strikeball(message, yy=None):
embed = discord.Embed(color=0xeee657, inline = False) #created embed
embed.set_author(name = message.author, icon_url = message.author.avatar_url)
today = datetime.datetime.today()
embed.set_footer(text = 'BotSiniy© | {}'.format(today.strftime("%H:%M %d/%m/%Y")))
if yy==None:
embed.add_field(name = 'Команда: strikeball', #description command
value = "**Описание:** Начинает игру в strikeball(перестрелка)\n**Cool:** 1 сообщения за 30 секунд\n**Использование:** `s!strikeball start`")
await message.channel.send(embed = embed)
elif yy=='start':
red=[]
blue=[]
await message.channel.send('Игра начнется через 60 секунд.... Для присоединения используйте `join (red или blue)`')
for x in range(1,6,1):
time.sleep(10)
await message.channel.send('Игра начнется через {} секунд.... Для присоединения используйте `join (red или blue)`'.format(int(60-(x*10))))
chat_history = await message.channel.history(limit=10, after = message).flatten()
for msg in chat_history:
if msg.author in red or msg.author in blue:
pass
else:
if msg.content== "join red":
red.append(msg.author)
ping = msg.author.mention
await message.channel.send(ping+',присоединился к красной команде!')
elif msg.content=='join blue':
blue.append(msg.author)
ping = msg.author.mention
await message.channel.send(ping+',присоединился к синей команде!')
time.sleep(10)
if len(red)<1 or len(blue)<1:
await message.channel.send("Недостаточно пользователей для начала игры...")
Eroor:
Ignoring exception in command strikeball:
Traceback (most recent call last):
File "bot.py", line 243, in strikeball
async for msg in message.channel.history(limit=10,after = message):
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\iterators.py", line 91, in anext
msg = await self.next()
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\iterators.py", line 285, in next
await self.fill_messages()
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\iterators.py", line 331, in fill_messages
data = await self._retrieve_messages(self.retrieve)
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\iterators.py", line 360, in _retrieve_messages_after_strategy
after = self.after.id if self.after else None
AttributeError: 'Context' object has no attribute 'id'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\ext\commands\bot.py", line 902, in invoke
await ctx.command.invoke(ctx)
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\ext\commands\core.py", line 864, in invoke
await injected(*ctx.args, **ctx.kwargs)
File "C:\Users\Trava\AppData\Local\Programs\Python\Python38-32\lib\site-packag
es\discord\ext\commands\core.py", line 94, in wrapped
raise CommandInvokeError(exc) from exc
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: Att
ributeError: 'Context' object has no attribute 'id
problem in after, but i dont understand why.
Please help me
The first argument in a command is always commands.Context, short ctx
You named it message wich is confusing because its still the commands.Context, not a discord.Message, as you can see in the docs 'Context' object has no attribute 'id' which is also the error
So first rename that:
# old
async def strikeball(message, yy=None):
# new
async def strikeball(ctx, yy=None):
If you want the commands message which is a discord.Message you can use ctx.message
#bot.command()
async cef strikeball(ctx, yy=None):
message = ctx.message
Now also await message.channel.history(limit=10, after=message).flatten() should work
So I asked this question and tried the ProcessPoolExecutor approach. I used the decorator suggested the following way:
Running Image Manipulation in run_in_executor. Adapting to multiprocessing
import asyncio
import functools
from concurrent import futures
from app.exceptions.errors import ManipulationError
_pool = futures.ProcessPoolExecutor()
def executor(function):
#functools.wraps(function)
def decorator(*args, **kwargs):
try:
partial = functools.partial(function, *args, **kwargs)
loop = asyncio.get_event_loop()
return loop.run_in_executor(_pool, partial)
except Exception as e:
raise ManipulationError(str(e))
return decorator
I then used it on a function like:
#executor
#pil
def blur(image):
frame = image.convert("RGBA")
return frame.filter(ImageFilter.BLUR)
Note the #pil is another decorator I made.
def pil(function):
#functools.wraps(function)
def wrapper(image, *args, **kwargs) -> BytesIO:
img = PILManip.pil_image(image)
if img.format == "GIF":
frames = []
for frame in ImageSequence.Iterator(img):
res_frame = function(frame, *args, **kwargs)
frames.append(res_frame)
return PILManip.pil_gif_save(frames), "gif"
elif img.format in ["PNG", "JPEG"]:
img = function(img, *args, **kwargs)
return PILManip.pil_image_save(img), "png"
else:
raise BadImage("Bad Format")
return wrapper
I called it in a FastApi route like so:
#router.get("/blur/", responses=normal_response)
async def blur_image(url: str):
byt = await Client.image_bytes(url)
img, image_format = await blur(byt)
return Response(img.read(), media_type=f"image/{image_format}")
I get some error about pickling.
500 Internal Server Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/uvicorn/protocols/http/httptools_impl.py", line 391, in run_asgi
result = await app(self.scope, self.receive, self.send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
return await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/fastapi/applications.py", line 199, in __call__
await super().__call__(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/applications.py", line 111, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 181, in __call__
raise exc from None
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 159, in __call__
await self.app(scope, receive, _send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 25, in __call__
response = await self.dispatch_func(request, self.call_next)
File "/home/codespace/workspace/dagpi-image/app/middleware/timer.py", line 8, in add_process_time_header
response = await call_next(request)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 45, in call_next
task.result()
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 38, in coro
await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 25, in __call__
response = await self.dispatch_func(request, self.call_next)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette_prometheus/middleware.py", line 56, in dispatch
raise e from None
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette_prometheus/middleware.py", line 52, in dispatch
response = await call_next(request)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 45, in call_next
task.result()
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/middleware/base.py", line 38, in coro
await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/exceptions.py", line 82, in __call__
raise exc from None
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/exceptions.py", line 71, in __call__
await self.app(scope, receive, sender)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/routing.py", line 566, in __call__
await route.handle(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/routing.py", line 227, in handle
await self.app(scope, receive, send)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/starlette/routing.py", line 41, in app
response = await func(request)
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/fastapi/routing.py", line 201, in app
raw_response = await run_endpoint_function(
File "/home/codespace/workspace/dagpi-image/.venv/lib/python3.8/site-packages/fastapi/routing.py", line 148, in run_endpoint_function
return await dependant.call(**values)
File "/home/codespace/workspace/dagpi-image/app/routes/image_routes.py", line 107, in blur_image
img, image_format = await blur(byt)
File "/opt/python/3.8.6/lib/python3.8/multiprocessing/queues.py", line 239, in _feed
obj = _ForkingPickler.dumps(obj)
File "/opt/python/3.8.6/lib/python3.8/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
_pickle.PicklingError: Can't pickle <function blur at 0x7f87524091f0>: it's not the same object as app.image.pil_manipulation.blur
Does someone know why this keeps happening?
I was told the objects have to be serializable, I believe BytesIO is the only in/out of the image. That should be serializable.
Decorators typically produce wrapped functions that aren't easy to pickle (serialize) because they contain hidden state. When dealing with multiprocessing, you should avoid decorators and send ordinary global functions to run_in_executor. For example, you could re-write your executor decorator into a utility function:
_pool = concurrent.futures.ProcessPoolExecutor()
async def exec_async(fn, *args):
loop = asyncio.get_event_loop()
return await loop.run_in_executor(_pool, fn, *args)
Instead of decorating a function with executor, you can just await it using await exec_async(some_function, arg1, arg2, ...). Likewise, you can rewrite the pil decorator into another utility:
def pil(image, transform):
img = PILManip.pil_image(image)
if img.format == "GIF":
frames = []
for frame in ImageSequence.Iterator(img):
res_frame = transform(frame)
frames.append(res_frame)
return PILManip.pil_gif_save(frames), "gif"
elif img.format in ["PNG", "JPEG"]:
img = transform(img)
return PILManip.pil_image_save(img), "png"
else:
raise BadImage("Bad Format")
The implementation of blur now becomes an ordinary function which calls pil, and which can be safely passed to exec_async:
def blur(image):
def transform(frame):
frame = frame.convert("RGBA")
return frame.filter(ImageFilter.BLUR)
return pil(image, transform)
#router.get("/blur/", responses=normal_response)
async def blur_image(url: str):
byt = await Client.image_bytes(url)
img, image_format = await exec_async(blur, byt)
return Response(img.read(), media_type=f"image/{image_format}")
Note: the above code is untested.
Well I have this weird problem. Same script works on windows7 and on windows10 throws AttributeError. And I just can't figure out what is the difference.
This is py code:
import asyncio
from proxybroker import Broker
async def use(proxies):
while True:
proxy = await proxies.get()
if proxy is None:
break
elif 'SOCKS5' in proxy.types: # filter by type
print('Found SOCKS5 proxy: %s' % proxy)
else:
print('Found proxy: %s' % proxy)
async def find(proxies, loop):
broker = Broker(queue=proxies,
timeout=8,
attempts_conn=3,
max_concurrent_conn=200,
judges=['https://httpheader.net/', 'http://httpheader.net/'],
providers=['http://www.proxylists.net/', 'http://fineproxy.org/eng/'],
verify_ssl=False,
loop=loop)
# only anonymous & high levels of anonymity for http protocol and high for others:
types = [('HTTP', ('Anonymous', 'High')), 'HTTPS', 'SOCKS4', 'SOCKS5']
countries = ['US', 'GB', 'DE']
limit = 10
await broker.find(types=types, countries=countries, limit=limit)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
proxies = asyncio.Queue(loop=loop)
tasks = asyncio.gather(find(proxies, loop), use(proxies))
loop.run_until_complete(tasks)
And this is the error that it throws:
C:\Users\draga\AppData\Local\Programs\Python\Python36-32\python.exe "C:/Users/draga/Desktop/Rts/PROXY/Proxy.py"
C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\client.py:576: DeprecationWarning: Use async with instead
warnings.warn("Use async with instead", DeprecationWarning)
C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\helpers.py:72: DeprecationWarning: ClientSession.close() is not coroutine
warnings.warn(self._msg, DeprecationWarning)
C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\helpers.py:72: DeprecationWarning: ClientSession.close() is not coroutine
warnings.warn(self._msg, DeprecationWarning)
C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\helpers.py:72: DeprecationWarning: ClientSession.close() is not coroutine
warnings.warn(self._msg, DeprecationWarning)
Traceback (most recent call last):
File "C:/Users/draga/Desktop/Rts/PROXY/Proxy.py", line 35, in <module>
loop.run_until_complete(tasks)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\asyncio\base_events.py", line 466, in run_until_complete
return future.result()
File "C:/Users/draga/Desktop/Rts/PROXY/Proxy.py", line 29, in find
await broker.find(types=types, countries=countries, limit=limit)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\api.py", line 108, in find
await self._run(self._checker.check_judges(), action)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\api.py", line 114, in _run
await tasks
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\api.py", line 153, in _grab
proxies = await task
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\asyncio\tasks.py", line 452, in _wait_for_one
return f.result() # May raise f.exception().
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\providers.py", line 68, in get_proxies
await self._pipe()
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\providers.py", line 101, in _pipe
await self._find_on_page(self.url)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\providers.py", line 117, in _find_on_page
page = await self.get(url, data=data, headers=headers, method=method)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\providers.py", line 132, in get
page = await self._get(url, data=data, headers=headers, method=method)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\proxybroker\providers.py", line 144, in _get
method, url, data=data, headers=headers) as resp:
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\client.py", line 637, in __aenter__
self._resp = yield from self._coro
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\client.py", line 231, in _request
conn = yield from self._connector.connect(req)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\connector.py", line 378, in connect
proto = yield from self._create_connection(req)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\connector.py", line 686, in _create_connection
_, proto = yield from self._create_direct_connection(req)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\connector.py", line 697, in _create_direct_connection
hosts = yield from self._resolve_host(req.url.raw_host, req.port)
File "C:\Users\draga\AppData\Local\Programs\Python\Python36-32\lib\site-packages\aiohttp\connector.py", line 666, in _resolve_host
self._cached_hosts.expired(key):
AttributeError: 'dict' object has no attribute 'expired'
Update: it seems the problem is in loop.run_until_complete(tasks) still dunno why.
ProxyBroker doesn't works after updating aiohttp to 2.1.0... You have to downgrade it to older version.