ClientDisconnectedError from aiohttp - python-3.x

I have a websocket server built with aiohttp.
I keep getting this exception in the server error stream.
Task exception was never retrieved
future: <Task finished coro=<read() done, defined at /usr/local/lib/python3.4/dist-packages/aiohttp/streams.py:576> exception=ClientDisconnectedError()>
Traceback (most recent call last):
File "/usr/lib/python3.4/asyncio/tasks.py", line 234, in _step
result = coro.throw(exc)
File "/usr/local/lib/python3.4/dist-packages/aiohttp/streams.py", line 578, in read
result = yield from super().read()
File "/usr/local/lib/python3.4/dist-packages/aiohttp/streams.py", line 433, in read
yield from self._waiter
File "/usr/lib/python3.4/asyncio/futures.py", line 386, in __iter__
yield self # This tells Task to wait for completion.
File "/usr/lib/python3.4/asyncio/tasks.py", line 287, in _wakeup
value = future.result()
File "/usr/lib/python3.4/asyncio/futures.py", line 275, in result
raise self._exception
aiohttp.errors.ClientDisconnectedError
The client shows a message as:
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x7f67ec0f0588>
The code in the handler is:
#asyncio.coroutine
def sync(self, request):
ws = web.WebSocketResponse()
yield from ws.prepare(request)
# while True:
msg = yield from ws.receive()
if msg.tp == aiohttp.MsgType.text:
payload = msg.data
pypayload = json.loads(payload)
result = {'result': {}}
for store in pypayload:
try:
sync_obj = yield from asyncio.async(self.prepare(store))
except (IndexError, TypeError, ValidationError) as exc:
yield from asyncio.async(self.handle_internal_error(exc, store))
else:
try:
sync_result, request_type = yield from asyncio.async(self.send_data(sync_obj))
except DuplicateMappingsFound as exc:
yield from asyncio.async(self.handle_internal_error(exc, store))
else:
if sync_result.status == 200 and request_type == 'post':
yield from asyncio.async(self.process_data(sync_result))
elif sync_result.status >= 400:
yield from asyncio.async(self.handle_error(sync_result, sync_obj))
result['result'].update(
{store['store_id']: sync_result.status}
)
yield from asyncio.async(sync_result.release())
ws.send_str(json.dumps(result))
elif msg.tp == aiohttp.MsgType.error:
print('ws connection closed with exception {0}'.format(ws.exception()))
yield from asyncio.async(ws.close())
print('websocket connection closed')
return ws
The client code is:
#asyncio.coroutine
def sync_store():
resp = yield from aiohttp.get('http://localhost/stores/search')
stores = yield from resp.json()
total_page = stores['page']['total_page']
page = stores['page']['current_page']
total_resp = []
ws_sockets = []
while True:
for page in range(page, total_page):
session = aiohttp.ClientSession()
ws = yield from asyncio.async(session.ws_connect('ws://localhost:8765/stores'))
ws_sockets.append(ws)
ws.send_str(json.dumps(stores['data']))
resp = yield from asyncio.async(ws.receive())
total_resp.append(resp.data)
# print(resp)
stores_resp = yield from asyncio.async(aiohttp.post('http://localhost/stores/search',
data=json.dumps({'page': page+1}),
headers={'content-type': 'application/json'}
))
stores = yield from asyncio.async(stores_resp.json())
while ws_sockets:
session = ws_sockets.pop(0)
msg = yield from session.receive()
if not(msg.tp == aiohttp.MsgType.close or msg.tp == aiohttp.MsgType.closed):
ws_sockets.append(session)
else:
print(ws_sockets)
break
print(total_resp)
What could be the problem with this?
I also tried enabling the debugging mode but that also doesn't seem to give any useful output.

Related

Traceback (most recent call last): File "main.py", line 135, in <module> run_bot(auth['bot_token'], vk_session) help error code python download music

code error
Traceback (most recent call last):
File "main.py", line 135, in <module>
run_bot(auth['bot_token'], vk_session)
File "main.py", line 112, in run_bot
bot.polling()
File "/usr/local/lib/python3.8/dist-packages/telebot/__init__.py", line 514, i n polling
self.__threaded_polling(none_stop, interval, timeout, long_polling_timeout)
File "/usr/local/lib/python3.8/dist-packages/telebot/__init__.py", line 573, i n __threaded_polling
raise e
File "/usr/local/lib/python3.8/dist-packages/telebot/__init__.py", line 536, i n __threaded_polling
self.worker_pool.raise_exceptions()
File "/usr/local/lib/python3.8/dist-packages/telebot/util.py", line 117, in ra ise_exceptions
raise self.exception_info
File "/usr/local/lib/python3.8/dist-packages/telebot/util.py", line 69, in run
task(*args, **kwargs)
File "main.py", line 37, in send_text
if users_data[message.chat.id]['mode'] == 'search':
KeyError: 1413951044
code
import itertools
import requests
import configparser
import telebot
import vk_api
from vk_api.audio import VkAudio
import logging
def run_bot(token, vk_session):
users_data = {}
bot = telebot.TeleBot(token)
vk_audio = VkAudio(vk_session)
#bot.message_handler(commands=['search', 'get'])
def start_message(message):
users_data[message.chat.id] = {}
command = message.text.split()[0][1:]
if command == 'search':
users_data[message.chat.id]['mode'] = 'search'
bot.send_message(message.chat.id, "how to search?")
elif command == 'get':
users_data[message.chat.id]['mode'] = 'get'
try:
users_data[message.chat.id]['user_id'] = message.text.split()[1]
users_data[message.chat.id]['page'] = 0
get(message)
except:
bot.send_message(message.chat.id, "error id (how - /get id)")
#bot.message_handler(content_types=['text'])
def send_text(message):
if users_data[message.chat.id]['mode'] == 'search':
search(message)
#bot.callback_query_handler(func=lambda call: True)
def callback_worker(call):
if users_data[call.message.chat.id]['mode'] == 'search':
download(call)
elif users_data[call.message.chat.id]['mode'] == 'get':
if call.data == 'back':
users_data[call.message.chat.id]['page'] -= 5
get(call.message, call.message.message_id)
elif call.data == 'forward':
users_data[call.message.chat.id]['page'] += 5
get(call.message, call.message.message_id)
else:
download(call)
def download(call):
data = users_data[call.message.chat.id][call.data]
try:
print("loading")
bot.send_message(call.message.chat.id, "loading")
open('tmp.mp3', 'wb').write(requests.get(data['url'], allow_redirects=True, verify=False).content)
audio = open('tmp.mp3', "rb")
bot.send_message(call.message.chat.id, "sending")
print("sending")
bot.send_audio(call.message.chat.id, audio, performer=data['artist'], title=data['title'])
except Exception as exc:
print("error")
bot.send_message(call.message.chat.id, exc)
def get_audio_name(audio):
m, s = divmod(audio["duration"], 60)
track_name = "{} - {} ({}:{:0<2})".format(audio["artist"], audio["title"], m, s)
return track_name
def search(message):
print("Message received:", message.text)
items = [it for it in vk_audio.search(message.text, 5)]
if len(items) != 0:
print("On request found", len(items))
keyboard = telebot.types.InlineKeyboardMarkup()
for item in items:
users_data[message.chat.id][str(items.index(item))] = {'artist': item["artist"], 'title': item["title"],
'url': item["url"]}
key = telebot.types.InlineKeyboardButton(text=get_audio_name(item),
callback_data=str(items.index(item)))
keyboard.add(key)
bot.send_message(message.chat.id, text="What are we going to download?", reply_markup=keyboard)
else:
print("Nothing was found on the request")
bot.send_message(message.chat.id, "Nothing was found on the request")
print("Конец")
def get(message, id_msg_to_edit=None):
try:
it = vk_audio.get_iter(users_data[message.chat.id]['user_id'])
itt = itertools.islice(it, users_data[message.chat.id]['page'], users_data[message.chat.id]['page'] + 5)
keyboard = telebot.types.InlineKeyboardMarkup()
costil = 0
for i in itt:
users_data[message.chat.id][str(costil)] = i
keyboard.add(telebot.types.InlineKeyboardButton(text=get_audio_name(i), callback_data=str(costil)))
costil += 1
if users_data[message.chat.id]['page'] != 0:
keyboard.add(telebot.types.InlineKeyboardButton(text="Back", callback_data="back"))
if costil != 4:
keyboard.add(telebot.types.InlineKeyboardButton(text="Forward", callback_data="forward"))
if id_msg_to_edit is None:
bot.send_message(message.chat.id, text='What will we download?', reply_markup=keyboard)
else:
bot.edit_message_reply_markup(message.chat.id, id_msg_to_edit, reply_markup=keyboard)
except Exception as exc:
bot.send_message(message.chat.id, text=exc)
bot.polling()
What could be the problem here?
message.chat.id at line 37 is resolved to value 1413951044 and this key is not present in users_data dictionary, hence KeyError is raised.
If you just want to get rid of this error, enclose line 37 in a try/except block like below:
try:
if users_data[message.chat.id]['mode'] == 'search':
except KeyError:
pass

Python Django LineBot Error 54 'Connection reset by peer'

I'm trying to crete linebot through python Django, I want to send some messages and let it scrape the website. Since there is form on the website, I use post request to send the form.
Although I scrape the data successfully, there is error message showed in the python. It seems linebot occupy post method in Django and I send another post request again. I'm not sure my understanding is correct. I can't find any solution about this. Could someone teach me how to fix it?
Exception happened during processing of request from ('127.0.0.1', 50246)
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.8/socketserver.py", line 650, in process_request_thread
self.finish_request(request, client_address)
File "/usr/local/anaconda3/lib/python3.8/socketserver.py", line 360, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/local/anaconda3/lib/python3.8/socketserver.py", line 720, in __init__
self.handle()
File "/usr/local/anaconda3/lib/python3.8/site-packages/django/core/servers/basehttp.py", line 174, in handle
self.handle_one_request()
File "/usr/local/anaconda3/lib/python3.8/site-packages/django/core/servers/basehttp.py", line 182, in handle_one_request
self.raw_requestline = self.rfile.readline(65537)
File "/usr/local/anaconda3/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 54] Connection reset by peer
Below is my code, I receive some keyword and post request to website. Finally, reply to user
#csrf_exempt
def callback(request):
if request.method == 'POST':
signature = request.META['HTTP_X_LINE_SIGNATURE']
body = request.body.decode('utf-8')
content = "None"
try:
events = parser.parse(body, signature)
except InvalidSignatureError:
return HttpResponseForbidden()
except LineBotApiError:
return HttpResponseBadRequest()
for event in events:
if isinstance(event, MessageEvent):
msg = event.message.text.strip()
if msg.startswith('!'):
msg = msg.replace('!', '')
if msg == 'temp':
content = "HELP"
elif msg.startswith(' '):
content = 'Command not found'
elif ' ' in msg:
info = msg.split(' ')
if len(info) > 2:
content = 'Too many arguments'
else:
ID = info[1]
temp = temperature.TempReport(ID)
content = temp.scrape()
#content = 'Test'
else:
content = 'Unknown command'
line_bot_api.reply_message(
event.reply_token,
TextSendMessage(text=content)
)
print('submit success')
else:
line_bot_api.reply_message(
event.reply_token,
TextSendMessage(text=msg)
)
return HttpResponse()
else:
return HttpResponseBadRequest()
Here is scrape code
def scrape(self):
if not self.postToServer():
return f'ID:{self.ID} submit temperature fail!'
return f'ID:{self.ID} submit temperature successfully
def postToServer(self):
self.session.post(self.url_in, data=self.payload)
sleep(0.1)
resp = self.session.get(self.url_out)
sleep(0.1)
soup = BeautifulSoup(resp.text, features='lxml')
result = soup.find(class_='title-text').text.strip()
return 0 if not 'completed' in result else
The post request work fine and the error happened when I return HttpResponse in callback function. I don't know what is the issue here...

Python OS polling select

I am following the David Beazley Python Concurrency From the Ground Up: LIVE! - PyCon 2015 talk where I aim to understand basics of Python concurrency.
I would like to know why Python select is failing which is used for polling the operating system (OS) and check if there is some task that needs to be done.
from socket import *
from select import select
from collections import deque
tasks = deque()
recv_wait = {}
send_wait = {}
def fib(n):
if n <= 2:
return 1
else:
return fib(n-1)+fib(n-2)
def run():
while any([tasks, recv_wait, send_wait]):
while not tasks:
can_recv, can_send, _ = select(recv_wait, send_wait, [])
for s in can_recv:
tasks.append(recv_wait.pop(s))
for s in can_send:
tasks.append(send_wait.pop(s))
task = tasks.popleft()
try:
why, what = next(task)
if why == 'recv':
recv_wait[what] = task
elif why == 'send':
send_wait[what] = task
else:
raise RuntimeError("Arg!!")
except StopIteration:
print("task done")
def fib_server(address):
sock = socket(AF_INET, SOCK_STREAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind(address)
sock.listen(5)
while True:
yield 'recv', sock
client, addr = sock.accept()
print("Connection", addr)
tasks.append(fib_handler(client))
def fib_handler(client):
while True:
yield 'recv', client
req = client.recv(100)
if not req:
break
n = int(req)
result = fib(n)
result = fib(n)
resp = str(result).encode('ascii') + b'\n'
yield 'send',resp
client.send(resp)
print("Closed")
tasks.append(fib_server(('', 25000)))
run()
# Separate terminal window
nc localhost 25000
12
# Running Python server
➜ python3 -i aserver.py
Connection ('127.0.0.1', 61098)
Traceback (most recent call last):
File "aserver.py", line 63, in <module>
run()
File "aserver.py", line 20, in run
can_recv, can_send, _ = select(recv_wait, send_wait,[])
TypeError: argument must be an int, or have a fileno() method.
def fib_handler(client):
while True:
yield 'recv', client
req = client.recv(100)
if not req:
break
n = int(req)
result = fib(n)
result = fib(n) # duplicate
resp = str(result).encode('ascii')+ b'\n'
yield 'send',resp # should be yielding fd (client)
client.send(resp)
print("Closed")

Keyboard Interrupting an asyncio.run() raises CancelledError and keeps running the code

I have inspected this SO question on how to gracefully close out the asyncio process. Although, when I perform it on my code:
async def ob_main(product_id: str, freq: int) -> None:
assert freq >= 1, f'The minimum frequency is 1s. Adjust your value: {freq}.'
save_loc = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data', 'ob', product_id)
while True:
close = False
try:
full_save_path = create_save_location(save_loc)
file = open(full_save_path, 'a', encoding='utf-8')
await ob_collector(product_id, file)
await asyncio.sleep(freq)
except KeyboardInterrupt:
close = True
task.cancel()
loop.run_forever()
task.exception()
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
print(error_msg)
logger.warning(f'[1]-Error encountered collecting ob data: {error_msg}')
finally:
if close:
loop.close()
cwow()
exit(0)
I get the following traceback printed in terminal:
^C['Traceback (most recent call last):\n', ' File "/anaconda3/lib/python3.7/asyncio/runners.py", line 43, in run\n return loop.run_until_complete(main)\n', ' File "/anaconda3/lib/python3.7/asyncio/base_events.py", line 555, in run_until_complete\n self.run_forever()\n', ' File "/anaconda3/lib/python3.7/asyncio/base_events.py", line 523, in run_forever\n self._run_once()\n', ' File "/anaconda3/lib/python3.7/asyncio/base_events.py", line 1722, in _run_once\n event_list = self._selector.select(timeout)\n', ' File "/anaconda3/lib/python3.7/selectors.py", line 558, in select\n kev_list = self._selector.control(None, max_ev, timeout)\n', 'KeyboardInterrupt\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "coinbase-collector.py", line 98, in ob_main\n await asyncio.sleep(freq)\n', ' File "/anaconda3/lib/python3.7/asyncio/tasks.py", line 564, in sleep\n return await future\n', 'concurrent.futures._base.CancelledError\n']
and the code keeps running.
task and loop are the variables from the global scope, defined in the __main__:
loop = asyncio.get_event_loop()
task = asyncio.run(ob_main(args.p, 10))
Applying this question's method solves the issue. So in the above case:
try:
loop.run_until_complete(ob_main(args.p, 10))
except KeyboardInterrupt:
cwow()
exit(0)
However, I do not uderstand why that works.

AttributeError: 'module' object has no attribute 'ensure_future'

Hi i am writing a n/w bound server application using python asyncio which can accept a post request.
In post request i am accepting a symbol parameter
please tell me the best way to deal with n/w bound application.where i am collecting the data from another web api's by sending the post request to them.
Following is the code :
import asyncio
import aiohttp
import json
import logging
# async def fetch_content(url, symbols):
# yield from aiohttp.post(url, symbols=symbols)
#asyncio.coroutine
def fetch_page(writer, url, data):
response = yield from aiohttp.post(url, data=data)
resp = yield from response.read_and_close()
print(resp)
writer.write(resp)
return
#asyncio.coroutine
def process_payload(writer, data, scale):
tasks = []
data = data.split('\r\n\r\n')[1]
data = data.split('\n')
data = [x.split(':') for x in data]
print(data)
data = {x[0]: x[1] for x in data}
print(data)
# data = data[0].split(':')[1]
data = data['symbols']
print(data)
data = data.split(',')
data_len = len(data)
data_first = 0
data_last = scale
url = 'http://xxxxxx.xxxxxx.xxx/xxxx/xxxx'
while data_last < data_len:
tasks.append(asyncio.ensure_future(fetch_page(writer, url,{'symbols': ",".join(data[data_first:data_last])})))
data_first += scale
data_last += scale
tasks.append(asyncio.ensure_future(fetch_page(writer, url,{'symbols': ",".join(data[data_first:data_last])})))
loop.run_until_complete(tasks)
return
#asyncio.coroutine
def process_url(url):
pass
#asyncio.coroutine
def echo_server():
yield from asyncio.start_server(handle_connection, 'xxxxxx.xxxx.xxx', 3000)
#asyncio.coroutine
def handle_connection(reader, writer):
data = yield from reader.read(8192)
if data:
message = data.decode('utf-8')
print(message)
yield from process_payload(writer, message, 400)
writer.write_eof()
writer.close()
#url = 'http://XXXXXXX.xxxxx.xxx/xxxx/xxxxxx/xxx'
data = {'symbols': 'GD-US,14174T10,04523Y10,88739910,03209R10,46071F10,77543110,92847N10'}
loop = asyncio.get_event_loop()
loop.run_until_complete(echo_server())
try:
loop.run_forever()
finally:
loop.close()
But i am receiving the following error:
future: <Task finished coro=<handle_connection() done, defined at fql_server_async_v2.py:53> exception=AttributeError("'module' object has no attribute 'ensure_future'",)>
Traceback (most recent call last):
File "/home/user/anugupta/lib/python3.4/asyncio/tasks.py", line 234, in _step
result = coro.send(value)
File "fql_server_async_v2.py", line 60, in handle_connection
yield from process_payload(writer, message, 400)
File "/home/user/anugupta/lib/python3.4/asyncio/coroutines.py", line 141, in coro
res = func(*args, **kw)
File "fql_server_async_v2.py", line 41, in process_payload
tasks.append(asyncio.ensure_future(fetch_page(writer, url, {'symbols':",".join(data[data_first:data_last])})))
AttributeError: 'module' object has no attribute 'ensure_future'
^CTraceback (most recent call last):
File "fql_server_async_v2.py", line 72, in <module>
loop.run_forever()
File "/home/user/anugupta/lib/python3.4/asyncio/base_events.py", line 236, in run_forever
self._run_once()
File "/home/user/anugupta/lib/python3.4/asyncio/base_events.py", line 1017, in _run_once
event_list = self._selector.select(timeout)
File "/home/user/anugupta/lib/python3.4/selectors.py", line 424, in select
fd_event_list = self._epoll.poll(timeout, max_ev)
ensure_future was added in asyncio 3.4.4, use async for earlier versions.
While async is deprecated now it will be supported in oversable future.

Resources