"delete_message" not working on sqs client under moto - python-3.x

I have a code like:
class SqsClientWrapper:
def __init__(self, sqs_client, queue_url):
self.sqs_client = sqs_client
self.queue_url = queue_url
def receive_message(self):
try:
while True:
response = self.sqs_client.receive_message(QueueUrl=self.queue_url, MaxNumberOfMessages=1)
if len(response.get("Messages", [])) > 0:
return response["Messages"][0]
except ClientError as e:
raise e
def delete_message(self, receipt_handle):
try:
response = self.sqs_client.delete_message(
QueueUrl=self.queue_url, ReceiptHandle=receipt_handle
)
except ClientError:
self.__logger.exception(
f"Could not delete the meessage from the - {self.__queue_url}."
)
raise
else:
return response
class Listener:
def listen(self, queue_url):
sqs_client = SqsClientWrapper(boto3.client("sqs"), queue_url)
while True:
try:
message = sqs_client.receive_message()
print(str(message))
sqs_client.delete_message(message["ReceiptHandle"])
except Exception as e:
continue
I'm trying to test the Listener using moto, I have the following test code:
logger = logging.getLogger()
class ListenerTest(unittest.TestCase):
def setUp(self) -> None:
self.executor = ThreadPoolExecutor()
self.futures = []
def tearDown(self) -> None:
for future in self.futures:
future.cancel()
self.executor.shutdown()
#mock_sqs
def test_simple(self):
sqs = boto3.resource("sqs")
queue = sqs.create_queue(QueueName="test-fake-queue")
queue_url = queue.url
listener = Listener()
self.futures.append(
self.executor.submit(listener.listen, queue_url)
)
sqs_client = boto3.client("sqs")
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=json.dumps({"id": "1234"}),
)
if __name__ == "__main__":
unittest.main()
When I run this, it seems to have received the message okay, but when deleting it from the queue it fails with the following error:
botocore.exceptions.ClientError: An error occurred (InvalidClientTokenId) when calling the DeleteMessage operation: The security token included in the request is invalid.
Anyone know what might be going on here? Thanks!

Related

Python continously receive message through websocket

I am trying to connect through websocket and keeping it alive to retrieve continously message from the server. I wrote the clientHelper and find the socketManager and ReconnectingWebsocket on the internet and I have not much idea on what going wrong with it as I do not receive anything thought the clientHelper.process_user_message function.
Can someone point me out the error please?
import websockets as ws
import asyncio
from client import Client
class ClientHelper(Client):
def __init__(self, api_key, api_secret):
super().__init__(api_key, api_secret)
self.loop = asyncio.get_event_loop()
def _request(self, method, uri, signed, force_params=False, **kwargs):
kwargs = self._get_request_kwargs(method, signed, force_params, **kwargs)
response = getattr(self.session, method)(uri, **kwargs)
return self._handle_response(response, method)
def _request(self, method, path, signed=False, version=API_VERSION, **kwargs):
uri = self._create_api_uri(path, signed, version)
return self._request(method, uri, signed, **kwargs)
def get_listen_key(self):
res = self._request('post', 'userDataStream', signed=True, data={})
return res['listenKey']
async def start_websockets(self):
self.sm = SocketManager(self, self.loop)
await self.sm.start_socket(self.process_user_message)
async def process_user_message(self, msg):
self.msg = msg
print(msg)
async def main(self)
await self.start_websockets()
while True:
await client.getInfo()
if 'data' in self.msg:
print(self.msg['data'])
def start(self):
self.loop.run_until_complete(self.main())
class SocketManager:
STREAM_URL = url
def __init__(self, client, loop, user_timeout=DEFAULT_USER_TIMEOUT):
self._client = client
self._loop = loop
self._conns = None
async def _start_user_socket(self, path, coro, prefix='ws/'):
if path in self._conns:
return False
self._conns[path] = ReconnectingWebsocket(self._loop, path, coro, prefix)
return path
async def start_user_socket(self, coro):
user_listen_key = await self._client.stream_get_listen_key() # manage to get the key from serveur
conn_key = await self._start_user_socket('user', user_listen_key, coro)
return conn_key
class ReconnectingWebsocket:
STREAM_URL = url
MAX_RECONNECTS = 5
MAX_RECONNECT_SECONDS = 60
MIN_RECONNECT_WAIT = 0.1
TIMEOUT = 10
def __init__(self, loop, path, coro, prefix='ws/'):
self._loop = loop
self._log = logging.getLogger(__name__)
self._path = path
self._coro = coro
self._prefix = prefix
self._reconnects = 0
self._conn = None
self._socket = None
self._connect()
def _connect(self):
self._conn = asyncio.ensure_future(self._run(), loop=self._loop)
async def _run(self):
keep_waiting = True
async with ws.connect(self.STREAM_URL) as socket:
self._socket = socket
self._reconnects = 0
try:
while keep_waiting:
try:
#evt = await self._coro(evt_obj)
evt = await asyncio.wait_for(self._socket.recv(), timeout=self.TIMEOUT)
except asyncio.TimeoutError:
#self._log.debug("no message in {} seconds".format(self.TIMEOUT))
print("no message in {} seconds".format(self.TIMEOUT))
await self.send_ping()
except asyncio.CancelledError:
#self._log.debug("cancelled error")
print("cancelled error")
await self.send_ping()
else:
try:
evt_obj = json.loads(evt)
except ValueError:
#self._log.debug('error parsing evt json:{}'.format(evt))
print('error parsing evt json:{}'.format(evt))
else:
await self._coro(evt_obj)
except ws.ConnectionClosed as e:
#self._log.debug('ws connection closed:{}'.format(e))
print('ws connection closed:{}'.format(e))
await self._reconnect()
except Exception as e:
#self._log.debug('ws exception:{}'.format(e))
print('ws exception:{}'.format(e))
await self._reconnect()
def _get_reconnect_wait(self, attempts: int) -> int:
expo = 2 ** attempts
return round(random() * min(self.MAX_RECONNECT_SECONDS, expo - 1) + 1)
async def _reconnect(self):
await self.cancel()
self._reconnects += 1
if self._reconnects < self.MAX_RECONNECTS:
reconnect_wait = self._get_reconnect_wait(self._reconnects)
await asyncio.sleep(reconnect_wait)
self._connect()
else:
self._log.error('Max reconnections {} reached:'.format(self.MAX_RECONNECTS))
async def send_ping(self):
if self._socket:
await self._socket.ping()
async def cancel(self):
self._conn.cancel()
self._socket = None

Error from callback function in python websocket: line 346, in _callback callback(self, *args)

I used the Gemini websocket code for a BTC feed but a callback error keeps popping up. However, it occasionally does run the code correctly. I have used 'price' and 'quantity' to test the code before I use the actual variables I want.
Error from running my gemini websocket code
Here is the code:
import websocket, ssl, json
import _thread as thread
websocket._logging._logger.level = -99
bestbid = {}
bestask = {}
top_of_book = 0
def on_message(self, message):
global bestbid, bestask
bestbid = json.loads(message)
bestask = json.loads(message)
print("Message received!")
print("{} {}".format(bestbid["price"], bestask ["quantity"]))
def on_error(self, error):
print(error)
def on_close(self):
print("Connection closed!")
def on_open(self):
print("Connection opened!")
def run(*args):
ws.send(logon_msg)
thread.start_new_thread(run, ())
if __name__ == "__main__":
logon_msg = '{"type": "subscribe","subscriptions":[{"name":"l2","symbols":["BTCUSD"]}]}'
websocket.enableTrace(True)
ws = websocket.WebSocketApp("wss://api.gemini.com/v2/marketdata/",
on_message = on_message,
on_error = on_error,
on_close = on_close,
on_open = on_open)
ws.on_open = on_open
ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
You should check that bestbid (bestbuy) JSON object has fields "price" and quantity". Errors most probably are caused by heartbeat or some diagnostic messages.

Custom exceptions in python starlette

I am trying to raise the custom exception using the starlette framework in python. I have the API call which checks some condtions depends on the result, it should raise exception.
I have two files app.py and error.py
#app.py
from starlette.applications import Starlette
from starlette.responses import JSONResponse
from starlette.routing import Route
from error import EmptyParamError
async def homepage(request):
a=1
b=0
if a == 1:
raise EmptyParamError(400, "status_code")
return JSONResponse({'hello': 'world'})
routes = [
Route("/", endpoint=homepage)
]
app = Starlette(routes=routes,debug=True)`
#error.py ```
from starlette.responses import JSONResponse
class BaseError(Exception):
def __init__(self, status_code: int, detail: str = None) -> None:
if detail is None:
detail = "http.HTTPStatus(status_code).phrase"
self.status_code = status_code
self.detail = detail
async def not_found(self):
return JSONResponse(content=self.title, status_code=self.status_code)
class EmptyParamError(BaseError):
""" Error is raised when group name is not provided. """
status_code = 400
title = "Missing Group Name"
When the condition is true, i want to raise the exception but its not returning the jsonrespnse but its returning the stacktrace on the console.
Please let me know if anything is wrong here
adding try block resolved the issue
try:
if a==1:
raise InvalidUsage(100,"invalid this one")
if b == 0:
raise EmptyParamError("this is empty paramuvi")
except InvalidUsage as e:
return JSONResponse({'hello': str(e.message)})
except EmptyParamError as e:
return JSONResponse({'hello': str(e.message)})

Sending many requests in one websocket connection without reconnect

My python program work so slow, because make socket reconnect for any request. I want make one connect and send request, with out reconnect
My functions in file send_by_socket.py, some other function and class call send_to_socket for send logmessage. Now it work, but very slow. Reason - make new connect for any message. I want single connection or poll for use it without reconnect. How make it, possibel have good example with sourcecode?
import asyncio
import websockets
from logging import StreamHandler
import json
async def async_send(message):
async with websockets.connect('wss://****.com/chat') as web_socket:
await web_socket.send(message)
class WebSocketHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
def emit(self, record):
msg = json.dumps({'log': {'message': record.message, 'date': record.asctime, 'level': record.levelname}})
try:
asyncio.get_event_loop().run_until_complete(async_send(msg))
except ConnectionRefusedError:
pass
def send_to_socket(msg_dict):
msg = json.dumps(msg_dict)
try:
asyncio.get_event_loop().run_until_complete(async_send(msg))
except ConnectionRefusedError:
pass
Now program spend about 1 - 1.2 sec for request. I try
con = websockets.connect('wss://****.com/chat')
con.send('some thing')
but have error AttributeError: 'Connect' object has no attribute 'send'
python
import asyncio
import websockets
from logging import StreamHandler
import json
import time
def singleton(cls):
instances = {}
def getinstance():
if cls not in instances:
instances[cls] = cls()
return instances[cls]
return getinstance
#singleton
class SendToWebSocket:
"""
Send message in web-socket, use one connection for sending.
Try make new connection, if old is lost.
"""
__ws = None
__url = "wss://***.com/chat"
def __init__(self):
self.retryTime = 0
self.retryRepeat = 30
self.__create_connect()
#asyncio.coroutine
def __create_connect(self):
if (time.time() - self.retryTime) > self.retryRepeat:
try:
self.__ws = yield from websockets.connect(self.__url)
self.retryTime = 0
except ConnectionRefusedError:
self.retryTime = time.time()
def send(self, message):
t = type(message)
if t is dict:
msg = json.dumps(message)
elif t is str:
msg = message
else:
raise ValueError("Message must be str or dict. Received %s" % type(t))
if self.__ws is not None:
try:
asyncio.get_event_loop().run_until_complete(self.__async_send(msg))
# print('Send normal')
except ConnectionRefusedError:
# print("Can't send")
# try recreate connect
self.__create_connect()
else:
asyncio.get_event_loop().run_until_complete(self.__create_connect())
async def __async_send(self, message):
await self.__ws.send(message)
class WebSocketHandler(StreamHandler):
"""Custom handler for logging library"""
def __init__(self):
StreamHandler.__init__(self)
self.web_socket = SendToWebSocket()
def emit(self, record):
msg = json.dumps({'log': {'message': record.message, 'date': record.asctime, 'level': record.levelname}})
try:
self.web_socket.send(msg)
except ConnectionRefusedError:
pass

'Connection' object is not callable while using sqlalchemy in python

I am getting
Connection' object is not callable
error while executing my python code in which I am using sqlalchemy package.
Code which is not working
import sys
import pymysql
import sqlalchemy.pool as pool
from init import db_config
DEBUG="True"
def _get_connection(**kwargs):
try:
return pymysql.Connection(**kwargs);
except pymysql.OperationalError as e:
if DEBUG=="True": print("Exception catched %s" %e.message)
return None
def connect_db(kwargs=db_config):
try:
conn = _get_connection(**kwargs)
if not conn == None:
mypool = pool.QueuePool(conn, max_overflow=1, pool_size=1)
print("====Type of pool", type(mypool))
local_conn = mypool.connect()
print("LocalConn object given: ", local_conn)
return local_conn
else:
print("conn is None")
except TypeError as e:
if DEBUG=="True": print("Exception catched during queue pooling:", e)
return None
def disconnect_db(dbh):
dbh.close()
def fetch_rows(query):
query = str(query)
if not query:
LOGGER
if DEBUG=="True": print("Input query is empty")
return None
rows=[]
local_conn = connect_db();
if not local_conn == None:
try:
print("fetch_rows.local_conn:", local_conn);
with local_conn.cursor() as cursor:
cursor.execute(query);
rows = cursor.fetchall();
except pymysql.OperationalError as e:
if DEBUG=="True": print("FetchRows excep:", e)
finally:
disconnect_db(local_conn)
else:
print("fetch_rows.local_conn is None");
return rows
Output:
Exception catched during queue pooling: 'Connection' object is not callable
fetch_rows.local_conn is None

Resources