I try to to collect table async with PySnmp, but example give back only first record.
Could anyone suggest what is wrong?
import asyncio
from pysnmp.hlapi.asyncio import *
#asyncio.coroutine
def run():
snmp_engine=SnmpEngine()
count=0
while True:
errorIndication, errorStatus, errorIndex, varBinds = yield from nextCmd(
snmp_engine,
CommunityData('public'),
UdpTransportTarget(('demo.snmplabs.com', 161)),
ContextData(),
ObjectType(ObjectIdentity('SNMPv2-MIB', 'system')),
)
for next_var in varBinds:
print(next_var[0])
count+=1
#break
if count > 10:
break
asyncio.get_event_loop().run_until_complete(run())
I have experienced same issue recently. I think the asynchronous nextCmd is not working like the sync nextCmd. You have to update the oid in each iteration. This code may help:
import asyncio
from pysnmp.hlapi.asyncio import *
def inc_oid(oid):
_oid = oid.split(".")
lastint = int(_oid[-1])
_oid[-1] = str(lastint + 1)
return ".".join(_oid)
snmp_engine=SnmpEngine()
#asyncio.coroutine
def snmp_walk_async(oid):
endoid = inc_oid(oid)
while True:
errorIndication, errorStatus, errorIndex, varBinds = yield from nextCmd(
snmp_engine,
CommunityData("public"),
UdpTransportTarget(("demo.snmplabs.com", 161)),
ContextData(),
ObjectType(ObjectIdentity(oid)),
lexicographicMode=False,
)
#Some Error Checkings
varBind = varBinds[0][0]
oid, value = varBind
oid = str(oid)
if oid >= endoid:
break
for varBind in varBinds:
print(' = '.join([x.prettyPrint() for x in varBind]))
asyncio.get_event_loop().run_until_complete(snmp_walk_async("1.3.6.1.2.1.1"))
If you read about how to query SNMP tables from a standard SNMP book, you will see that it was designed to be a few sync calls (GET NEXT or GET BULK), so you cannot run them in parallel.
But if you change your code to generate GET BULK requests, it will be a lot faster than generating GET NEXT requests.
Related
Currently I am working on a project which involves usage of Asynchronous functions, due to the usage of certain set of libraries. My code runs fine as long as I don't integrate a web-socket server implementing functionality in my code.
But, I wish to stream the output 'Result' continuously in a websocket stream. So, I tried integrating websocket from socketio library as an AsyncServer.
Firstly, in my code, I want to gather all my inputs, and keep displaying the possible Result in a terminal. Once my inputs are finalized, I wish my result to be streamed over Websocket.
Initially, I just tried using web.run_app() in an asynchronous task in the main thread. Refer code below with #Type-1 comments. (Make sure that the lines with comment #Type-2 should be commented out). But I get the following exception "This event loop is already running".
I thought maybe if I run web.run_app() in a separate thread, then this issue might not come up. So, I changed my implementation slightly. Refer code below with #Type-2 comments. (Make sure that the lines with comment #Type-1 should be commented out). Now, I get another issue "set_wakeup_fd only works in main thread of the main interpreter".
Can someone please help me solve this issue, and let me know how must I use web.run_app()?
Here is the code:
import os, sys
import asyncio
import platform
import threading
import socketio
import json
from aioconsole import ainput
from aiohttp import web
from array import *
Result = -1
Inputs_Required = True
Input_arr = array('i')
sio = socketio.AsyncServer()
app = web.Application()
sio.attach(app)
Host = "192.168.0.7"
Port = 8050
async def IOBlock():
global Input_arr
global Inputs_Required
while(True):
response = input("Enter new input? (y/n): ")
if('y' == response or 'Y' == response):
Input = input("Enter number to be computed: ")
Input_arr.append(int(Input))
break
elif('n' == response or 'N' == response):
Inputs_Required = False
break
else:
print("Invalid response.")
async def main():
global Results
global Inputs_Required
global Input_arr
WebSocketStarted = False
#WebSocketThread = threading.Thread(target = WebStreaming, daemon = True) #Type-2
try:
while True:
if(Inputs_Required == True):
Task_AddInput = asyncio.create_task(IOBlock())
await Task_AddInput
elif (WebSocketStarted == False):
WebSocketStarted = True
#WebSocketThread.start() #Type-2
WebTask = asyncio.create_task(WebStreaming()) #Type-1
await WebTask #Type-1
if(len(Input_arr) > 0):
Task_PrintResult = asyncio.create_task(EvaluateResult())
await Task_PrintResult
except Exception as x:
print(x)
finally:
await Cleanup()
async def WebStreaming(): #Type-1
#def WebStreaming(): #Type-2
print("Starting web-socket streaming of sensor data..")
Web_loop = asyncio.new_event_loop #Type-1 or 2
asyncio.set_event_loop(Web_loop) #Type-1 or 2
web.run_app(app, host=Host, port=Port)
async def EvaluateResult():
global Input_arr
global Result
Result = 0
for i in range (0, len(Input_arr)):
Result += Input_arr[i]
print(f"The sum of inputs fed so far = {Result}.")
await asyncio.sleep(5)
async def Cleanup():
global Input_arr
global Inputs_Required
global Result
print("Terminating program....")
Result = -1
Inputs_Required = True
for i in reversed(range(len(Input_arr))):
del Input_arr[i]
#sio.event
async def connect(sid, environ):
print("connect ", sid)
#sio.event
async def OnClientMessageReceive(sid, data):
global Result
print("Client_message : ", data)
while True:
msg = json.dumps(Result)
print(msg)
await sio.send('OnServerMessageReceive', msg)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
if __name__ == "__main__":
asyncio.run(main())
I want to initiate a PULL in a port and want to receive from other ports to my PULL port. In the case of a PULL port, it listens asynchronously and when it receives a message, it just prints the message in the console. So for that I have written a method inside a Push-class, which will send the message to the PULL port.
My code is as follows :
import random
import zmq
import time
import sys
import string
import asyncio
import zmq.asyncio
class Push():
def __init__(self, port, addr='localhost'):
self.port = port
self.addr = addr
self.ctx = zmq.Context()
self.scoket = self.ctx.socket(zmq.PUSH)
self.scoket.connect(f'tcp://{self.addr}:{selfa.port}')
def send(self):
chars = string.ascii_uppercase + string.ascii_lowercase
message = ''.join(random.choice(chars) for _ in range(4))
self.scoket.send(bytes(message, 'utf-8'))
print(f'sending: {message}')
class Pull():
def __init__(self, port, addr='*'):
self.port = port
self.addr = addr
self.ctx = zmq.Context()
self.socket = self.ctx.socket(zmq.PULL)
self.socket.bind(f'tcp://{self.addr}:{self.port}')
async def listen(self, listener):
while True:
string = await self.socket.recv()
listener(string)
if __name__ == '__main__':
push = Push('55501')
async def send():
while True:
await asyncio.sleep(5)
print('Sending...')
push.send()
pull = Pull('55501')
try:
asyncio.run(
pull.listen(print),
send(),
)
except KeyboardInterrupt:
print('exiting...')
exit()
The above code is not running. The code stops at the listen method.
#ADAPTED FROM PYMATA EXPRESS EXAMPLE CONCURRENTTAKS
#https://github.com/MrYsLab/pymata-express/
import asyncio
import zmq
import json
import zmq.asyncio as zmq_asyncio
from pymata_express.pymata_express import PymataExpress
class ConcurrentTasks:
def __init__(self, board):
self.loop = board.get_event_loop()
self.board = board
self.ctxsync = zmq.Context()
self.context = zmq.asyncio.Context()
self.rep = self.context.socket(zmq.REP)
self.rep.bind("tcp://*:5558")
self.trigger_pin = 53
self.echo_pin = 51
loop.run_until_complete(self.async_init_and_run())
### START: NEW CODE THAT RESOLVED THE ISSUE
async def pingsonar(self):
value = await self.board.sonar_read(self.trigger_pin)
return value
async def readsonar(self):
while True:
rep_recv = await self.rep.recv()
value = await asyncio.wait([self.pingsonar()])
valuesonar = list(value[0])[0].result()
json_data = json.dumps(valuesonar)
await self.rep.send(json_data.encode())
await asyncio.sleep(1 / 1000) #maybe this line isn't necessary
### END : NEW CODE THAT RESOLVED THE ISSUE
async def async_init_and_run(self):
await self.board.set_pin_mode_sonar(self.trigger_pin, self.echo_pin)
readsonar = asyncio.create_task(self.readsonar())
await readsonar
# OTHER CREATED_TASK GO HERE, (removed them in the MVE, but they work fine)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
my_board = PymataExpress()
try:
ConcurrentTasks(my_board)
except (KeyboardInterrupt, RuntimeError):
loop.run_until_complete(my_board.shutdown())
print('goodbye')
finally:
loop.close()
The above code is not running.
The code is running,yet there are mistakes ( in concept ) and typos in the source code.
The code as-is inhibits the Push-class from ever become .connect()-ed, thus the Pull-counterparty .bind()-s, yet has nobody to talk to.
SOLUTION
1 )Repair this typo ( + better, explicitly detect and handle all potential error-states )
self.scoket.connect(f'tcp://{self.addr}:{selfa.port}') # this will NEVER FLY
#--------------------------------------------^
self.scoket.connect(f'tcp://{self.addr}:{self.port}') # this will ... ( + detect Error-state(s)
2 )Correct the concept - mixing low-hanging fruits ( several asynchronous frameworks used at once, one working against all the others ) is a sign of shallow understanding of the responsibility of professional engineering in reliable and robust-enough distributed-computing ( flaws in the concept of control, like during landing Apollo-11 on Moon or, on the contrary, the Chernobyl-kind of mis-management system + mentality, are clean, inspiring and warning enough examples how bad practices ( if not prohibited + detected + corrected + penalised ) can & will harm again and again and again ).
The Best Next Step
If you are keen to get to the professional-level, start with Pieter Hintjens' book "Code Connected, Volume 1" - worth time, worth efforts, worth understanding their discussed concepts.
I am trying to get websockets, asyncio and multiprocess to work together. I have been stuck on this for 2 days and could appreciate some help.
I have searched for websockets asyncio and multiprocessing on stackoverflow as well as general internet searches. I have found threading examples, which I can make work.
import asyncio
import websockets
import threading
class Connection():
def __init__(self):
self.loop = asyncio.new_event_loop()
sock_thread = threading.Thread(target=self.new_loop)
sock_thread.start()
self.x = 0
async def connect_to_socket(self):
self.websocket = await websockets.connect('ws://demos.kaazing.com/echo')
await self.websocket.send("hello")
response = await self.websocket.recv()
print(response)
async def listen_to_socket(self):
while True:
await asyncio.sleep(0)
print('Listening for a message...')
while self.x < 5:
message = await self.websocket.recv()
print("< {}".format(message))
print('\n\n')
print(self.x)
self.x += 1
self.task.cancel()
self.loop.close()
def stop(self):
print('canceling task\n\n')
self.x = 0
self.task.cancel()
def new_loop(self):
self.task = self.loop.create_task(self.connect_to_socket())
self.loop.run_forever()
def make_task(self):
self.task = self.loop.create_task(self.listen_to_socket())
if __name__ == '__main__':
conn=Connection()
This works with no issues. I have seen examples where multiprocessing opens a process in an event loop, this is not what I want. I want to ope However, this is not what I want. I want to open a new process and run an event loop in the new process. Inside the event loop, I want to run my sockets. I want to free my main process from listening to sockets and use a child process to listen to the sockets while I do computationally expensive work on my main process.
When I try the following code. I get nothing.
import asyncio
import websockets
import multiprocessing
class Connection(multiprocessing.Process):
def __init__(self, tasks, results):
super().__init__()
self.tasks = tasks
self.results = results
self.loop = asyncio.new_event_loop()
print('create event loop')
self.x = 0
self.task = self.loop.create_task(self.test())
print('done with connecting')
#connect to socket and get response
async def test(self):
self.ws = await websockets.connect('ws://demos.kaazing.com/echo')
await self.websocket.send("hello")
response = await self.websocket.recv()
print(response)
#listen to socket long term after connection
async def listen_to_socket(self):
while True:
await asyncio.sleep(0)
print('Listening for a message...')
while self.x < 5:
await self.websocket.send("hello")
message = await self.websocket.recv()
print("< {}".format(message))
print('\n\n')
print(self.x)
self.x += 1
self.results.put(message)
self.task.cancel()
self.loop.close()
#stop task
def stop(self):
print('canceling task\n\n')
self.x = 0
self.task.cancel()
# listen to socket long term
#I have not called this as I can't even get a response from test()
def make_task(self):
self.task = self.loop.create_task(self.listen_to_socket())
if __name__ == '__main__':
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
process = Connection(tasks, results)
if tasks.empty():
print('empty')
else:
print(tasks.get())
I expect to connect with the socket and receive a response. However, I get nothing. I get no error messages,no printout from the connection, I get an empty queue and that's all. How do I get the return values from my websocket?
I am still new enough, I am not sure what I am doing wrong. Any advice would help me out.
Thank you
Anyone interested, I got this to work. It is very much a work in progress and I am adding to it, and since this is for me and relatively simple, I didn't comment it.
I started with the code from this answer and built on it.
Python3 Websockets / Multithreading issue
import asyncio
import websockets
import sys
import time
import multiprocessing
class connect():
def __init__(self, results, tasks):
self.x = 0
self.results = results
self.tasks = tasks
self.loop = asyncio.new_event_loop()
async def commander_start(self):
while not self.tasks.empty():
self.uri = self.tasks.get()
self.tasks.task_done()
self.ws = await websockets.connect(self.uri)
while True:
await asyncio.sleep(0.1)
print('Listening for a message...')
while self.x < 5:
await self.ws.send("hello")
message = await self.ws.recv()
message = message+str(self.x)
print("< {}".format(message))
print('\n\n')
print(self.x)
self.x += 1
self.results.put(message)
self.ws.close()
self.x = 0
print('ws clsed')
self.task.cancel()
await asyncio.sleep(1)
self.loop.close()
def run_commander(self):
self.task = self.loop.create_task(self.commander_start())
self.loop.run_forever()
def main(self):
self.commander = multiprocessing.Process(target=self.run_commander)
self.commander.start()
time.sleep(3)
self.commander.kill()
print('is alive:', self.commander, self.commander.is_alive())
if __name__ == "__main__":
size_q = 10
tasks = multiprocessing.JoinableQueue(maxsize=size_q)
results = multiprocessing.Queue(maxsize=size_q)
conn = connect(results,tasks)
tasks.put('ws://demos.kaazing.com/echo')
conn.main()
print('tasks2 put')
tasks.put('wss://echo.websocket.org')
conn.main()
if not results.empty():
for x in range(size_q):
print(results.get())
There is a bunch I am going to change and improve, I just wanted the base system to work so I could build from there, so that anyone that uses this will need to modify it to suit their needs. For instance, I spawn a new process and kill it, instead of running a continuous process and giving it work to do, I also am trying to figure out the specifics of the joinable queue and how to use it to add jobs after the process and event loop has been created.
I have a python 3.4.3, postgreSQL 9.4, aiopg-0.7.0. An example of multi-threaded applications, was taken from this site. How to use the pool? The thread hangs when the operation of the select.
import time
import asyncio
import aiopg
import functools
from threading import Thread, current_thread, Event
from concurrent.futures import Future
class B(Thread):
def __init__(self, start_event):
Thread.__init__(self)
self.loop = None
self.tid = None
self.event = start_event
def run(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
self.tid = current_thread()
self.loop.call_soon(self.event.set)
self.loop.run_forever()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
def add_task(self, coro):
"""this method should return a task object, that I
can cancel, not a handle"""
def _async_add(func, fut):
try:
ret = func()
fut.set_result(ret)
except Exception as e:
fut.set_exception(e)
f = functools.partial(asyncio.async, coro, loop=self.loop)
if current_thread() == self.tid:
return f() # We can call directly if we're not going between threads.
else:
# We're in a non-event loop thread so we use a Future
# to get the task from the event loop thread once
# it's ready.
fut = Future()
self.loop.call_soon_threadsafe(_async_add, f, fut)
return fut.result()
def cancel_task(self, task):
self.loop.call_soon_threadsafe(task.cancel)
#asyncio.coroutine
def test(pool, name_task):
while True:
print(name_task, 'running')
with (yield from pool.cursor()) as cur:
print(name_task, " select. ")
yield from cur.execute("SELECT count(*) FROM test")
count = yield from cur.fetchone()
print(name_task, ' Result: ', count)
yield from asyncio.sleep(3)
#asyncio.coroutine
def connect_db():
dsn = 'dbname=%s user=%s password=%s host=%s' % ('testdb', 'user', 'passw', '127.0.0.1')
pool = yield from aiopg.create_pool(dsn)
print('create pool type =', type(pool))
# future.set_result(pool)
return (pool)
event = Event()
b = B(event)
b.start()
event.wait() # Let the loop's thread signal us, rather than sleeping
loop_db = asyncio.get_event_loop()
pool = loop_db.run_until_complete(connect_db())
time.sleep(2)
t = b.add_task(test(pool, 'Task1')) # This is a real task
t = b.add_task(test(pool, 'Task2'))
while True:
time.sleep(10)
b.stop()
Not return result in 'yield from cur.execute("SELECT count(*) FROM test")'
Long story short: you cannot share aiopg pool object from different event loops.
Every aiopg.Pool is coupled to event loop. If you don't specify loop parameter explicitly it is taken from asyncio.get_event_loop() call.
So it your example you have a pool coupled to event loop from main thread.
When you execute db query from separate thread you trying to accomplish it by executing thread's loop, not the main one. It doesn't work.
I am trying to write a decorator, that takes a function which interacts with mongodb and if an exception occurs it retries the interaction.
I have the following code:
def handle_failover(f):
def wrapper(*args):
for i in range(40):
try:
yield f(*args)
break
except pymongo.errors.AutoReconnect:
loop = IOLoop.instance()
yield gen.Task(loop.add_timeout, time.time() + 0.25)
return wrapper
class CreateHandler(DatabaseHandler):
#handle_failover
def create_counter(self, collection):
object_id = yield self.db[collection].insert({'n': 0})
return object_id
#gen.coroutine
def post(self, collection):
object_id = yield self.create_counter(collection)
self.finish({'id': object_id})
But this doesn't work. It gives an error that create_counter yields a generator. I've tried making all the functions #gen.coroutines and it didn't help.
How can I make handle_failover decorator work?
edit:
No decorators for now. This should create a counter reliably and return object_id to the user. If exception is raised 500 page gets displayed.
class CreateHandler(DatabaseHandler):
#gen.coroutine
def create_counter(self, collection, data):
for i in range(FAILOVER_TRIES):
try:
yield self.db[collection].insert(data)
break
except pymongo.errors.AutoReconnect:
loop = IOLoop.instance()
yield gen.Task(loop.add_timeout, time.time() + FAILOVER_SLEEP)
except pymongo.errors.DuplicateKeyError:
break
else:
raise Exception("Can't create new counter.")
#gen.coroutine
def post(self, collection):
object_id = bson.objectid.ObjectId()
data = {
'_id': object_id,
'n': 0
}
yield self.create_counter(collection, data)
self.set_status(201)
self.set_header('Location', '/%s/%s' % (collection, str(object_id)))
self.finish({})
Although I still don't know how to make increment of the counter idempotent because the trick with DuplicateKeyError is not applicable here:
class CounterHandler(CounterIDHandler):
def increment(self, collection, object_id, n):
result = yield self.db[collection].update({'_id': object_id}, {'$inc': {'n': int(n)}})
return result
#gen.coroutine
def post(self, collection, counter_id, n):
object_id = self.get_object_id(counter_id)
if not n or not int(n):
n = 1
result = yield self.increment(collection, object_id, n)
self.finish({'resp': result['updatedExisting']})
You most likely don't want to do this. Better to show an error to your user than to retry an operation.
Blindly retrying any insert that raises AutoReconnect is a bad idea, because you don't know if MongoDB executed the insert before you lost connectivity or not. In this case you don't know whether you'll end up with one or two records with {'n': 0}. Thus you should ensure that any operation you retry this way is idempotent. See my "save the monkey" article for detailed information.
If you definitely want to make a wrapper like this, you need to make sure that f and wrapper are both coroutines. Additionally, if f throws an error 40 times you must re-raise the final error. If f succeeds you must return its return value:
def handle_failover(f):
#gen.coroutine
def wrapper(*args):
retries = 40
i = 0
while True:
try:
ret = yield gen.coroutine(f)(*args)
raise gen.Return(ret)
except pymongo.errors.AutoReconnect:
if i < retries:
i += 1
loop = IOLoop.instance()
yield gen.Task(loop.add_timeout, time.time() + 0.25)
else:
raise
return wrapper
But only do this for idempotent operations!