SNMP Agent using pysnmp in a multithreaded environment Timeout - multithreading

Question

I am trying to write an SNMP agent that I can use to monitor my python processes. To do so I have written a class that implements the SNMP agent using pysnmp.
The core part of this agent works (i.e. i can use snmpwalk to interrogate the agent and the returned data is correct). To allow me to update the agent MIB values I have run the dispatcher() within it's own thread. My problem is that I then get timeouts when talking to the agent using snmpwalk (snmpwalk correctly walks the MIB but then times out).
Does anyone have insight into what I'm doing wrong?
The agent code is as follows:
import logging
from pysnmp import debug
from pysnmp.carrier.asyncore.dgram import udp
from pysnmp.entity import engine, config
from pysnmp.entity.rfc3413 import cmdrsp, context
from pysnmp.smi import exval
import threading
formatting = '[%(asctime)s-%(levelname)s]-(%(module)s) %(message)s'
logging.basicConfig(level=logging.DEBUG, format=formatting, )
class SNMPAgent(object):
def _main(self):
logging.debug("Creating SNMP Agent....")
self._snmpEngine = engine.SnmpEngine()
config.addTransport(
self._snmpEngine,
udp.domainName,
udp.UdpTransport().openServerMode((self._agentHost, self._agentPort))
)
config.addV1System(self._snmpEngine, 'my-area', self._communityName)
config.addVacmUser(self._snmpEngine,
2,
'my-area',
'noAuthNoPriv',
(1, 3, 6),
(1, 3, 6))
snmpContext = context.SnmpContext(self._snmpEngine)
mibBuilder = snmpContext.getMibInstrum().getMibBuilder()
mibBuilder.loadModules('HOST-RESOURCES-MIB')
self._mibInstrum = snmpContext.getMibInstrum()
self._hostRunTable, = mibBuilder.importSymbols('HOST-RESOURCES-MIB', 'hrSWRunEntry')
self._instanceId = self._hostRunTable.getInstIdFromIndices(1)
# The following shows the OID name mapping
#
# hrSWRunTable 1.3.6.1.2.1.25.4.2 <TABLE>
# hrSWRunEntry 1.3.6.1.2.1.25.4.2.1 <SEQUENCE>
# hrSWRunIndex 1.3.6.1.2.1.25.4.2.1.1 <Integer32>
# hrSWRunName 1.3.6.1.2.1.25.4.2.1.2 <InternationalDisplayString> 64 Char
# hrSWRunID 1.3.6.1.2.1.25.4.2.1.3 <ProductID>
# hrSWRunPath 1.3.6.1.2.1.25.4.2.1.4 <InternationalDisplayString> 128 octets
# hrSWRunParameters 1.3.6.1.2.1.25.4.2.1.5 <InternationalDisplayString> 128 octets
# hrSWRunType 1.3.6.1.2.1.25.4.2.1.6 <INTEGER>
# hrSWRunStatus 1.3.6.1.2.1.25.4.2.1.7 <INTEGER> <<===== This is the key variable used by Opennms
self._setVars()
cmdrsp.GetCommandResponder(self._snmpEngine, snmpContext)
cmdrsp.SetCommandResponder(self._snmpEngine, snmpContext)
cmdrsp.NextCommandResponder(self._snmpEngine, snmpContext)
cmdrsp.BulkCommandResponder(self._snmpEngine, snmpContext)
def runAgent(self):
'''
Run the Agent
'''
t = threading.Thread(target=self._runAgentFunc, args = ())
t.daemon = True
t.start()
def _runAgentFunc(self):
try:
self._snmpEngine.transportDispatcher.jobStarted(1)
self._snmpEngine.transportDispatcher.runDispatcher()
except:
self._snmpEngine.transportDispatcher.closeDispatcher()
raise
def updateAgentStatus(self, runStatus, text1, text2):
self._mibDict = {"hrSWRunIndex" : 1,
"hrSWRunName" : self._name,
"hrSWRunID" : self._enterpriseMIB,
"hrSWRunPath" : text1[:128] if text1 is not None else '',
"hrSWRunParameters" : text2[:128] if text1 is not None else '',
"hrSWRunType" : 4,
"hrSWRunStatus" : 1
}
self._setVars()
def _setVars(self):
varBinds = self._mibInstrum.writeVars((
(self._hostRunTable.name + (1,) + self._instanceId, self._mibDict["hrSWRunIndex"]),
(self._hostRunTable.name + (2,) + self._instanceId, self._mibDict["hrSWRunName"]), # <=== Must match OpenNMS service-name variable
(self._hostRunTable.name + (3,) + self._instanceId, self._mibDict["hrSWRunID" ]), #
(self._hostRunTable.name + (4,) + self._instanceId, self._mibDict["hrSWRunPath"]),
(self._hostRunTable.name + (5,) + self._instanceId, self._mibDict["hrSWRunParameters"]),
(self._hostRunTable.name + (6,) + self._instanceId, self._mibDict["hrSWRunType"]), # Values are ==> unknown(1), operatingSystem(2), deviceDriver(3), application(4)
(self._hostRunTable.name + (7,) + self._instanceId, self._mibDict["hrSWRunStatus"]) #<<=== This is the status number OpenNMS looks at Values are ==> running(1), runnable(2), notRunnable(3), invalid(4)
))
def __init__(self, name, host, port, community, text1='Service up', text2=''):
'''
#=======================================================================
# Constructor
# name -- the (process) name the agent should publish (must match
# the openNMS name
# host -- the host name or ip the agent will run on
# port -- the port the snmp agent will listen on
# community -- the community name the agent will use (usually 'public')
# text1 -- the first status text string published (128 char max)
# text2 -- the second status text string published (128 char max)
#=======================================================================
'''
self._name = name
self._agentHost = host
self._agentPort = port
self._communityName = community
self._enterpriseMIB = (1, 3, 6, 1, 4, 1, 50000, 0) # Made up for now
self._mibInstrum = None
self._snmpEngine = None
self._dataChanged = False
self._mibDict = {"hrSWRunIndex" : 1,
"hrSWRunName" : self._name,
"hrSWRunID" : self._enterpriseMIB,
"hrSWRunPath" : text1[:128] if text1 is not None else '',
"hrSWRunParameters" : text2[:128] if text1 is not None else '',
"hrSWRunType" : 4,
"hrSWRunStatus" : 1
}
self._main()
and I call this code as follows (this is just testing that I can change the status):
from SNMPAgent import SNMPAgent
from time import sleep
agent = SNMPAgent("test", "127.0.0.1", 12345, "public", "This is my test message 1", "This is my test message 2")
agent.runAgent()
sleep(10) # Wait for it to start
while True:
agent.updateAgentStatus(3, "Oh no", "Something is wrong!")
sleep(30)
agent.updateAgentStatus(2, "Whew", "Everything is fixed")
sleep(30)
To walk the agent MIB i use:
snmpwalk -v 2c -c public -n my-context 127.0.0.1:12345 1.3.6.1.2.1.25.4.2
This shows the data updating but at the end of walking the MIB the agent times out:
HOST-RESOURCES-MIB::hrSWRunIndex.1 = INTEGER: 1
HOST-RESOURCES-MIB::hrSWRunName.1 = STRING: "test"
HOST-RESOURCES-MIB::hrSWRunID.1 = OID: SNMPv2-SMI::enterprises.50000.0
HOST-RESOURCES-MIB::hrSWRunPath.1 = STRING: "Whew"
HOST-RESOURCES-MIB::hrSWRunParameters.1 = STRING: "Everything is fixed"
HOST-RESOURCES-MIB::hrSWRunType.1 = INTEGER: application(4)
HOST-RESOURCES-MIB::hrSWRunStatus.1 = INTEGER: running(1)
Timeout: No Response from 127.0.0.1:12345

Answer 1

Enabling pysnmp debugging reveals serialization error caused by unset integer value for specific OID:
[2017-01-13 02:05:18,387-DEBUG]-(debug) generateResponseMsg: Message:
version='version-2c'
community=public
data=PDUs:
response=ResponsePDU:
request-id=1950819527
error-status='noError'
error-index=0
variable-bindings=VarBindList:
VarBind:
name=1.3.6.1.2.1.25.5.1.1.1.1
=_BindValue:
value=ObjectSyntax:
simple=SimpleSyntax:
integer-value=<no value>
2017-01-13 02:05:18,387 pysnmp: generateResponseMsg: serialization failure: Uninitialized ASN.1 value ("__eq__" attribute looked up)
[2017-01-13 02:05:18,387-DEBUG]-(debug) generateResponseMsg: serialization failure: Uninitialized ASN.1 value ("__eq__" attribute looked up)
2017-01-13 02:05:18,388 pysnmp: StatusInformation: {'errorIndication': <pysnmp.proto.errind.SerializationError object at 0x10162f828>}
[2017-01-13 02:05:18,388-DEBUG]-(debug) StatusInformation: {'errorIndication': <pysnmp.proto.errind.SerializationError object at 0x10162f828>}
You should probably make sure that you set values for all non-defaulted SNMP table columns.
As a side note, you seem to manage your MIB from main and SNMP agent threads without explicit synchronization. That may cause race conditions...