RabbitMQ Pika connection reset , (-1, ConnectionResetError(104, 'Connection reset by peer')) - python-3.x

searched through stackoverflow and posting this question because no solution worked for me and my question might be different from other question.
I am writing a script which gets an article from rabbitMQ queue and process the article to count words and extract key words from it and dump it in db. my script is working fine but after some time of execution i get this exception
(-1, "ConnectionResetError(104, 'Connection reset by peer')")
I have no idea why am I getting this. I have tried a lot of solutions available on stackover flow none is working for me. I havr written my script and tried it in two different ways. both work fine but after some time same exception occurs.
here is my first code:
def app_main():
global channel, results, speedvars
Logger.log_message('Starting app main')
# Edit 4
def pika_connect():
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
channel = connection.channel()
print ("In pika connect")
Logger.log_message('Setting up input queue consumer')
channel.queue_declare(Config.AMQ_DAEMONS['consumer']['input'], durable=True)
channel.basic_consume(on_message, queue=Config.AMQ_DAEMONS['consumer']['input'], no_ack=True)
Logger.log_message('Starting loop')
channel.start_consuming()
#########
speedvars = SpeedVars()
speedtracker = SpeedTracker(speedvars)
speedtracker.start()
sender = ResultsSender(results, speedvars)
sender.start()
# Edit 5 starting 10 threads to listen to pika
for th in range(qthreads):
Logger.log_message('Starting thread: '+str(th))
try:
t = Thread(target=pika_connect, args=())
t.start()
except Exception as e:
Logger.error_message("Exception in starting threads " + str(e))
try:
app_main()
except Exception as e:
Logger.error_message("Exception in APP MAIN " + str(e))
here is my second code:
def app_main():
global channel, results, speedvars
Logger.log_message('Starting app main')
speedvars = SpeedVars()
speedtracker = SpeedTracker(speedvars)
speedtracker.start()
sender = ResultsSender(results, speedvars)
sender.start()
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
channel = connection.channel()
print ("In app main")
Logger.log_message('Setting up input queue consumer')
channel.queue_declare(Config.AMQ_DAEMONS['consumer']['input'], durable=True)
channel.basic_consume(on_message, queue=Config.AMQ_DAEMONS['consumer']['input'], no_ack=True)
Logger.log_message('Starting loop')
try:
channel.start_consuming()
except Exception as e:
Logger.error_message("Exception in start_consuming in main " + str(e))
raise e
try:
app_main()
except Exception as e:
Logger.error_message("Exception in APP MAIN " + str(e))
in my first code i used threading because i want to speed up the process of processing articles.
this is my call back fuction
def on_message(ch, method, properties, message):
Logger.log_message("Starting parsing new msg ")
handle_message(message)
EDIT: Full Code
import os
abspath = os.path.abspath(__file__)
dname = os.path.dirname(abspath)
os.chdir(dname)
from Modules import Logger
import pika
import Config
import json
import pickle
import Pipeline
import sys
import time
import datetime
import threading
import queue
import functools
from pid.decorator import pidfile
Logger.log_init(Config.AMQ_DAEMONS['consumer']['log-ident'])
#qthreads = Config.AMQ_DAEMONS['consumer']['threads']
results = queue.Queue()
channel = None
speedvars = None
SPD_RECEIVED = 'received'
SPD_DISCARDED = 'discarded'
SPD_SENT = 'sent'
class SpeedVars(object):
vars = {}
lock = None
def __init__(self):
self.lock = threading.Lock()
def inc(self, var):
self.lock.acquire()
try:
if var in self.vars:
self.vars[var] += 1
else:
self.vars[var] = 1
finally:
self.lock.release()
def dec(self, var):
self.lock.acquire()
try:
if var in self.vars:
self.vars[var] -= 1
else:
Logger.error_message('Cannot decrement ' + var + ', not tracked')
finally:
self.lock.release()
def get(self, var):
out = None
self.lock.acquire()
try:
if var in self.vars:
out = self.vars[var]
else:
Logger.error_message('Cannot get ' + var + ', not tracked')
finally:
self.lock.release()
return out
def get_all(self):
out = None
self.lock.acquire()
try:
out = self.vars.copy()
finally:
self.lock.release()
return out
class SpeedTracker(threading.Thread):
speedvars = None
start_ts = None
last_vars = {}
def __init__(self, speedvars):
super(SpeedTracker, self).__init__()
self.start_ts = time.time()
self.speedvars = speedvars
Logger.log_message('Setting up speed tracker')
def run(self):
while True:
time.sleep(Config.AMQ_DAEMONS['consumer']['speed-tracking-interval'])
prev = self.last_vars
cur = self.speedvars.get_all()
now = time.time()
if len(prev) > 0:
q = {}
for key in cur:
qty = cur[key] - prev[key]
avg = qty / Config.AMQ_DAEMONS['consumer']['speed-tracking-interval']
overall_avg = cur[key] / (now - self.start_ts)
Logger.log_message('Speed-tracking (' + key + '): total ' + str(cur[key])
+ ', delta ' + str(qty) + ', speed ' + '%0.2f' % avg + '/sec, '
+ ', overall speed ' + '%0.2f' % overall_avg + '/sec')
pending = cur[SPD_RECEIVED] - cur[SPD_DISCARDED] - cur[SPD_SENT]
pending_avg = pending / (now - self.start_ts)
Logger.log_message('Speed-tracking (pending): total ' + str(pending)
+ ', overall speed ' + '%0.2f' % pending_avg + '/sec')
self.last_vars = cur
class ResultsSender(threading.Thread):
channel = None
results = None
speedvars = None
def __init__(self, results, speedvars):
super(ResultsSender, self).__init__()
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
self.channel = connection.channel()
Logger.log_message('Setting up output exchange')
self.channel.exchange_declare(exchange=Config.AMQ_DAEMONS['consumer']['output'], exchange_type='direct')
self.results = results
self.speedvars = speedvars
def run(self):
while True:
item = self.results.get()
self.channel.basic_publish(
exchange=Config.AMQ_DAEMONS['consumer']['output'],
routing_key='',
body=item)
self.speedvars.inc(SPD_SENT)
def parse_message(message):
try:
bodytxt = message.decode('UTF-8')
body = json.loads(bodytxt)
return body
except Exception as e:
Logger.error_message("Cannot parse message - " + str(e))
raise e
def get_body_elements(body):
try:
artid = str(body.get('article_id'))
article_dt = datetime.datetime.fromtimestamp(body.get('pubTime'))
date = article_dt.strftime(Config.DATE_FORMAT)
article = "\n".join([body.get('title', ''), body.get('subheading', ''), body.get('content', '')])
return (artid, date, article)
except Exception as e:
Logger.error_message("Cannot retrieve article attributes " + str(e))
raise e
def process_article(id, date, text):
global results, speedvars
try:
Logger.log_message('Processing article ' + id)
keywords = Pipeline.extract_keywords(text)
send_data = {"id": id, "date": date, "keywords": keywords}
results.put(pickle.dumps(send_data))
# print('Queue Size:',results.qsize())
except Exception as e:
Logger.error_message("Problem processing article " + str(e))
raise e
def ack_message(ch, delivery_tag):
"""Note that `channel` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if channel.is_open:
channel.basic_ack(delivery_tag)
else:
Logger.error_message("Channel is already closed, so we can't ACK this message" + str(e))
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
#pass
def handle_message(connection, ch, delivery_tag, message):
global speedvars
start = time.time()
thread_id = threading.get_ident()
try:
speedvars.inc(SPD_RECEIVED)
body = parse_message(message)
(id, date, text) = get_body_elements(body)
words = len(text.split())
if words <= Config.AMQ_DAEMONS['consumer']['word-count-limit']:
process_article(id, date, text)
else:
Logger.log_message('Ignoring article, over word count limit')
speedvars.inc(SPD_DISCARDED)
except Exception as e:
Logger.error_message("Could not process message - " + str(e))
cb = functools.partial(ack_message, ch, delivery_tag)
connection.add_callback_threadsafe(cb)
Logger.log_message("Thread id: "+str(thread_id)+" Delivery tag: "+str(delivery_tag))
Logger.log_message("TOtal time taken to handle message : "+ str(time.time()-start))
# CALL BACK
## def on_message(ch, method, properties, message):
## global executor
## executor.submit(handle_message, message)
def on_message(ch, method, header_frame, message, args):
(connection, threads) = args
delivery_tag = method.delivery_tag
t = threading.Thread(target=handle_message, args=(connection, ch, delivery_tag, message))
t.start()
threads.append(t)
####################################################
#pidfile(piddir=Config.AMQ_DAEMONS['base']['pid-dir'], pidname=Config.AMQ_DAEMONS['consumer']['pid-file'])
def app_main():
global channel, results, speedvars
speedvars = SpeedVars()
speedtracker = SpeedTracker(speedvars)
speedtracker.start()
sender = ResultsSender(results, speedvars)
sender.start()
# Pika Connection
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=Config.AMQ_DAEMONS['base']['amq-host']))
channel = connection.channel()
Logger.log_message('Setting up input queue consumer')
channel.queue_declare(Config.AMQ_DAEMONS['consumer']['input'], durable=True)
#channel.basic_consume(on_message, queue=Config.AMQ_DAEMONS['consumer']['input'], no_ack=True)
channel.basic_qos(prefetch_count=1)
threads = []
on_message_callback = functools.partial(on_message, args=(connection, threads))
channel.basic_consume(on_message_callback, Config.AMQ_DAEMONS['consumer']['input'])
Logger.log_message('Starting loop')
## channel.start_consuming()
try:
channel.start_consuming()
except KeyboardInterrupt:
channel.stop_consuming()
Wait for all to complete
for thread in threads:
thread.join()
connection.close()
app_main()
pika is not taking a lot of time to process message still i am facing connection reset issue.
**TOtal time taken to handle message : 0.0005991458892822266
**

Your handle_message method is blocking heartbeats because all of your code, including the Pika I/O loop, is running on the same thread. Check out this example of how to run your work (handle_message) on a separate thread from Pikas I/O loop and then acknowledge messages correctly.
NOTE: the RabbitMQ team monitors the rabbitmq-users mailing list and only sometimes answers questions on StackOverflow.

I was getting the same issue . Increasing the duration of heart-beat & connection timeouts configuration didn't work out for me. I finally figured out that, if you have
already created a channel and you are not publishing anything on it for
several minutes(20 mins in my case) ,in that case we get this error.
The Solutions which worked for me:
Create channel immediately just before publishing any message. OR
Use try-except and if you get an exception , create another channel and republish. ie.
try:
channel.basic_publish(exchange='', routing_key='abcd', body=data)
except Exception as e1:
connection=pika.BlockingConnection(pika.ConnectionParameters(host='1.128.0.3',credentials=credentials))
channel = connection.channel()
channel.basic_publish(exchange='', routing_key='abcd', body=data)
This will atleast keep the things running and prevent from losing any data. I'm not an expert in this, but hope this helps someone!

I also faced the same issue and resolved by increasing the duration for heart-beat & connection timeouts configuration.
Many thanks to #LukeBakken who has actually identified the root cause.
Here is how you can configure the timeouts:
import pika
def main():
# NOTE: These parameters work with all Pika connection types
params = pika.ConnectionParameters(heartbeat=600, blocked_connection_timeout=300)
conn = pika.BlockingConnection(params)
chan = conn.channel()
chan.basic_publish('', 'my-alphabet-queue', "abc")
# If publish causes the connection to become blocked, then this conn.close()
# would hang until the connection is unblocked, if ever. However, the
# blocked_connection_timeout connection parameter would interrupt the wait,
# resulting in ConnectionClosed exception from BlockingConnection (or the
# on_connection_closed callback call in an asynchronous adapter)
conn.close()
if __name__ == '__main__':
main()
Reference: https://pika.readthedocs.io/en/stable/examples/heartbeat_and_blocked_timeouts.html

Related

Exception error in EReader thread running ibapi

I am running Python ib-api to receive the realtime market data from Interactive Brokers. It can provide the data I expected but it ends with "unhandled exception in EReader thread".
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract as IBcontract
from threading import Thread
import queue
import pandas as pd
from ibapi.ticktype import TickTypeEnum`
`DEFAULT_PRICE_DATA_ID = 1001`
`FINISHED = object()
STARTED = object()
TIME_OUT = object()`
class finishableQueue(object):
def __init__(self, queue_to_finish):
self._queue = queue_to_finish
self.status = STARTED
def get(self, timeout):
contents_of_queue=[]
finished=False
while not finished:
try:
current_element = self._queue.get(timeout=timeout)
if current_element is FINISHED:
finished = True
self.status = FINISHED
else:
contents_of_queue.append(current_element)
except queue.Empty:
finished = True
self.status = TIME_OUT
return contents_of_queue
def timed_out(self):
return self.status is TIME_OUT
class TestWrapper(EWrapper):
def __init__(self):
self._my_price_data_dict = {}
def get_error(self, timeout=5):
if self.is_error():
try:
return self._my_errors.get(timeout=timeout)
except queue.Empty:
return None
return None
def is_error(self):
an_error_if=not self._my_errors.empty()
return an_error_if
def init_error(self):
error_queue=queue.Queue()
self._my_errors = error_queue
def error(self, id, errorCode, errorString):
## Overriden method
errormsg = "IB error id %d errorcode %d string %s" % (id, errorCode, errorString)
self._my_errors.put(errormsg)
def init_ibprices(self, tickerid):
ibprice_data_queue = self._my_price_data_dict[tickerid] = queue.Queue()
return ibprice_data_queue
def tickPrice(self, reqId, tickType, price, attrib):
tickdata = (TickTypeEnum.to_str(tickType), price)
price_data_dict = self._my_price_data_dict
if reqId not in price_data_dict.keys():
self.init_ibprices(reqId)
price_data_dict[reqId].put(tickdata)
class TestClient(EClient):
def __init__(self, wrapper):
EClient.__init__(self, wrapper)
def error(self, reqId, errorCode, errorString):
print("Error: ", reqId, " ", errorCode, " ", errorString)
def getIBrealtimedata(self, ibcontract, tickerid=DEFAULT_PRICE_DATA_ID):
ib_data_queue = finishableQueue(self.init_ibprices(tickerid))
self.reqMktData(
tickerid,
ibcontract,
"",
False,
False,
[]
)
MAX_WAIT_SECONDS = 5
print("Getting data from the server... could take %d seconds to complete " % MAX_WAIT_SECONDS)
price_data = ib_data_queue.get(timeout = MAX_WAIT_SECONDS)
while self.wrapper.is_error():
print(self.get_error())
if ib_data_queue.timed_out():
print("Exceeded maximum wait for wrapper to confirm finished - seems to be normal behaviour")
self.cancelMktData(tickerid)
return price_data
class TestApp(TestWrapper, TestClient):
def __init__(self, ipaddress, portid, clientid):
TestWrapper.__init__(self)
TestClient.__init__(self, wrapper=self)
self.connect(ipaddress, portid, clientid)
thread = Thread(target = self.run)
thread.start()
setattr(self, "_thread", thread)
self.init_error()
def main(slist):
app = TestApp("127.0.0.1", 7497, 1)
for i in slist:
ibcontract = IBcontract()
ibcontract.secType = "STK"
ibcontract.symbol = i
ibcontract.exchange ="SEHK"
Lastprice = app.getIBrealtimedata(ibcontract)
df = pd.DataFrame(Lastprice)
print(ibcontract.symbol, df.head())
app.disconnect()
if __name__ == "__main__":
seclist = [700,2318,5,12]
main(seclist)
Here are the error messages:
unhandled exception in EReader thread
Traceback (most recent call last):
File "D:\Anaconda3\envs\myweb\lib\site-packages\ibapi\reader.py", line 34, >in run
data = self.conn.recvMsg()
File "D:\Anaconda3\envs\myweb\lib\site-packages\ibapi\connection.py", line >99, in recvMsg
buf = self._recvAllMsg()
File "D:\Anaconda3\envs\myweb\lib\site-packages\ibapi\connection.py", line >119, in _recvAllMsg
buf = self.socket.recv(4096)
OSError: [WinError 10038] An operation was attempted on something that is >not a socket
A separate thread is started to read incoming messages from the socket:
thread = Thread(target = self.run)
thread.start()
But this thread is never stopped, and is still running when you call disconnect(). As a result, it tries to access the socket object which is now None, triggering the error. Try stopping the EReader thread prior to disconnecting by setting done=True.
As a side note, since this error happens at the very end of the program at the disconnection it shouldn't interfere with receiving the expected data.
A workaround to avoid the warning.
Implement this at your EClient / EWrapper subclass:
Create a socket shutdown function:
import socket, time
[...]
def _socketShutdown(self):
self.conn.lock.acquire()
try:
if self.conn.socket is not None:
self.conn.socket.shutdown(socket.SHUT_WR)
finally:
self.conn.lock.release()
Use it before closing the connection:
self._socketShutdown()
time.sleep(1)
self.disconnect()

Flask SocketIO Eventlet Second Simultaneous Read Error

I am trying to use Flask-Socketio to connect to 3 separate RabbitMQ queues:
- 1 that listens for a config update message
- and 2 that are defined in a database
On the server start, I am connecting to the database, getting the configs there, and starting the consumers. Then, in a web frontend, if one of the configuration settings are changed, these changes are being written to the database, and a config update message is being added to the first RabbitMQ queue. Ideally what this would trigger a shutdown of the Pika consumer that I currently have running, a joining of that thread, and a relaunch of another thread with the new configuration information.
Everything that I just laid out is working, but on the first attempt to shut down the consumer, I'm always getting the error:
There was an error stopping the consumer: Second simultaneous read on fileno x detected.
Unless you really know what you're doing, make sure that only one greenthread can read any particular socket.
Consider using a pools.Pool. If you do know what you're doing and want to disable this error, call eventlet.debug.hub_prevent_multiple_readers(False)...
The consumers are eventually closed, and restarted, however, I would like to understand why this is occurring, and how I could change my code to stop it. The place where the error always happens is in the hand off between these 2 function, the first is in my Consumer class and the second is in my Queue class:
def run(self):
while True:
self.go = True
self.message_queue = Queue(self.configs, self.go, self.mongo_config)
self.message_queue.start()
# here I wait for an event which I set when the config is updated
self.event.wait()
self.go = False
setattr(self.message_queue, 'go', False)
new_config = self.refresh_configs()
setattr(self, 'configs', new_config)
# when this is called, it should close the existing connection and join the thread
self.message_queue.refresh_connection()
self.message_queue.join()
def refresh_connection(self):
while True:
if not self.go:
break
self.rmq_connection = rabbitmq_consumer(...)
self.rmq_connection.start_consuming()
self._lock.acquire()
try:
# right here is where the second read error occurs
self.rmq_connection.stop_consuming()
self.rmq_connection.close()
except Exception as e:
print('There was an error stopping the consumer: {0}'.format(e))
self._lock.release()
Below is a much more complete example of the code, in case it helps shed some light on the issue.
thread = None
thread_lock = Lock()
event = Event()
class Queue(Thread):
def __init__(self, configs, go, outbound):
Thread.__init__(self)
self._lock = eventlet.semaphore.Semaphore(1)
self.go = go
self.configs = configs
self.outbound = outbound
...
self.rmq_connection = None
def on_rmq_message(self, ...):
...
self._lock.acquire()
socketio.emit('eventEmit', {'data': results}, namespace='/')
result = rabbitmq_producer(...)
self._lock.release()
def refresh_connection(self):
while True:
if not self.go:
break
self.rmq_connection = rabbitmq_consumer(...)
self.rmq_connection.start_consuming()
self._lock.acquire()
try:
self.rmq_connection.stop_consuming()
self.rmq_connection.close()
except Exception as e:
print('There was an error stopping the consumer: {0}'.format(e))
self._lock.release()
def run(self):
self.refresh_connection()
class Consumer(Thread):
def __init__(self, configs, event, channel, mongo_config):
Thread.__init__(self)
self.configs = configs
self.mongo_config = mongo_config
self.event = event
self.message_queue = None
self.go = None
...
def refresh_configs(self):
r = None
mconnection = connect_mongodb(...)
results = retrieve(...)
for result in results.data:
if result.get('channel') == self.channel:
r = result
return r
def run(self):
while True:
self.go = True
self.message_queue = Queue(self.configs, self.go, self.mongo_config)
self.message_queue.start()
self.event.wait()
self.go = False
setattr(self.message_queue, 'go', False)
new_config = self.refresh_configs()
setattr(self, 'configs', new_config)
self.message_queue.refresh_connection()
self.message_queue.join()
class Producer(Thread):
def __init__(self, configs, event):
Thread.__init__(self)
self._lock = eventlet.semaphore.Semaphore(1)
self.configs = configs
self.event = event
...
self.channel = self.configs.get('channel', None)
self.config_consumer = None
def on_config_message(self, ...):
...
self._lock.acquire()
socketio.emit('configEmit', {'data': results}, namespace='/')
self._lock.release()
self.event.set()
self.event.clear()
def run(self):
self.config_consumer = rabbitmq_consumer(...)
self.config_consumer.start_consuming()
def init_connections():
with app.test_request_context('/'):
mconnection = connect_mongodb(...)
results = retrieve(mconnection.engine, mongo_collection, mongo_db, cursor=False)
...
t1 = Producer(configs, event)
for result in results.data:
config = {
...
}
t2 = Consumer(result, event, result['channel'], config)
t2.start()
t1.start()
t1.join()
t2.join()
#socketio.on('connect')
def handle_connection():
global thread
socketio.emit('connectEmit', {'data': 'Connected!'}, namespace='/')
with thread_lock:
if thread is None:
thread = socketio.start_background_task(target=init_connections)
Thank you for any help you can offer.

Synchronize two functions into separate classes

In a class there are different functions. In a separate file in another class I want to catch the messages and print to gui.
As a simulation I have the following code:
import threading
import time
import logging
logging.basicConfig(level=logging.DEBUG, format='(%(threadName)-9s) %(message)s',)
message = None
def messages_generator(condition):
global message
with condition:
logging.debug('Condition: {}'.format(condition))
for i in range(5):
message = 'i = ' + str(i)
time.sleep(1)
logging.debug('Condition wait')
condition.wait()
def messages_sow(condition):
global message
with condition:
print(message)
logging.debug('Condition notify')
condition.notify()
logging.debug('Tread finished')
condition = threading.Condition()
messages_generator_thread = threading.Thread(name='Message Generator', target=messages_generator, args=(condition,))
messages_sow_thread = threading.Thread(name='Message Sow', target=messages_sow, args=(condition,))
messages_generator_thread.start()
messages_sow_thread.start()
What I want is the messages_generator to wait for the message to be printed by the messages_sow emphasized text and continue until it is completed. When I run the above code, the program freezes on the second 'Condition wait'.
Any advice to be welcomed.
I finally managed to work the code above, but not on the basic program which I develop based on the Model - View - Controller programming model.
I quote the code that works.
import threading
import time
import logging
logging.basicConfig(level=logging.DEBUG, format='(%(threadName)-9s) %(message)s',)
message = None
def messages_generator(condition):
logging.debug('--- Start ---')
global message
messages_number = 5
for i in range(messages_number):
logging.debug('Inside For. i = {}'.format(i))
condition.acquire()
if message is not None:
logging.debug('Condition wait')
condition.wait()
if i == (messages_number - 1):
message = 'end'
logging.debug('Message = {}'.format(message))
else:
message = 'i = ' + str(i)
time.sleep(1)
logging.debug('Condition notify')
condition.notify()
logging.debug('Condition release')
condition.release()
def messages_sow(condition):
logging.debug('--- Start ---')
global message
while True:
logging.debug('Inside While. stop = {}'.format(True))
condition.acquire()
if message is None:
logging.debug('Condition wait')
condition.wait()
else:
print(message)
if message == 'end':
break
message = None
condition.notify()
condition.release()
logging.debug('Tread finished')
condition = threading.Condition()
messages_generator_thread = threading.Thread(name='Message Generator', target=messages_generator, args=(condition,))
messages_sow_thread = threading.Thread(name='Message Sow', target=messages_sow, args=(condition,))
messages_generator_thread.start()
messages_sow_thread.start()

Stop server from client's thread / Modify server's variable from client's thread

I would like to write an application that could stop the server based on client's input. The server is multi-threaded and I do not understand how can I do this.
Basically, I described my problem here: Modify server's variable from client's thread (threading, python).
However, this is the Python solution, not the general solution I could implement in Java, C, C++, etc.
I need to close other clients, when one of them guesses the number, but the server should be still alive, ready for the new game.
Can I ask for some advices, explanations?
I tried this (still do not know how to port it to C or Java), but it lets the clients send the numbers even if one of them just guesses it. It seems to me that kill_em_all does not do it's job, it does not close all the connections and does not disconnect the other clients as it should. How to improve this?
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber, server):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
self.server = server
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print(data)
try:
num = int(data)
if self.server.guess(num) :
print 'someone guessed!'
self.server.kill_em_all()
break
else :
msg = "Try again!"
self.connection.sendall(msg.encode())
except ValueError as e:
msg = "%s" % e
self.connection.sendall(msg.encode())
else:
msg = "error"
self.connection.send(msg.encode())
except socket.error:
break
self.connection.close()
def send(self, msg):
self.connection.sendall(msg)
def close(self):
self.connection.close()
class Server:
randnum = randint(1,100)
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
def guess(self, no):
if self.randnum == no:
self.randnum = randint(1, 100)
print("New number is ", self.randnum )
result = True
else:
result = False
return result
def kill_em_all(self):
for c in self.clients:
c.send("BYE!")
c.close()
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
self.clients = []
print('Num is %s' % self.randnum)
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum, self)
c.start()
self.clients.append(c)
except socket.error as e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 7777)
s.run()
Client code:
import socket
import sys
port = 7777
s = None
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = socket.gethostname()
s.connect(('127.0.0.1', port))
except socket.error, (value, message):
if s:
s.close()
print "Could not open socket: " + message
sys.exit(1)
while True:
data = raw_input('> ')
s.sendall(data)
data = s.recv(1024)
if data:
if data == "BYE!":
break
else:
print "Server sent: %s " % data
s.close()
Log in. Using whatever protocol you have, send the server a message telliing it to shut down. In the server, terminate your app when you get the shutdown message. That's it. It's not a problem with any OS I have used - any thread of a process can terminate that process.

Pass asynchronous websocket.send() to stdout/stderr wrapper class

I have a class function which unbuffers stdout and stderr, like so:
class Unbuffered:
def __init__(self, stream):
self.stream = stream
def write(self, data):
data = data.strip()
if data.startswith("INFO: "):
data = data[6:]
if '[' in data:
progress = re.compile(r"\[(\d+)/(\d+)\]")
data = progress.match(data)
total = data.group(2)
current = data.group(1)
data = '{0}/{1}'.format(current, total)
if data.startswith("ERROR: "):
data = data[7:]
self.stream.write(data + '\n')
self.stream.flush()
def __getattr__(self, attr):
return getattr(self.stream, attr)
The output is from a function run in ProcessPoolExecutor when inbound from websocket arrives.
I want the output printed in console as well as sent to my websocket client. I tried asyncing Unbuffered and passing websocket instance to it but no luck.
UPDATE: The essentials of run() and my websocket handler() look something like this:
def run(url, path):
logging.addLevelName(25, "INFO")
fmt = logging.Formatter('%(levelname)s: %(message)s')
#----
output.progress_stream = Unbuffered(sys.stderr)
stream = Unbuffered(sys.stdout)
#----
level = logging.INFO
hdlr = logging.StreamHandler(stream)
hdlr.setFormatter(fmt)
log.addHandler(hdlr)
log.setLevel(level)
get_media(url, opt)
async def handler(websocket, path):
while True:
inbound = json.loads(await websocket.recv())
if inbound is None:
break
url = inbound['url']
if 'path' in inbound:
path = inbound['path'].rstrip(os.path.sep) + os.path.sep
else:
path = os.path.expanduser("~") + os.path.sep
# blah more code
while inbound != None:
await asyncio.sleep(.001)
await loop.run_in_executor(None, run, url, path)
run(), handler() and Unbuffered are separate from each other.
Rewriting get_media() to use asyncio instead of running it in a different thread would be the best. Otherwise, there are some options to communicate between a regular thread and coroutines, for example, using a socketpair:
import asyncio
import socket
import threading
import time
import random
# threads stuff
def producer(n, writer):
for i in range(10):
# print("sending", i)
writer.send("message #{}.{}\n".format(n, i).encode())
time.sleep(random.uniform(0.1, 1))
def go(writer):
threads = [threading.Thread(target=producer, args=(i + 1, writer,))
for i in range(5)]
for t in threads:
t.start()
for t in threads:
t.join()
writer.send("bye\n".encode())
# asyncio coroutines
async def clock():
for i in range(11):
print("The time is", i)
await asyncio.sleep(1)
async def main(reader):
buffer = ""
while True:
buffer += (await loop.sock_recv(reader, 10000)).decode()
# print(len(buffer))
while "\n" in buffer:
msg, _nl, buffer = buffer.partition("\n")
print("Got", msg)
if msg == "bye":
return
reader, writer = socket.socketpair()
reader.setblocking(False)
threading.Thread(target=go, args=(writer,)).start()
# time.sleep(1.5) # socket is buffering
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait([clock(), main(reader)]))
loop.close()
You can also try this 3rd-party thread+asyncio compatible queue: janus

Resources