I have this code that is supposed to log to a memory variable:
(the_message_lines):
import logging
from thompcoutils import log_utils
the_message_lines = []
class MemoryHandler(logging.StreamHandler):
"""
Handler that keeps all log messages in memory until the beginning of the day or the size exceeds a value
"""
def emit(self, record: logging.LogRecord):
global the_message_lines
try:
msg = self.format(record)
the_message_lines.append(msg)
self.flush()
except (KeyboardInterrupt, SystemExit):
raise
except:
self.handleError(record)
#staticmethod
def get_lines():
return the_message_lines
#staticmethod
def reset_lines():
global the_message_lines
the_message_lines.clear()
if __name__ == '__main__':
log_utils.load_log_config('logging.ini')
logger = log_utils.get_logger()
logger.warning('beginning')
for i in range(3):
lines = MemoryHandler.get_lines()
logger.warning('remaining %d seconds', i, extra={'same_line':True})
logger.warning('end')
for line in MemoryHandler.get_lines():
print(line)
It behaves as it should but the_message_lines come and go(?)
If I put a break point in the emit method and observe the_message_lines, it behaves as it should, accumulating log messages every time.
If I put a break point in the loop that is logging, the_message_lines is empty every time!
So, between log requests, the_message_lines appears to delete itself, but in the emit, it is fine.
At the end of the main() method, get_lines() returns an empty array.
What am I missing?
Related
I'm parsing the last line of a continuously updating log file. If it matches, I want to return the match to a list and start another function using that data. I need to keep watching for new entries and parse them even while the new function continues.
I've been working this from a few different angles for about a week with varying success. I tried threading, but ran into issues getting the return value, I tried using a global var but couldn't get it working. I'm now trying asyncio, but having even more issues getting that to work.
def tail():
global match_list
f.seek(0, os.SEEK_END)
while True:
line = f.readline()
if not line:
time.sleep(0.1)
continue
yield line
def thread():
while True:
tail()
def somefun(list):
global match_list
#do things here
pass
def main():
match_list = []
f = open(r'file.txt')
thread=threading.Thread(target=thread, args=(f,))
thread.start()
while True:
if len(match_list) >= 1:
somefun(match_list)
if __name__ == '__main__':
main()
Wrote the above from memory..
I want tail() to return the line to a list that somefun() can use.
I'm having issues getting it to work, I will use threading or asyncio.. anything to get it running at this point.
In asyncio you might use two coroutines, one that reads from file, and the other that processes the file. Since they communicate using queue, they don't need the global variable. For example:
import os, asyncio
async def tail(f, queue):
f.seek(0, os.SEEK_END)
while True:
line = f.readline()
if not line:
await asyncio.sleep(0.1)
continue
await queue.put(line)
async def consume(queue):
lines = []
while True:
next_line = await queue.get()
lines.append(next_line)
# it is not clear if you want somefun to receive the next
# line or *all* lines, but it's easy to do either
somefun(next_line)
def somefun(line):
# do something with line
print(f'line: {line!r}')
async def main():
queue = asyncio.Queue()
with open('file.txt') as f:
await asyncio.gather(tail(f, queue), consume(queue))
if __name__ == '__main__':
asyncio.run(main())
# or, on Python older than 3.7:
#asyncio.get_event_loop().run_until_complete(main())
The beauty of an asyncio-based solution is that you can easily start an arbitrary number of such coroutines in parallel (e.g. you could start gather(main1(), main2()) in an outer coroutine, and run that), and have them all share the same thread.
with a few small fixes you almost run this :) (comments inside)
match_list # should be at the module scope
def tail():
# f = open(...) ???
f.seek(0, os.SEEK_END)
while True:
line = f.readline()
if not line:
time.sleep(0.1)
continue
yield line
def thread():
for line in tail():
match_list.append(line) # append line
print("thread DONE!")
def somefun(list):
#do things here
while match_list:
line = match_list.pop(0)
print(line)
def main():
match_list = []
f = open(r'file.txt')
thread=threading.Thread(target=thread, args=(f,))
thread.start()
while True:
if match_list:
somefun(match_list)
time.sleep(0.1) # <-- don't burn the CPU :)
I would like to capture the console output at the end of a Python script. That is, I want to both print to console as normal and, at the end of execution, save the console output to a file.
I have seen various related SO questions 1, 2, 3 though they either simply redirect the output and not display it or use logging. From what I can tell from reading the logging doc you can only log output from the code you've written.
The issue with all the links above is console output from code you have not written that also prints to console. I want the entire console output of the program at the end of execution.
My first instinct was something along the lines of
logFile = open('LogFile.txt', 'w')
def print_log(msg):
print(msg)
logFile.write(msg)
print_log('Hello World!')
logFile.close()
But this would still fail to capture console output from other code and libraries in use. Is there a way to save a python script's console output at the end of execution? I feel like there should be a simple way to do this but all my research has led to no appropriate solution.
I've used this one in one of my projects:
import io
import sys
from enum import Enum
class Tee(io.StringIO):
class Source(Enum):
STDOUT = 1
STDERR = 2
def __init__(self, clone=Source.STDOUT, *args, **kwargs):
super().__init__(*args, **kwargs)
self._clone = clone
if clone == Tee.Source.STDOUT:
self._out = sys.stdout
elif clone == Tee.Source.STDERR:
self._out = sys.stderr
else:
raise ValueError("Clone has to be STDOUT or STDERR.")
def write(self, *args, **kwargs):
self._out.write(*args, **kwargs)
return super().write(*args, **kwargs)
def __enter__(self):
if self._clone == Tee.Source.STDOUT:
sys.stdout = self
else:
sys.stderr = self
self.seek(io.SEEK_END)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self._clone == Tee.Source.STDOUT:
sys.stdout = self._out
else:
sys.stderr = self._out
self.seek(0)
return False
Basically it does exactly what Maksym Markov said in the comment with one difference. I usually don't wanna stall any outputs so I've written this Tee which capture all text going on stdout (or stderr) immediately prints it and save into the buffer for later usage. It also take care about "fixing" the sys.stdout when the code exits the with block.
The example of usage:
if __name__ == "__main__":
with Tee() as tee:
print("Hello World!")
print(tee.read())
There are some drawbacks like without additional code you can't use tee.read() inside the with block. But in my case I always need process the output of the whole block.
I'm very new to PyQt and threading (only about a year). I apologize for the long post (most of the questions are at the end). I could use some assistance with why my worker thread is losing output when performing the work. This is PyQt5 and Python 3.5.
#Creates your own signal class to output stdout.
class EmittingStream(QtCore.QObject):
textWritten = QtCore.pyqtSignal(str)
def write(self, text):
self.textWritten.emit(str(text))
def flush(self):
pass
class MainWindow(QtWidgets.QMainWindow, Ui_mainWindow):
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent=parent)
self.setupUi(self)
self.closeButton.clicked.connect(self.close)
self.clearButton.clicked.connect(self.clear_all)
self.submitButton.clicked.connect(self.submit_ips)
self.addButton.clicked.connect(self.add_ip_on_button_press)
#Keep Save button inactive until an IP(s) have been submitted.
self.saveButton.clicked.connect(self.write_mem)
self.saveButton.setEnabled(False)
sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)
sys.stderr = EmittingStream(textWritten=self.errorOutputWritten)
#Creates instance of WorkerThread in the main window
self.workerThread = WorkerThread()
#Creates isntance of WorkerThreadSave in the main window
self.workerThreadSave = WorkerThreadSave()
def __del__(self):
# Restore sys.stdout
sys.stdout = sys.__stdout__
# Uses EmittingStream class to output stdout to outputTextBox
def normalOutputWritten(self, text):
"""Append text to the QTextEdit."""
cursor = self.outputTextBox.textCursor()
cursor.movePosition(QtGui.QTextCursor.End)
cursor.insertText(text)
# self.outputTextBox.setTextCursor(cursor)
self.outputTextBox.ensureCursorVisible()
def errorOutputWritten(self, text):
cursor = self.outputTextBox.textCursor()
cursor.movePosition(QtGui.QTextCursor.End)
cursor.insertText(text)
self.outputTextBox.ensureCursorVisible()
def submit_ips(self):
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Warning)
msgBox.setWindowTitle("Error")
if not hostiplist and not cidriplist:
msgBox.setText("Ooops, Please enter an acceptible host or network IP Address.")
msgBox.exec()
else:
print("*************** Process Started ***************\n")
self.submitButton.setEnabled(False)
self.addButton.setEnabled(False)
self.clearButton.setEnabled(False)
self.saveButton.setEnabled(True)
self.workerThread.start()
#Performs Save Function through API
def write_mem(self):
print("*************** Saving Device(s) Started ***************\n")
self.saveButton.setEnabled(False)
self.addButton.setEnabled(True)
self.clearButton.setEnabled(True)
self.submitButton.setEnabled(True)
del hostiplist[:]
del cidriplist[:]
self.workerThreadSave.start()
The above code is obviously my main UI. When clicking the submit button it starts a separate thread to perform the processing and work (and also to not freeze the UI).
Unfortunately, I end up losing some of the print statements (I'm assuming my signals and slots aren't performing or the thread stops before the output is printed).
class WorkerThread(QThread):
def __init__(self, parent=None):
super(WorkerThread, self).__init__(parent)
def run(self):
#Post function for adding host IP's
def post_object (f):
self.api_path = "/api/objects/networkobjects"
self.req = urllib.request.Request(str(server) + self.api_path, json.dumps(self.post_data).encode('utf-8'), headers)
self.base64string = base64.encodestring(('%s:%s' % (username,password)).encode()).decode().replace('\n', '')
self.req.add_header("Authorization", "Basic %s" % self.base64string)
try:
f = urllib.request.urlopen(self.req)#Had to add .request between the url.
self.status_code = f.getcode()
# print("Status code is {}.".format(self.status_code))
if self.status_code == 201:
print ("The following object was created successfully: {}".format(self.post_data["name"]))
except urllib.error.HTTPError as err: #Python3 requires urllib.error.HTTPError to work.
print ("Error received from server: {}. HTTP Status code: {}".format(err.reason, err.code))
try:
print("Yes {}".format(json.loads(err.read))) #json.loads is not being sent...if you format the error it'll send that...not the error itself.
json_error = json.loads(err.read)
if json_error:
print (json.dumps(json_error,sort_keys=True,indent=4, separators=(',', ': '))) #Error won't print json.dumps error
print ("Error received from server: {}. HTTP Status code: {}".format(err.reason, err.code))
except ValueError:
pass
finally:
if f: f.close()
return (f)
I run the post_object function within a for loop below:
try:
#Starts for loop REST API adds.
for s in serverList:
server = s
# Operation for CIDR IPs
for c in cidriplist:
self.network = c
self.patch_data_group = {
"members.add": [
{
"kind": "IPv4Network",
"value": str(self.network)
}]
}
patch_object_group(f)
jsonoutputlist.append(self.patch_data_group)
#Operation for single host IPs
for h in hostiplist:
self.host = h
self.post_data = {
"host": {
"kind": "IPv4Address",
"value": str(self.host)},
"kind": "object#NetworkObj",
"name": str(self.host),
"objectId": str(self.host)
}
self.patch_data_group = {
"members.add": [
{
"kind": "objectRef#NetworkObj",
"objectId": str(self.host)
}]
}
jsonoutputlist.append(self.post_data)
post_object(f)
patch_object_group(f)
# print ("\n JSON Data format for change control: \n{}".format(self.patch_data_group))
print("\n*************** IPs Add Complete ***************\n")
except:
(type, value, traceback) = sys.exc_info()
sys.excepthook(type, value, traceback)
This is where the main issues arise. During the operation of the for loops, the print statements and return API call statements are dropped or cut off.
For example,
if self.status_code == 201:
print ("The following object was created successfully: {}".format(self.post_data["name"]))
The above statement only prints one line below the if statement...it won't allow me to do more than one line...and the one line print statement is usually cut off. As well as the json.dumps statements won't print at all...
Any help would be greatly appreciated.
Thanks,
I'm currently getting this error. I'm confused because from what I can tell Generator Exit just gets called whenever a generator finishes, but I have a ton of other Generators inheriting this class that do not call this error. Am I setting the Generator up properly? or is there some implicit code I'm not taking into account that is calling close()?
"error": "Traceback (most recent call last):\n File \"/stashboard/source/stashboard/checkers.py\", line 29, in run\n yield self.check()\nGeneratorExit\n",
the code where this yield statement is called:
class Checker():
def __init__(self, event, frequency, params):
self.event = event
self.frequency = frequency
self.params = params
#gen.coroutine
def run(self):
""" Run check method every <frequency> seconds
"""
while True:
try:
yield self.check()
except GeneratorExit:
logging.info("EXCEPTION")
raise GeneratorExit
except:
data = {
'status': events.STATUS_ERROR,
'error': traceback.format_exc()
}
yield self.save(data)
yield gen.sleep(self.frequency)
#gen.coroutine
def check(self):
pass
#gen.coroutine
def save(self, data):
yield events.save(self.event, data)
and this is the code that is inheriting from it:
class PostgreChecker(Checker):
# checks list of Post
formatter = 'stashboard.formatters.PostgreFormatter'
def __init__(self, event, frequency, params):
super().__init__(event, frequency, params)
self.clients = []
for DB in configuration["postgre"]:
# setup and create connections to PG servers.
postgreUri = queries.uri(DB["host"], DB["port"], DB["dbName"],
DB["userName"], DB["password"])
# creates actual link to DB
client = queries.TornadoSession(postgreUri)
# starts connection
client.host = DB["host"]
self.clients.append(client)
#gen.coroutine
def check(self):
for client in self.clients:
try:
yield client.validate()
self.save({'host': client.host,
'status': events.STATUS_OK})
except (ConnectionError, AutoReconnect, ConnectionFailure):
self.save({'host': client.host,
'status': events.STATUS_FAIL})
Tornado never calls close() on your generators, but the garbage collector does (starting in Python 3.4 I think). How is checker.run() called? Use IOLoop.spawn_callback() for fire-and-forget coroutines; this will keep a reference to them and allow them to keep running indefinitely.
the specific issue here was that my db cursors were not automatically re-connecting. I was using the queries library, but switched over to momoko and the issue is gone
in Python 2.7 I am successful in using the following code to listen to a direct message stream on an account:
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy import API
from tweepy.streaming import StreamListener
# These values are appropriately filled in the code
consumer_key = '######'
consumer_secret = '######'
access_token = '######'
access_token_secret = '######'
class StdOutListener( StreamListener ):
def __init__( self ):
self.tweetCount = 0
def on_connect( self ):
print("Connection established!!")
def on_disconnect( self, notice ):
print("Connection lost!! : ", notice)
def on_data( self, status ):
print("Entered on_data()")
print(status, flush = True)
return True
# I can add code here to execute when a message is received, such as slicing the message and activating something else
def on_direct_message( self, status ):
print("Entered on_direct_message()")
try:
print(status, flush = True)
return True
except BaseException as e:
print("Failed on_direct_message()", str(e))
def on_error( self, status ):
print(status)
def main():
try:
auth = OAuthHandler(consumer_key, consumer_secret)
auth.secure = True
auth.set_access_token(access_token, access_token_secret)
api = API(auth)
# If the authentication was successful, you should
# see the name of the account print out
print(api.me().name)
stream = Stream(auth, StdOutListener())
stream.userstream()
except BaseException as e:
print("Error in main()", e)
if __name__ == '__main__':
main()
This is great, and I can also execute code when I receive a message, but the jobs I'm adding to a work queue need to be able to stop after a certain amount of time. I'm using a popular start = time.time() and subtracting current time to determine elapsed time, but this streaming code does not loop to check the time. I just waits for a new message, so the clock is never checked so to speak.
My question is this: How can I get streaming to occur and still track time elapsed? Do I need to use multithreading as described in this article? http://www.tutorialspoint.com/python/python_multithreading.htm
I am new to Python and having fun playing around with hardware attached to a Raspberry Pi. I have learned so much from Stackoverflow, thank you all :)
I'm not sure exactly how you want to decide when to stop, but you can pass a timeout argument to the stream to give up after a certain delay.
stream = Stream(auth, StdOutListener(), timeout=30)
That will call your listener's on_timeout() method. If you return true, it will continue streaming. Otherwise, it will stop.
Between the stream's timeout argument and your listener's on_timeout(), you should be able to decide when to stop streaming.
I found I was able to get some multithreading code the way I wanted to. Unlike this tutorial from Tutorialspoint which gives an example of launching multiple instances of the same code with varying timing parameters, I was able to get two different blocks of code to run in their own instances
One block of code constantly adds 10 to a global variable (var).
Another block checks when 5 seconds elapses then prints var's value.
This demonstrates 2 different tasks executing and sharing data using Python multithreading.
See code below
import threading
import time
exitFlag = 0
var = 10
class myThread1 (threading.Thread):
def __init__(self, threadID, name, counter):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
def run(self):
#var counting block begins here
print "addemup starting"
global var
while (var < 100000):
if var > 90000:
var = 0
var = var + 10
class myThread2 (threading.Thread):
def __init__(self, threadID, name, counter):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
def run(self):
#time checking block begins here and prints var every 5 secs
print "checkem starting"
global var
start = time.time()
elapsed = time.time() - start
while (elapsed < 10):
elapsed = time.time() - start
if elapsed > 5:
print "var = ", var
start = time.time()
elapsed = time.time() - start
# Create new threads
thread1 = myThread1(1, "Thread-1", 1)
thread2 = myThread2(2, "Thread-2", 2)
# Start new Threads
thread1.start()
thread2.start()
print "Exiting Main Thread"
My next task will be breaking up my twitter streaming in to its own thread, and passing direct messages received as variables to a task queueing program, while hopefully the first thread continues to listen for more direct messages.