Python watchdog module duplicate events (edit: was not an watchdog issue) - python-3.x

I am creating a python script that will identify changes to a log file and print some data from the new logs.
I use watchdog to create an event handler and everything seems to work fine except from that, I get duplicate events every time I modify the file. I checked creation and delete, they both work as expected and trigger one time.
I have read the similar question which explains having a created and a modified event when I save a file but this is not my case. I just get two modification events.
Here is my code:
import os, sys, time
import subprocess
import threading
import win32print
from tkinter import filedialog
from tkinter import *
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
class Handler(FileSystemEventHandler):
# docstring for FileSystemEventHandler
def __init__(self, observer, filename, dirname):
# super(Handler, FileSystemEventHandler).__init__(self,)
self.observer = observer
self.filename = filename
self.dirname = dirname
print("Handler filename = " , self.filename)
print("Handler dirname = " , self.dirname)
def on_modified(self, event):
if self.filename == event.src_path:
print("The file was modified")
print (event.src_path)
# go get the last line and print the data
# try:
# hJob = win32print.StartDocPrinter (hPrinter, 1, ("test of raw data", None, "RAW"))
# try:
# win32print.StartPagePrinter (hPrinter)
# win32print.WritePrinter (hPrinter, raw_data)
# win32print.EndPagePrinter (hPrinter)
# finally:
# win32print.EndDocPrinter (hPrinter)
# finally:
# win32print.ClosePrinter (hPrinter)
def on_created(self, event):
print("A file was created (", event.src_path, ")")
def on_deleted(self, event):
print("A file was deleted (", event.src_path, ")")
if __name__ == "__main__":
Flags=2
Name=None
Level=1
printers = win32print.EnumPrinters(Flags, Name, Level)
print("\nChoose a printer to use:")
i=1
for p in printers:
print(i,')' , p[2])
i = i+1
if sys.version_info >= (3,):
raw_data = bytes ("This is a test", "utf-8")
else:
raw_data = "This is a test"
printer = int(input())
printer_name = printers[printer-1][2] #win32print.GetDefaultPrinter ()
print("You chose ", printer_name, "\nI will now print from the specified file with this printer")
hPrinter = win32print.OpenPrinter (printer_name)
# root = Tk()
# root.filename = filedialog.askopenfilename(initialdir = "/Desktop",title = "Select file",filetypes = (("log files","*.log"),("all files","*.*")))
file_path = "some_file_path" # root.filename
file_directory = os.path.dirname(file_path)
# print (file_path)
print (file_directory)
observer = Observer()
event_handler = Handler(observer, file_path, file_directory)
observer.schedule(event_handler, path=file_directory, recursive=False)
observer.start()
observer.join()
any ideas would be appreciated
EDIT:
After some debugging I found out that Windows10 is changing the file modification time twice every time I save it.
The proof of concept code is this:
prev_modification_time = os.path.getmtime(file_path)
while True:
current_mod_time = os.path.getmtime(file_path)
if prev_modification_time != current_mod_time :
print ("the file was modified, last modification time is: ", current_mod_time)
prev_modification_time = current_mod_time
pass
Final edit:
After testing my code on linux (Debian Stretch to be exact) it worked like a charm. So this combined with the previous edit probably shows that watchdog works fine and it is windows10 that has some issue. Should I post it on a different question or here?

Related

Cannot write CSV file from python

I'm been trying to create a csv file from this code, but it fails every time, I have tried different ways to place it inside the code but nothing has work so far.
I'm new to python and to Stack overflow.
If somebody can explain what I'm doing wrong it will be helpful.
Thanks in advance for any help.
from time import sleep
import os
import sys
from bleson import get_provider, Observer, UUID16
import csv
GOVEE_BT_mac_OUI_PREFIX = "A4:C1:38"
H5075_UPDATE_UUID16 = UUID16(0xEC88)
govee_devices = {}
# ###########################################################################
FORMAT_PRECISION = ".2f"
# Decode H5075 Temperature into degrees Fahrenheit
def decode_temp_in_f(encoded_data):
return format((((encoded_data / 10000) * 1.8) + 32), FORMAT_PRECISION)
# Decode H5075 percent humidity
def decode_humidity(encoded_data):
return format(((encoded_data % 1000) / 10), FORMAT_PRECISION)
#focus here
with open('temp.csv','w',newline='') as record:
record = csv.writer(record)
record.writerow(['Device Name','Device Address','Temp','Humidity'])
def print_values(mac):
govee_device = govee_devices[mac]
print(govee_device['name'],govee_device['address'],govee_device['tempInF'],govee_device['humidity'],govee_device['battery'])
record.writerow(govee_device['name'])
# On BLE advertisement callback
def on_advertisement(advertisement):
if advertisement.address.address.startswith(GOVEE_BT_mac_OUI_PREFIX):
mac = advertisement.address.address
if mac not in govee_devices:
govee_devices[mac] = {}
if H5075_UPDATE_UUID16 in advertisement.uuid16s:
# HACK: Proper decoding is done in bleson > 0.10
name = advertisement.name.split("'")[0]
encoded_data = int(advertisement.mfg_data.hex()[6:12], 16)
battery = int(advertisement.mfg_data.hex()[12:14], 16)
govee_devices[mac]["address"] = mac
govee_devices[mac]["name"] = name
govee_devices[mac]["mfg_data"] = advertisement.mfg_data
govee_devices[mac]["data"] = encoded_data
govee_devices[mac]["tempInF"] = decode_temp_in_f(encoded_data)
govee_devices[mac]["humidity"] = decode_humidity(encoded_data)
govee_devices[mac]["battery"] = battery
print_values(mac)
if advertisement.rssi is not None and advertisement.rssi != 0:
govee_devices[mac]["rssi"] = advertisement.rssi
# ###########################################################################
adapter = get_provider().get_adapter()
observer = Observer(adapter)
observer.on_advertising_data = on_advertisement
try:
while True:
observer.start()
sleep(2)
observer.stop()
except KeyboardInterrupt:
try:
observer.stop()
sys.exit(0)
except SystemExit:
observer.stop()
os._exit(0)
Error that Im getting is:
File "/home/pi/GoveeWatcher-master/python/goveeWatcher.py", line 37, in print_values
record.writerow(govee_device['name'])
ValueError: I/O operation on closed file.
I would be tempted to put the CSV writing functionality inside of the print_values function so it opens the file, writes the data, and then closes the file on each value that is found by the observer.
For example:
#focus here
def print_values(mac):
govee_device = govee_devices[mac]
print(govee_device['name'], govee_device['tempInF'])
with open('temp.csv','a',newline='') as record:
writer = csv.DictWriter(record, fieldnames=govee_device.keys())
writer.writerow(govee_device)

Display messages in window tkinter

I am working on the following small tkinter application.
The URL & token is for a public development demo which can be found here if interested - https://snipe-it.readme.io/reference#api-overview
My current issue is displaying logging messages to the current tkinter window.
Currently it just prints to the console. I've tried a few different methods but I'm not very good with tkinter at the moment.
I have tried solutions like so - How do I create a popup window in tkinter? - but this just duplicates the main window each time.
Any help or pointers would be appreciated.
The csv file looks like so:
asset_tag,
1382613192,
1169063140,
785100692,
4565667567345457574535,
import csv
import logging
from datetime import datetime
from multiprocessing import Process, Queue
from tkinter import *
from tkinter import filedialog, ttk
from tkinter.messagebox import showinfo
import requests
#todo set initialdir as current user
# initiate logging module
logging.basicConfig(level=logging.INFO)
# set headers are they are used in several functions
headers = {
"Accept": "application/json",
"Authorization": "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImp0aSI6ImVmMGVhY2Y4MjAyYzgwZWI2M2JkNmIwZDc0OGYwY2FkYzU2Y2ZlMzgyNzY4ODY0N2EwNmU4ZTBlNmYwZDgwODNjZmMyMzI2YWYyYTZlMTFkIn0.eyJhdWQiOiIxIiwianRpIjoiZWYwZWFjZjgyMDJjODBlYjYzYmQ2YjBkNzQ4ZjBjYWRjNTZjZmUzODI3Njg4NjQ3YTA2ZThlMGU2ZjBkODA4M2NmYzIzMjZhZjJhNmUxMWQiLCJpYXQiOjE0OTMzMzI2MjgsIm5iZiI6MTQ5MzMzMjYyOCwiZXhwIjoxODA4ODY1NDI4LCJzdWIiOiIyIiwic2NvcGVzIjpbXX0.NU7ZRIt-d4b0o8uv9ipo1vSWcg1svbmPp47kHErafm9iuK4FjygKd2_4Hp73HKAmjiYcEn3r39pwNh2t9BMFnTXv0KeDGC8zfZ9z7OJN_a59LPoarWBFzCsYETyAm-CeeFnfdj9Cr0ZeGOnnaPuWPYxicwKFeqJI4Hn8nCymcamDGE0u4WOO95ihGOAen4_fqpj-kkBDsvsGhB-cQxeuKdlbvO1yOsKmgQv-kQuxiFMn1zgU7P02mC6XXrbw6jTm7JOaBSbvqSwNtsrSKZkim1jxLsQ4dm36lFmeMkU6hZvNSUnxg8JwbmoxQ_3tZlG3IJh3Sc9ZUi-AEAQ4bbGzi_xNS9fenIdzLDaSiv_esYyNOYXqOuSBk8Yr-720N9OcVjGLnPrV3RtmPisV1aLFgKWLImtlyQgUq3d5LA3QXz8Q_8isvO9Am1u8ri2plbHGJLJ6GRW_mYcBEYMwUozaeXTUe_FUSSO8gpGtO9Hpa5SbERY272_tojyVXpYPaPdUYYmS9CP332jBNESPT8wGwpOM-iddeVo_n82w3dHmDEdp1Brbs3_vKk0AcgvDLsAbd4dZZO-UqddVx6SDb3HLw1Pmw1wGGYHA6w8wWQAiS9kg2xMcz5i75HOULaN3miqYvcPCvHpI2CBfuvdplI8QNm_XzFPmoQRu_5kR8knzla4",
"Content-Type": "application/json"
}
# functions
def check_in(id, asset_tag, headers):
# check in function, takes the asset ID and POST's to the url
# can optionally set a checkin note in the payload
url = "https://develop.snipeitapp.com/api/v1/hardware/"+id+"/checkin"
# modify if you would like a note with the checkin
payload = {"note": "checked in by ME"}
try:
response = requests.request("POST", url, json=payload, headers=headers)
checkin_response = response.json()
'''
This if statement is required here because if the asset is already checked in, it still returns an http successful message.
So this checks if the value in the status key is equal to "error" and if so then it will log this as an error.
'''
if "error" in checkin_response['status']:
logging.error("Unable to checkin asset %s - %s"
% (asset_tag, checkin_response['messages']))
else:
logging.info("Successfully checked in asset %s" % (asset_tag))
# catch any http errors and continue
except requests.exceptions.RequestException as e:
logging.error(e)
pass
def get_asset_id(asset_tag, output_q, headers):
# this function gets the asset id from a given asset tag
url = "https://develop.snipeitapp.com/api/v1/hardware/bytag/"+asset_tag+""
output_dict = {}
try:
response = requests.request("GET", url, headers=headers, verify=False)
response.raise_for_status()
json_response = response.json()
'''
This checks if the status key exists in the response and if it does then it returns an error.
This is because the status key does not exist at all in a successful query.
Again this is required because even if the asset tag doesn't exist, it still returns http 200.
'''
if "status" in json_response:
logging.error("Unable to retrieve asset ID for asset tag %s - %s"
% (asset_tag, json_response['messages']))
else:
logging.info("Successfully retrieved asset ID from %s" %
(asset_tag))
asset_id = str(json_response['id'])
check_in(asset_id, asset_tag, headers)
except requests.exceptions.RequestException as e:
logging.error(e)
pass
output_q.put(output_dict)
# class for tkinter configuration
class App:
def __init__(self, master):
self.label = ttk.Label(master, text="Bulk Checkin")
self.label.grid(row=0, column=0, columnspan=2)
ttk.Button(master, text="Select File",
command=self.select_file).grid(row=2, column=0)
ttk.Button(master, text="Checkin Assets",
command=self.checkin_assets).grid(row=2, column=1)
def select_file(self):
filename = filedialog.askopenfilename(initialdir="C:/Users/fraser/Desktop/", filetypes=(("CSV Files","*.csv"),))
self.infile = open(filename, "r")
print(self.infile.name)
def checkin_assets(self):
# function for Checkin Assets button click
# start a timer so we can see how long it takes
start_time = datetime.now()
output_q = Queue(maxsize=20)
procs = []
with open(self.infile.name, 'r') as myfile:
csv_reader = csv.DictReader(myfile)
line_count = 0
for row in csv_reader:
if line_count == 0:
", ".join(row)
line_count += 1
line_count += 1
asset_tag = row["asset_tag"]
my_proc = Process(target=get_asset_id, args=(
asset_tag, output_q, headers))
my_proc.start()
procs.append(my_proc)
# Make sure all processes have finished
for a_proc in procs:
a_proc.join()
# log how long this took
logging.info("\nElapsed time: " + str(datetime.now() - start_time))
def main():
main = Tk()
app = App(main)
main.mainloop()
if __name__ == "__main__":
main()
Here you go, this is assuming you want the messages as a popup. If not, create your own messaging window, or add a label or something you can update to the main one. I used the two messaging classes from the referenced post, but you can call them from anywhere and pass a message you want displayed.
import csv
import logging
from datetime import datetime
from multiprocessing import Process, Queue
from tkinter import *
from tkinter import filedialog, ttk
from tkinter.messagebox import showinfo
class App(ttk.Frame):
def __init__(self, master):
ttk.Frame.__init__(self, master)
self.label = ttk.Label(master, text="Bulk Checkin")
self.label.grid(row=0, column=0, columnspan=2)
ttk.Button(master, text="Select File",
command=self.select_file).grid(row=2, column=0)
ttk.Button(master, text="Checkin Assets",
command=self.checkin_assets).grid(row=2, column=1)
def select_file(self):
filename = filedialog.askopenfilename(initialdir="C:/Users/fraser/Desktop/", filetypes=(("CSV Files","*.csv"),))
self.infile = open(filename, "r")
print(self.infile.name)
def checkin_assets(self):
# function for Checkin Assets button click
# start a timer so we can see how long it takes
start_time = datetime.now()
output_q = Queue(maxsize=20)
procs = []
with open(self.infile.name, 'r') as myfile:
csv_reader = csv.DictReader(myfile)
line_count = 0
for row in csv_reader:
if line_count == 0:
", ".join(row)
line_count += 1
line_count += 1
asset_tag = row["asset_tag"]
my_proc = Process(target=get_asset_id, args=(
asset_tag, output_q, headers))
my_proc.start()
procs.append(my_proc)
# Make sure all processes have finished
for a_proc in procs:
a_proc.join()
# log how long this took
logging.info("\nElapsed time: " + str(datetime.now() - start_time))
def popup_bonus(self, message):
win = Toplevel()
win.wm_title("Message")
l = Label(win, text=message)
l.grid(row=0, column=0)
b = ttk.Button(win, text="Ok", command=win.destroy)
b.grid(row=1, column=0)
def popup_showinfo(self, message):
showinfo("Message", message)
def main():
main = Tk()
app = App(main)
main.mainloop()
if __name__ == "__main__":
main()
I use easygui because it is the simplest and one-liner. It can work alongside tkinter so there is no compatibility issues. For example, if you want a messagebox/alert, just do this:
import easygui
easygui.msgbox('in cmd, type: "shutdown -a" within 20 seconds', title="Chance to abort shutdown")

Watchdog in Python to look for filesystem changes not working (freezing)

I'm working on a Python script to monitor a folder to check whether a new *.JPG file is added to that folder and then do some tasks. The code is working, but after some time after being started, it seems to be freezing and stops working even a new file is added to the folder.
Here is the code:
# -*- encoding: iso-8859-1 -*-
import time
import os
import flickrapi
import shutil
from PIL import Image
from PIL.ExifTags import TAGS
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
if __name__ == "__main__":
patterns = ["*.jpg"]
ignore_patterns = None
ignore_directories = False
case_sensitive = False
my_event_handler = PatternMatchingEventHandler(patterns, ignore_patterns, ignore_directories, case_sensitive)
def get_exif(img):
''' Extract Exif data from image '''
def copyright(img):
# get the path of img and create an output filename
tail = os.path.split(img)[1]
filename = 'wtmk_' + tail[:-3] + 'png'
#open the base image and get it's dimensions
while True:
try:
# read file
base_image = Image.open(img)
bw, bh = base_image.size
bw2 = bw // 2
break
except IOError:
time.sleep(5)
''' Add watermark to image '''
return waterMarkedImage
def on_created(event):
# Add the watermark
file = copyright(event.src_path)
# extract EXIF data
exifdata = get_exif(event.src_path)
''' Send the image to FLICKR '''
# Create and start the observer
my_event_handler.on_created = on_created
path = "c:\\temp"
go_recursively = False
my_observer = Observer()
my_observer.schedule(my_event_handler, path, recursive=go_recursively)
my_observer.start()
try:
while True:
time.sleep(5)
except KeyboardInterrupt:
my_observer.stop()
my_observer.join()
I'm running the above code using Python 3.8 on a Windows 10 machine. Any help would be awesome!
Marcio
It's probably an issue with your cmd.exe
To fix the script freezing open your cmd, click on your cmd icon in the top left, then go to Defaults and uncheck the QuickEdit Mode.
Then restart cmd and start your script. Hope that helps!

How to find out how long a search for files will take on python?

So I have a little app that searches for all xml files on my pc, copying the files that have 44 digits as the filename to the "output" folder.
The problem is that the final user needs an indication of the progress and remaining time of the task.
This is the module to copy files:
xml_search.py
import os
import re
from threading import Thread
from datetime import datetime
import time
import shutil
import winsound
os.system('cls')
def get_drives():
response = os.popen("wmic logicaldisk get caption")
list1 = []
t1 = datetime.now()
for line in response.readlines():
line = line.strip("\n")
line = line.strip("\r")
line = line.strip(" ")
if (line == "Caption" or line == ""):
continue
list1.append(line + '\\')
return list1
def search1(drive):
for root, dir, files in os.walk(drive):
for file in files:
if re.match("\d{44}.xml", file):
filename = os.path.join(root, file)
try:
shutil.copy(filename, os.path.join('output', file))
except Exception as e:
pass
def exec_(callback):
t1 = datetime.now()
list2 = [] # empty list is created
list1 = get_drives()
for each in list1:
process1 = Thread(target=search1, args=(each,))
process1.start()
list2.append(process1)
for t in list2:
t.join() # Terminate the threads
t2 = datetime.now()
total = str(t2-t1)
print(total, file=open('times.txt', 'a'), end="\n")
for x in range(3):
winsound.Beep(2000,100)
time.sleep(.1)
callback()
if __name__ == "__main__":
exec_()
The below code uses progressbar library and it shows
indication of the progress and remaining time of the task
import progressbar
from time import sleep
bar = progressbar.ProgressBar(maxval=1120, \
widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.ETA()])
bar.start()
for i in range(1120):
bar.update(i+1)
sleep(0.1)
bar.finish()
You would need to add the above modified code to your code.
So in your case, you would need to count the number of files and provide it as input to ProgressBar constructor's maxval argument and remove sleep call.
The suggested solution with progress bar should work with one thread. You would need to figure out how to initiate the progress bar and where to put the updates if you insist to work with multiple threads.
Try to implement a timer decorator like the following:
import time
def mytimer(func):
def wrapper():
t1 = time.time()
result = func()
t2 = time.time()
print(f"The function {func.__name__} was run {t2 - t1} seconds")
return result
return wrapper
#mytimer
def TimeConsumingFunction():
time.sleep(3)
print("Hello timers")
TimeConsumingFunction()
Output:
/usr/bin/python3.7 /home/user/Documents/python-workspace/timers/example.py
Hello timers
The function TimeConsumingFunction was run 3.002610206604004 seconds
Process finished with exit code 0

How can I save output tho the same file that I have got the data from, in Python 3

I am trying to open a file, remove some characters (defined in dic) and then save it to the same the file.
I can print the output and it looks fine, but I cannot save it into the same file that the original text is being loaded from.
from tkinter import *
from tkinter.filedialog import askopenfilename
from tkinter.messagebox import showerror
import sys
import fileinput
dic = {'/':' ', '{3}':''};
def replace_all(text, dic):
for i, j in dic.items():
text = text.replace(i, j)
return text
class MyFrame(Frame):
def __init__(self):
Frame.__init__(self)
self.master.title("Example")
self.master.rowconfigure(5, weight=1)
self.master.columnconfigure(5, weight=1)
self.grid(sticky=W+E+N+S)
self.button = Button(self, text="Browse", command=self.load_file, width=10)
self.button.grid(row=1, column=0, sticky=W)
def load_file(self):
fname = askopenfilename(filetypes=(("Napisy", "*.txt"),
("All files", "*.*") ))
if fname:
try:
with open (fname, 'r+') as myfile: #here
data = myfile.read() #here
data2 = replace_all(data, dic) #here
print(data2) #here
data.write(data2) #and here should it happen
except:
showerror("Open Source File", "Failed to read file\n'%s'" % fname)
return
if __name__ == "__main__":
MyFrame().mainloop()
I have tried several commands but either I am receiving python errors or it is simply not working.
This is often implemented by writing to a temp file and then moving it to the original file's name.
Strings do not have a .write method. The following should work (I tried it): replace
data.write(data2) #and here should it happen
with
myfile.seek(0)
myfile.truncate()
myfile.write(data2)
The truncate() call is needed if data2 is shorter than data as otherwise, the tail end of data will be left in the file.

Resources