Unable to repopulate a queue in python 3 - multithreading

The code is getting exited when I run the following code
import mysql.connector
import mysql.connector.pooling as x
import time
from threading import Thread
from queue import Queue
dbconfig = {
"database": "mydb",
"user": "root",
"password": "rootpass",
"host":"127.0.0.1"
}
cnxpool =x.MySQLConnectionPool(pool_name = "mypool",pool_size = 2,**dbconfig)
q = Queue(maxsize=20)
threads = []
limit = 0
fetch_size = 20
flag = True
def populate():
cnx = cnxpool.get_connection()
cursor = cnx.cursor()
global flag,fetch_size,limit
while flag:
cursor.execute("SELECT mycol from mytab limit " + str(limit)+","+ str(fetch_size))
rows = cursor.fetchall()
for row in rows:
print("Putting "+ str(row[0]))
q.put(row[0])
if cursor.rowcount < 20 :
for i in range(4):
q.put(None)
flag = False
limit = limit + 20
cursor.close()
cnx.close()
def consume():
while True:
item = q.get()
if item is None:
q.task_done()
break
time.sleep(1)
print(item)
q.task_done()
for i in range(4):
t = Thread(target=consume)
t.start()
threads.append(t)
y = Thread(target=populate)
y.start()
q.join()
y.join()
for t in threads:
t.join()
print("Completed")
Code is getting completed in jupyter notebook
But when I try to repopulate the queue even after removing the variable it hangs.
import mysql.connector
import mysql.connector.pooling as x
import time
from threading import Thread
from queue import Queue
dbconfig = {
"database": "mydb",
"user": "root",
"password": "rootpass",
"host":"127.0.0.1"
}
cnxpool =x.MySQLConnectionPool(pool_name = "mypool",pool_size = 2,**dbconfig)
q = Queue(maxsize=20)
threads = []
limit = 0
fetch_size = 20
flag = True
def populate():
cnx = cnxpool.get_connection()
cursor = cnx.cursor()
global flag,fetch_size,limit
while flag:
cursor.execute("SELECT mycol from mytab limit " + str(limit)+","+ str(fetch_size))
rows = cursor.fetchall()
for row in rows:
print("Putting "+ str(row[0]))
q.put(row[0])
if cursor.rowcount < 20 :
for i in range(4):
q.put(None)
flag = False
limit = limit + 20
cursor.close()
cnx.close()
def consume():
while True:
item = q.get()
if item is None:
q.task_done()
break
time.sleep(1)
print(item)
q.task_done()
for i in range(4):
t = Thread(target=consume)
t.start()
threads.append(t)
y = Thread(target=populate)
y.start()
q.join()
y.join()
for t in threads:
t.join()
print("Completed")
del q
q = Queue(maxsize=20)
for i in range(4):
t = Thread(target=consume)
t.start()
threads.append(t)
y = Thread(target=populate)
y.start()
q.join()
y.join()
for t in threads:
t.join()
print("Completed")
It hangs for ever. Even creating a new queue doesn't help. Where am I going wrong. Kindly guide me.
Thanks in advance.
I am using Python 3.

Related

Open multiple windows per time interval, using same long function, receiving different data with Python and PySimpleGui

I'm making a system that receives a list of login data (username and password), then opens each user and password in a window, consuming the same function that I receive as user and password parameters.
Below is a reduced example of what I'm doing.
from multiprocessing.dummy import Process
import PySimpleGUI as sg
from time import sleep
import threading
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import multiprocessing as mp
from multiprocessing import Process
import os, json
from datetime import datetime, timezone
import time
import random
HOME = './Dados'
def current_location(self):
geometry = self.TKroot.geometry()
location = geometry[geometry.find('+') + 1:].split('+')
x, y = int(location[0]), int(location[1])
return(x, y+-30)
sg.Window.current_location = current_location
def function_thread(usuario, senha):
global window
window['output'].print(f'Login {usuario} e {senha}')
for i in range(5):
#print(f'Tarefas no perfil {usuario}')
#window['output'].update(str('Tarefas no perfil {usuario}'))
time.sleep(random.randint(1,2))
horaatual = datetime.today().strftime('[%H:%M:%S:%f] - ')
window['output'].print(horaatual+': Tarefas: '+ str(i))
#sleep(2)
#window.write_event_value('output', i)
#window.write_event_value('output', usuario)
window['output'].print(f'Completado {usuario}')
#sleep(1)
x=-7
y=515
def abrir_abas():
global x,y, usuario_insta, senha_insta
window2 = None
cont = 0
#list_user = [('perfil1','senha1'),('perfil2', 'senha2'),('perfil3', 'senha3'),('perfil4', 'senha4'),('perfil5','senha5'),('perfil6', 'senha6'),('perfil7', 'senha7'),('perfil8', 'senha8')]
'''
executor = concurrent.futures.ProcessPoolExecutor(8)
futures = [executor.submit(function_thread(group[0],group[1])for group in list_user)]
concurrent.futures.wait(futures)
'''
with open(os.path.join(HOME, f'Config/abas.txt'), 'r') as f:
abas = f.readlines()
lista_conta = []
for index_lista in range(len(abas)):
abas[index_lista] = abas[index_lista].rstrip('\n')
if abas[index_lista] != '' or None:
lista_conta.append(abas[index_lista])
#with concurrent.futures.ProcessPoolExecutor() as executor:
for i_lista_conta in range(len(lista_conta)):
qtd_lista_conta = int(len(lista_conta)/2)
if i_lista_conta % 2 == 0:
usuario = lista_conta[i_lista_conta].split(' ')
usuario_insta = usuario[0]
senha_insta = usuario[1]
#self.output(usuario_insta, senha_insta)
titulo = 'TAB '+usuario_insta
window2 = output(usuario_insta, senha_insta, loc=(x,y))
window2.TKroot.title(titulo)
t1 = threading.Thread(target=function_thread(usuario_insta, senha_insta), daemon=True)
t1.start()
y=y+-30
y=y
cont = cont + 1
'''
processos = []
for i_lista_conta in range(len(lista_conta)):
qtd_lista_conta = int(len(lista_conta)/2)
if i_lista_conta % 2 == 0:
usuario_insta = lista_conta[i_lista_conta]
senha_insta = lista_conta[i_lista_conta+1]
#self.output(usuario_insta, senha_insta)
titulo = 'TAB '+usuario_insta
window2 = output(loc=(x,y))
window2.TKroot.title(titulo)
#threading.Thread(target=self.thread_acao(usuario_insta, senha_insta), daemon=True).start()
p = Process(target=function_thread(usuario_insta, senha_insta))
p.start()
processos.append(p)
y=y+-30
y=y
cont = cont + 1
'''
#for process in processos:
# process.join()
'''
for i in list_user:
titulo = 'TAB '+i[0]
#with concurrent.futures.ProcessPoolExecutor() as executor:
# window2 = executor.map(output(loc=(x,y)))
window2 = output(loc=(x,y))
window2.TKroot.title(titulo)
#with concurrent.futures.ProcessPoolExecutor() as executor:
# executor.map(function_thread(i[0],i[1]))
#threading.Thread(target=function_thread(i[0],i[1]), daemon=True).start()
Process(target=function_thread(i[0],i[1])).start()
y=y+-30
y=y
cont = cont + 1
sleep(2)
'''
def output(login, senha, loc=(-7, 515))->sg.Window:
global window
voltar = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsTAAALEwEAmpwYAAAF8WlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDggNzkuMTY0MDM2LCAyMDE5LzA4LzEzLTAxOjA2OjU3ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHhtcDpDcmVhdGVEYXRlPSIyMDIwLTAzLTAzVDA5OjUwOjM0LTAzOjAwIiB4bXA6TW9kaWZ5RGF0ZT0iMjAyMC0wNS0wMlQxNzo1Nzo0Mi0wMzowMCIgeG1wOk1ldGFkYXRhRGF0ZT0iMjAyMC0wNS0wMlQxNzo1Nzo0Mi0wMzowMCIgZGM6Zm9ybWF0PSJpbWFnZS9wbmciIHBob3Rvc2hvcDpDb2xvck1vZGU9IjMiIHBob3Rvc2hvcDpJQ0NQcm9maWxlPSJzUkdCIElFQzYxOTY2LTIuMSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo0MGZlY2VlMS0yYmE4LTI4NGQtYTk4NC0zYjQxNTIxNzA1ZGQiIHhtcE1NOkRvY3VtZW50SUQ9ImFkb2JlOmRvY2lkOnBob3Rvc2hvcDo2MmQ1MzYwYi0zMTQ5LTI2NDAtYWZhMC03YzdiNjkzNDFkY2YiIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDowZDY2MTNmZi0wMGIzLWMzNDQtYTgzOS1jMTUwZTc4ZGZlODEiPiA8eG1wTU06SGlzdG9yeT4gPHJkZjpTZXE+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJjcmVhdGVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjBkNjYxM2ZmLTAwYjMtYzM0NC1hODM5LWMxNTBlNzhkZmU4MSIgc3RFdnQ6d2hlbj0iMjAyMC0wMy0wM1QwOTo1MDozNC0wMzowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDIxLjAgKFdpbmRvd3MpIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo0MGZlY2VlMS0yYmE4LTI4NGQtYTk4NC0zYjQxNTIxNzA1ZGQiIHN0RXZ0OndoZW49IjIwMjAtMDUtMDJUMTc6NTc6NDItMDM6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyMS4wIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz4M/LCOAAABD0lEQVQ4y43SMSvFURjH8XPvVchCScpwB2VRykIpG7u7KLeMZGGQd6BMXoDZwAtQUgaTkZKNRalPSgYZrAbn1On0v/f+Tz2d0/P0fM/z/J4nIBTWyN5rOMU5zrCD4RhrIvRKHsEFXmLiIU5wjycsJUioqCDgCtcYrYgd+D/zZQWteO/FX3J/I5acKjzGTVULAXfYiu+hHi2O4w0LZWAy/j5XIWhpl+iUgGk8YLYGYAJjVYDHmoCQNGhmApaAVpp3P0BuJSAMqiRgFSvRMRM1SCIuYr0fJKCDTyzHNX3GFNr4wG6+ur1a2MY7urjFBl5x1C85FzFBvvGFn7iyoa6IaQqb+MV+4a81hSRSe1BSbn+7uNbAq7k3vAAAAABJRU5ErkJggg=='
pause = 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAACXBIWXMAABYlAAAWJQFJUiTwAAAGvmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDggNzkuMTY0MDM2LCAyMDE5LzA4LzEzLTAxOjA2OjU3ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHhtcDpDcmVhdGVEYXRlPSIyMDIwLTA1LTAyVDE3OjQ4OjI0LTAzOjAwIiB4bXA6TW9kaWZ5RGF0ZT0iMjAyMC0wNS0wM1QxMTowNjo0NC0wMzowMCIgeG1wOk1ldGFkYXRhRGF0ZT0iMjAyMC0wNS0wM1QxMTowNjo0NC0wMzowMCIgZGM6Zm9ybWF0PSJpbWFnZS9wbmciIHBob3Rvc2hvcDpDb2xvck1vZGU9IjMiIHBob3Rvc2hvcDpJQ0NQcm9maWxlPSJzUkdCIElFQzYxOTY2LTIuMSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo0ODFlYzZjZS1jYTNhLWIxNDgtYjlmYy0yZmZmYjVhNzI3ZTEiIHhtcE1NOkRvY3VtZW50SUQ9ImFkb2JlOmRvY2lkOnBob3Rvc2hvcDo1NmE4ZDEzNC0wNmIzLWVkNGMtOWYwYi1kMjk4YmU1YTlmYzAiIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDo0ZmQ2MzhkYy05YzYyLTFjNGUtYjg2Ny03ZDNmOTdlMmE5YTgiPiA8eG1wTU06SGlzdG9yeT4gPHJkZjpTZXE+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJjcmVhdGVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjRmZDYzOGRjLTljNjItMWM0ZS1iODY3LTdkM2Y5N2UyYTlhOCIgc3RFdnQ6d2hlbj0iMjAyMC0wNS0wMlQxNzo0ODoyNC0wMzowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDIxLjAgKFdpbmRvd3MpIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo0N2MwYWRlMy02NmVjLWVjNDEtYjE1YS0zNmI2ZmZmOGEzNWYiIHN0RXZ0OndoZW49IjIwMjAtMDUtMDJUMTc6NTI6MTYtMDM6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyMS4wIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8cmRmOmxpIHN0RXZ0OmFjdGlvbj0ic2F2ZWQiIHN0RXZ0Omluc3RhbmNlSUQ9InhtcC5paWQ6NDgxZWM2Y2UtY2EzYS1iMTQ4LWI5ZmMtMmZmZmI1YTcyN2UxIiBzdEV2dDp3aGVuPSIyMDIwLTA1LTAzVDExOjA2OjQ0LTAzOjAwIiBzdEV2dDpzb2Z0d2FyZUFnZW50PSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHN0RXZ0OmNoYW5nZWQ9Ii8iLz4gPC9yZGY6U2VxPiA8L3htcE1NOkhpc3Rvcnk+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+sKbvRwAAAWBJREFUOI3t1L2KFFEQBeCvZ8afBZENFmEEBU0MxGQfQBE00jfYh9DUFxBBQzMx8A00MlVMDQVRFARxXDN1BWfH6W2DPpe5jLA7xnqS+1NV51Rfuk4zmUwE6ziR/R5G+IFJznJ3CkfwK+c5PmE6Ho+NLHAW59CF4BC+YgffkrOBTRzFLDltYu+L4nrInuNYFGuhO7iV4ge4XuW0GOItLmF7hDEuhGwLr1I8x31cDvEaLuIR7uWzZxG4jdOFUJQ6vMDHqrsPOJN9F4I3ES04mXUOg7zXHpq8UY3jJTGEbe5qbFRxA/ujOyD+R85BhH+N/4T/ImGzQs2+/+ZAP9xlBD8vxXdwuBIrDlSj1AxZOEbxuy29cwxzv2lhXcXSruC13sKmuFZ3PtKbQasf+LtL6jPcyH6Kh7iJq0t5j/GuEP7EFzzBd+ymmzVs42WKWjzD+XS3m44bPM3zaLpulflfHb8BEgRaa6It0wUAAAAASUVORK5CYII='
col_v = sg.Column(
[
[sg.Button(image_data=pause, size=(None,20), key='Executar',button_color=(sg.theme_background_color(), sg.theme_background_color()), border_width=0)],
[sg.Text('')],
[sg.Button(image_data=voltar, key='voltar',tooltip="Voltar",button_color=(sg.theme_background_color(), sg.theme_background_color()), border_width=0)],
], k='c_v', visible=True)
main_layout =[
#[sg.Titlebar(title = texto, key = 'titulo')],
[sg.StatusBar('O Rei Dos BOTs',justification='center', font='Arial 12')],
[col_v,sg.Multiline(size=(20,2),expand_x=True, expand_y=True, key='output')],
[sg.StatusBar('INICIANDO', k='status+', auto_size_text=True, justification='center', font=('Arial', 12, 'bold'), text_color='#FFFFFF', background_color=None, s=(None, 1))],
]
window = sg.Window('ABA', size=(253,181), finalize=True, location=loc, background_color=None, auto_size_text=True, font='Arial 7', resizable=True, auto_size_buttons=False,right_click_menu = False, margins=(1, 1)).Layout(main_layout).Finalize()
#with concurrent.futures.ProcessPoolExecutor() as executor:
# executor.map(function_thread(login, senha))
return window
def main_output():
global window, usuario_insta, senha_insta
sg.theme('Topanga')
layout = [ [sg.Text('Janela')],
[sg.Text(size=(10,1), key='-OUT-')],
[sg.Button('Executar'), sg.Button('Sair')],
[sg.StatusBar('INICIANDO', k='status', auto_size_text=True, justification='center', font='Arial 12', text_color='#FFFFFF', background_color=None, s=(None, 1))],
]
window = sg.Window('Janela principal', layout, size=(253,181), use_default_focus=False,finalize=True, use_custom_titlebar=False, no_titlebar=False)
while True:
event, values = window.read()
if event == sg.WIN_CLOSED or event == 'Sair':
break
if event == 'Executar':
window.hide()
#with ThreadPoolExecutor() as executor:
# executor.map(abrir_abas())
#threading.Thread(target=abrir_abas(), daemon=True).start()
abrir_abas()
#threading.Thread(target=abrir_abas(), args=(window,), daemon=False).start()
#with concurrent.futures.ProcessPoolExecutor() as executor:
# executor.map(abrir_abas())
if event == 'Ok':
print('Completado')
if event == 'voltar':
print('Voltar')
window.close()
#threading.Thread(target=main_output(), args=(window,), daemon=True).start()
main_output()
I can only do this by opening the first window and waiting for the function to complete and then opening the second window and so on.
The function is looping and can take more than 24 hours.

Python loop does not work accurately and reliably with time.sleep

What would be the best way to run a program once a day.
time.sleep is inaccurate on long runs.
I try to get def send_stock_data (self): run once a day, for example. time.sleep drifts too much for this purpose.
The program reads the excel file and sends the data by e-mail if the column 'Amount' is 0.
class progloop():
def prog_loop (self):
progress.pack_forget()
while run_cycle == True:
get_stock_and_send_mail()
sleep(48000)
continue
def __init__(self):#adds Threading to loop
t = threading.Thread(target=self.prog_loop)
t.start()
The above class currently handles the loop.
Below is the full script.
import threading
import tkinter as tk
import os
import smtplib
import datetime
import time
import pandas as pds
import traceback
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
from tkinter import BOTTOM,HORIZONTAL, Label, Menu, PhotoImage, messagebox, filedialog, ttk
from datetime import datetime
from time import sleep
root = tk.Tk()
sec = 0
action = None
run_cycle = True
stop_cycle = True
today = datetime.now()
clock_time = today.strftime('%H:%M:%S')
date = today.strftime('%d/%m/%Y')
weekday = datetime.today().strftime('%A')
def on_run():#start send_stock_data loop and timer
global run_cycle, action
global action
string_var.set('Application state: Running')
run_cycle = True
if not action:
timer()
def on_stop():#pause send_stock_data loop and timer
global run_cycle, action
string_var.set('Application state: Paused')
run_cycle = False
messagebox.showinfo('Attention','Program is paused, press Run to resume execution')
if action:
root.after_cancel(action)
action = None
def Info():#Info btn data
messagebox.showinfo('Application info','''-The application reads the stock balance from the excel file and sends out e-mail information if the part (s)
of the warehouse balance has run out of stock within the alarm limit.\n
-The default path to the excel file is specified in the Windows environment variables as 'FilePath'\n
-In case of problems contact the author
\n\nDefault alarm limit = 0 parts in stock
\n\nDefault Google smtp server:'smtp.gmail.com'\nDefault port: 587 ''')
def write_err_log():#writes an error log with traceback
traceback_err = traceback.format_exc()
date_time = [clock_time,date]
write_log = open("ErrorLog.txt", "a")
write_log.write(str(date_time)) and write_log.write(str(traceback_err))
write_log.close()
#Gui
string_var = tk.StringVar()
string_var.set('Application state: Waiting for commands')
root.iconbitmap(r'C:\\Users\\Lauri\\Visual_studio\\Bitmap_2.ico')
root.title('Warehouse assistant')
root.geometry('460x360')
root.resizable(False,False)
background_image = PhotoImage(file = r'C:\\Users\\Lauri\\Visual_studio\\BcImage.png')
background_label = tk.Label(image=background_image)
background_label.place(x=0, y=0, relwidth=1, relheight=1)
Label_header = ttk.Label(root, font=('underline', 12),text='Parts balance monitoring application').pack()
Label_start = ttk.Label(root,font=('underline',8), text='Press Run to start the application\nPress Pause to stop the application').pack()
Label_appState = ttk.Label(root, font=('underline', 10),textvariable=string_var).pack()
btnInfo = ttk.Button(root, text = 'Info', width= 15, command = Info).pack()
Start_btn = ttk.Button(root, text = 'Run', width= 15, command=lambda:[bar(),progloop(),root.after(1000,on_run())]).pack()
Stop_btn = ttk.Button(root, text = 'Pause', width= 15, command = on_stop).pack()
exit_btn = ttk.Button(root, text = 'Exit', width= 25, command =lambda:[root.after(1000,),root.destroy()]).pack(side=BOTTOM)
menu = Menu(root)
root.config(menu=menu)
contact_num = os.environ.get('Contact_number')
contact_email = os.environ.get('Contact_email')
helpmenu = Menu(menu)
menu.add_cascade(label='Contact', menu=helpmenu)
helpmenu.add_command(label=contact_email)
helpmenu.add_command(label=contact_num)
helpmenu.add_command(label= 'Exit', command=root.destroy)
progress = ttk.Progressbar(root, orient = HORIZONTAL,length = 158, mode = 'determinate')
def bar():#progressbar
progress['value'] = 5
root.update_idletasks()
root.after(50)
progress['value'] = 43
root.update_idletasks()
root.after(300)
progress['value'] = 68
root.update_idletasks()
root.after(50)
progress['value'] = 100
root.update_idletasks()
root.after(1)
progress.pack(side=BOTTOM)
class get_stock_and_send_mail: #send inventory balance information by email
def send_stock_data(self):
count = 0
pds.set_option('display.max_colwidth', -1)
pds.set_option('colheader_justify','left')
#reads excel file and searches for desired inventory balance value
while True:
file = os.environ.get('FilePath')#default path to file specified in the Windows environment variables as 'FilePath'
try:
dataFrame = pds.read_excel(file)
break
except PermissionError as error:
write_err_log()
messagebox.showerror(title='error',message= 'Permission Error Permission error has occured during read file?\n\n\n'+str(error)+"'.",
detail=traceback.format_exc())
except FileNotFoundError as error:
messagebox.showwarning(title='Select Excel File',
message='DEFAULT FILE NOT FOUND.\nChoose Excel file from directory.\nCONTINUE --> PRESS OK')
file = filedialog.askopenfilename()#manual path to file if default path doesn't exist
try:
dataFrame = pds.read_excel(file)
break
except FileNotFoundError:
write_err_log()
continue
df_bool = []
for Amount in dataFrame.Amount:
if Amount <= 0:
df_bool.append(True)
else:
df_bool.append(False)
if True in df_bool:
while run_cycle == True:
today = datetime.now()
clock_time = today.strftime('%H:%M:%S')
date = today.strftime('%d/%m/%Y')
weekday = datetime.today().strftime('%A')
Email_adress = os.environ.get('email_user')
Email_password = os.environ.get('email.password')
empty_stock = dataFrame.loc[dataFrame.Amount <= 0]
try:
with smtplib.SMTP('smtp.gmail.com',587) as server:
server.ehlo()
server.starttls()
server.ehlo_or_helo_if_needed()
server.set_debuglevel(1)
server.login(Email_adress,Email_password)
subject = 'no-reply'
body = f"""Row no.:{empty_stock}\nPart (s) is out of stock\n
Date: {weekday} {date}\n
Time: {clock_time}
\n\n\nThis was an automatically generated message.\nDo not reply to this message!
"""
msg = f'subject:{subject}\n\n{body}'
server.sendmail(Email_adress,Email_adress, msg)
server.quit
except smtplib.SMTPException as error:
write_err_log()
count +=1
if count <= 5:
continue
elif count >= 6:
messagebox.showerror(title='error',message= 'ERROR'+str(error)+"'.",
detail=traceback.format_exc())
break
def __init__(self):#adds Threading to loop
t = threading.Thread(target=self.send_stock_data)
t.start()
class progloop():
def prog_loop (self):
progress.pack_forget()
while run_cycle == True:
get_stock_and_send_mail()
sleep(48000)
continue
def __init__(self):#adds Threading to loop
t = threading.Thread(target=self.prog_loop)
t.start()
def timer(): #Runtime timer
global sec
if run_cycle == False:
return
sec += 1
time['text'] = f'Runtime: {sec} sec'
time.after(1000, timer)
time = Label(root, fg='black')
time.pack(side=BOTTOM)
root.mainloop()
If you really want to actually use a sleep, and time shifting/drifting over multiple days is the problem, what you can do, although not perfectly accurate, is to readjust the sleep time based on current time. Something like :
def seconds_since_midnight():
now = datetime.now()
return (now - now.replace(hour=0, minute=0, second=0, microsecond=0)).total_seconds()
def one_day_to(day_time_secs):
return 86400+day_time_secs-seconds_since_midnight()
And using for instance
sleep(one_day_to(3600*18))
to schedule at about 18h00 the next day, without drifting from day to day

How can I speed up this loop using multiprocessing or multithreading?

I am afraid that I'm not doing the multithreading thing the right way, so I came here in search of wisdom. I have two arrays of addresses and I have to check if the address of the first array exists in the second array and in case it doesn't look for the most similar address in array 2.
The array that has the "oficial" addresses is called directory and the array that I need to validate is called look_address.
The code goes as follows:
import pandas as pd
import numpy as np
from fuzzywuzzy import fuzz
from fuzzywuzzy import process
from datetime import datetime,timedelta
import threading
import queue
class myThread(threading.Thread):
def __init__(self,threadID,name,q):
threading.Thread.__init__(self)
self.threadID = threadID
self.name=name
self.q = q
def run(self):
print(f"starting {self.name}")
process_data(self.name,self.q)
print(f"ending {self.name}")
locs = []
ratios={}
def process_data(threadName,q):
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
d = q.get()
queueLock.release()
d = d.strip()
if directory.isin([d]).any():
locs.append(d)
else:
pos = process.extract(d,directory.values,scorer=fuzz.ratio,limit=50)
ratios[d] = pos
else:
queueLock.release()
threadlist = ["T-1","T-2","T-3","T-4","T-5","T-6","T-7","T-8","T-9","T-10"]
nameList = look_address
queueLock = threading.Lock()
workQueue = queue.Queue(len(nameList)+1)
threads=[]
threadID=1
exitFlag=0
for name in threadlist:
thread = myThread(threadID,name,workQueue)
thread.start()
threads.append(thread)
threadID+=1
queueLock.acquire()
for addr in nameList:
workQueue.put(addr)
queueLock.release()
total_steps = len(workQueue.queue)
tot_sec = 0
t0 = datetime.now()
while not workQueue.empty():
total_seconds =(datetime.now()-t0).total_seconds()
if total_seconds == 0:
total_seconds = 1e-8
progress = 1-len(workQueue.queue)/total_steps
tot_sec+=total_seconds
print("\rProgreso: {pr:.2f}% || Buenas/Errores: {gb}/{bd}".format(
pr = progress*100,
its = 1/total_seconds,
elap = timedelta(seconds=np.round(tot_sec)),
gb=len(locs),
bd=len(errors),
eta = timedelta(seconds=np.round(total_seconds*(total_steps-len(workQueue.queue))))),end="",flush=True)
exitFlag = 1
for t in threads:
t.join()
print("\nExiting Main Thread")
Each request in process.extract takes around 25s (did a %timeit). Now, with the script above it doesn't seems to speed up the data processing. It has been running for like 2 hours and it has progressed by around 4.29%.
My two questions are:
Is the implementation of multithreading correct?
How can I speed up the data processing? Maybe run this on a VPS on amazon or google?
I want to understand why this is so slow and how I can speed things up.
EDIT: Changed from:
if not workQueue.empty():
d = q.get()
d = d.strip()
if directory.isin([d]).any():
locs.append(d)
else:
pos = process.extract(d,directory.values,scorer=fuzz.ratio,limit=50)
ratios[d] = pos
queueLock.release()
to:
if not workQueue.empty():
d = q.get()
queueLock.release()
d = d.strip()
if directory.isin([d]).any():
locs.append(d)
else:
pos = process.extract(d,directory.values,scorer=fuzz.ratio,limit=50)
ratios[d] = pos

Python 3.5 Async & await Ping

I try to make Async ping process using subprocess.Popen , I try to understand how i implement it in this case
aList = []
async def sn(frm, to):
i = 0
for i in list(range(frm, to)):
aList.append(i)
cmd = "ping -n 1 " + '10.0.0.'
coroutines = [subprocess.Popen(cmd + str(i), stdout=subprocess.PIPE) for i in aList]
results = await asyncio.gather(*coroutines)
print(results)
loop = asyncio.get_event_loop()
loop.run_until_complete(sn(frm, to))
loop.close()
You can find simpler code for pinging host without async-await. But if necessary you can try the following working example to ping with async-await
import platform
import subprocess
import aiohttp
import asyncio
async def getPingedHost(host, netTimeout=3):
""" Description: Function to ping a host and get string of outcome or False
Import: from shared.getPingedHost import getPingedHost
Testing: python -m shared.getPingedHost
"""
args = ['ping']
platformOs = platform.system().lower()
if platformOs == 'windows':
args.extend(['-n', '1'])
args.extend(['-w', str(netTimeout * 1000)])
elif platformOs in ('linux', 'darwin'):
args.extend(['-c', '1'])
args.extend(['-W', str(netTimeout)])
else:
raise NotImplemented('Unsupported OS: {}'.format(platformOs))
args.append(host)
output = ''
try:
outputList = []
if platformOs == 'windows':
output = subprocess.run(args, check=True, universal_newlines=True,
stdout=subprocess.PIPE, # Capture standard out
stderr=subprocess.STDOUT, # Capture standard error
).stdout
outputList = str(output).split('\n')
if output and 'TTL' not in output:
output = False
else:
subprocess.run(args, check=True)
output = outputList[2]
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
output = False
return output
async def main():
async with aiohttp.ClientSession() as client:
output = await getPingedHost('google.com')
print(output)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
class rscan(object):
state = {'online': [], 'offline': []} # Dictionary with list
ips = [] # Should be filled by function after taking range
# Amount of pings at the time
thread_count = 8
# Lock object to prevent race conditions
lock = threading.Lock()
# Using Windows ping command
def ping(self, ip):
answer = subprocess.call(['ping','-n','1',ip],stdout = open('1.txt','w'))
return answer == 0 and ip
def pop_queue(self):
ip = None
self.lock.acquire() # lock !!!
if self.ips:
ip = self.ips.pop()
self.lock.release()
return ip
def noqueue(self):
while True:
ip = self.pop_queue()
if not ip:
return None
result = 'online' if self.ping(ip) else 'offline'
self.state[result].append(ip) ### check again
def start(self):
threads = []
for i in range(self.thread_count):
t = threading.Thread(target=self.noqueue)
t.start()
threads.append(t)
# Wait for all threads
[ t.join() for t in threads ]
return self.state
def rng(self, frm, to, ip3):
self.frm = frm
self.to = to
self.ip3 = ip3
for i in range(frm, to):
ip = ip3 + str(i)
self.ips.append(ip)
if __name__== '__main__':
scant = rscan()
scant.thread_count = 8
edited a bit class i have found also used threads instead of Async & await
Credit: http://blog.boa.nu/2012/10/python-threading-example-creating-pingerpy.html

2 threads each serving 2 different HTTP Server, will not stop after a given duration

So I have spent at least a couple of days on this problem.
I would like to have 2 threads HTTP server each serving two different IP:ports.
I open a FF and navigate to either say: http://196.64.131.250:8001/ or http://196.64.131.250:8002 and it should do a GET.
Also I like my threads or program itself stop after a given duration from command line say 5 sec.
I have done everything I could, even tried SIGAlarm and variable "keepRunning" which gets changed by a third thread after duration , but my program does Not stop. What am I doing wrong please.
note that I have commented the daemon: i.e (ZhttpThread[1-2].daemon = True)
if I dont comment it my threads stop right away. I want my HTTP server threads continue servicing, and if the duration DUR expires , then the program stops or threads stop.
import SimpleHTTPServer, SocketServer, logging, cgi, sys, signal, threading, time
import subprocess as sub
keepRunning = True
origTime = int(time.time())
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
logging.warning("======= GET STARTED =======")
getHdr = self.headers
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
getHdr = self.headers
print(', '.join((getHdr)))
#if ("accept-encoding" in getHdr):
if ("accept-encoding" in (', '.join((getHdr)))):
print ('Test Passed ---------------')
signal.alarm(1)
class threadWithTO(threading.Thread):
def __init__(self, thID, ip, port, timeout):
threading.Thread.__init__(self)
self.ip = ip
self.port = port
self.handler = ServerHandler
self.httpd = SocketServer.TCPServer((self.ip, self.port), self.handler)
def run(self):
print (self.httpd)
#self.httpd.serve_forever()
if (keepRunning == True):
self.httpd.handle_request()
else:
self._stop.set()
def Run(self):
self.start()
def timeHandler(signum, frame):
print('Times up', signum)
sys.exit()
def timeChkr( threadName, dur, t1, t2):
print (origTime)
print ('Begin Timer thread')
while True:
nowTime = int(time.time())
print (nowTime)
delta = nowTime - origTime
if (delta >= dur):
keepRunning = False
t1.stop()
t2.stop()
else:
time.sleep(1)
def main():
#signal.signal(signal.SIGALRM, timeHandler)
#signal.alarm(DUR)
origTime = int(time.time())
ZhttpThread1 = threadWithTO("thread1", I, PORT, DUR)
ZhttpThread2 = threadWithTO("thread2", I, (int(PORT)+1), DUR)
t = threading.Thread(target=timeChkr, args = ("thread3",DUR))
t.daemon = True
t.start()
#ZhttpThread1.daemon = True
#ZhttpThread2.daemon = True
ZhttpThread1.Run()
ZhttpThread2.Run()
ok i figured out the issue is with socket. My socket is open and even though I have tried socket.settimeout I still cant get the socket to close
Thanks to Andrew.. my son whom sparked the idea in my head... here is the solution..
class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def do_GET(self):
logging.warning("======= GET STARTED =======")
logging.warning(self.headers)
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
def do_POST(self):
logging.warning("======= POST STARTED =======")
logging.warning(self.headers)
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD':'POST',
'CONTENT_TYPE':self.headers['Content-Type'],
})
logging.warning("======= POST VALUES =======")
print form.list
'''
for item in form.list:
logging.warning(item) '''
logging.warning("\n")
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
class StoppableHTTPServer(BaseHTTPServer.HTTPServer):
def server_bind(self):
BaseHTTPServer.HTTPServer.server_bind(self)
self.socket.settimeout(1)
self.run = True
def get_request(self):
while self.run:
try:
sock, addr = self.socket.accept()
sock.settimeout(None)
return (sock, addr)
except socket.timeout:
pass
def stop(self):
self.run = False
def serve(self):
while self.run:
#print "1"
self.handle_request()
if __name__=="__main__":
if len(sys.argv) < 1:
I = ""
PORT = 8000
DUR = 10
else:
I = sys.argv[1]
PORT = int(sys.argv[2])
DUR = int(sys.argv[3])
#httpd1 = StoppableHTTPServer((I,PORT), SimpleHTTPServer.SimpleHTTPRequestHandler)
#httpd2 = StoppableHTTPServer((I,(int(PORT)+1)), SimpleHTTPServer.SimpleHTTPRequestHandler)
httpd1 = StoppableHTTPServer((I,PORT), ServerHandler)
httpd2 = StoppableHTTPServer((I,(int(PORT)+1)), ServerHandler)
thread.start_new_thread(httpd1.serve, ())
thread.start_new_thread(httpd2.serve, ())
#raw_input("Press <RETURN> to stop server\n")`enter code here`
print '0'
time.sleep(DUR)
print 'Times up Dude'
httpd1.stop()
httpd2.stop()

Resources