Can't get pyqtgraph chart to update - python-3.x

I'm trying to import tick data from MT5 and display it on a candlestick chart in pyqtgraph but the graph only displays the first two candles which are preloaded to meet minimum data requirements to prevent an exception.
Beyond the original two candles the chart does not update with new values.
import pyqtgraph as pg
import numpy as np
from PyQt5 import QtWidgets, QtCore, QtGui
from pyqtgraph import PlotWidget, plot, QtCore, QtGui
import sys
import os
from random import randint
import time
import threading
import os
import queue
import random
import copy
import MetaTrader5 as mt5
from datetime import datetime
#------------------------------------------------------------------------------
'''chart items'''
class CandlestickItem(pg.GraphicsObject):
_boundingRect = QtCore.QRectF()
# ...
def __init__(self):
pg.GraphicsObject.__init__(self)
self.flagHasData = False
def set_data(self, data):
self.data = data
self.flagHasData = True
self.generatePicture()
self.informViewBoundsChanged()
def generatePicture(self):
self.picture = QtGui.QPicture()
path = QtGui.QPainterPath()
p = QtGui.QPainter(self.picture)
p.setPen(pg.mkPen('w'))
w = (self.data[1][0] - self.data[0][0]) / 3.
for (t, open, close) in self.data:
# line = QtCore.QLineF(t, min, t, max)
# path.moveTo(line.p1())
# path.lineTo(line.p2())
# p.drawLine(line)
rect = QtCore.QRectF(t-w, open, w*2, close-open)
path.addRect(rect)
if open > close:
p.setBrush(pg.mkBrush('r'))
else:
p.setBrush(pg.mkBrush('g'))
p.drawRect(rect)
p.end()
self._boundingRect = path.boundingRect()
def paint(self, p, *args):
if self.flagHasData:
p.drawPicture(0, 0, self.picture)
def boundingRect(self):
return self._boundingRect
#------------------------------------------------------------------------------
# establish connection to the MetaTrader 5 terminal
if not mt5.initialize():
print("initialize() failed, error code =",mt5.last_error())
quit()
# attempt to enable the display of the GBPUSD in MarketWatch
selected=mt5.symbol_select("EURUSD",True)
if not selected:
print("Failed to select EURUSD")
mt5.shutdown()
quit()
#------------------------------------------------------------------------------
class tick:
last_tick = 0
current_tick = 0
current_tick_number = 1
def __init__(self):
self.tick_array = np.zeros((100000,3), dtype = float)
self.pass_data = False
def _time(self, tick_index_number):
return self.tick_array[self.tick_index(tick_index_number),0]
def _open(self, tick_index_number):
# print(tick_index_number)
return self.tick_array[self.tick_index(tick_index_number),1]
def _close(self, tick_index_number):
return self.tick_array[self.tick_index(tick_index_number),2]
def _min(self, tick_index_number):
return self.tick_array[self.tick_index(tick_index_number),3]
def _max(self, tick_index_number):
return self.tick_array[self.tick_index(tick_index_number),4]
#return a negative index of n
def tick_index(self, n = 0):
return self.current_tick_number - n
#gets EURUSD current tick values
def get_tick(self):
return mt5.symbol_info_tick("EURUSD")
#add self.time/bid/ask to get_tick instead
#------------------------------------------------------------------------------
#updates tick array
def update_tick(self):
while True:
#get current tick value
current_tick = self.get_tick()
#if current tick is unique, add that value to last_tick and continue
if self.last_tick != current_tick:
self.last_tick = current_tick
#update the array with the new tick values
self.tick_array[self.current_tick_number,0], self.tick_array[self.current_tick_number,1], self.tick_array[self.current_tick_number,2] = datetime.fromtimestamp(current_tick[5] / 1000.0).strftime("%m%d%Y%I%M%S"), self.tick_array[self.current_tick_number-1, 2] , current_tick[1]
self.current_tick_number += 1
q.put(self.tick_array[:self.current_tick_number])
def tick_datafeed(self):
return self.tick_array[:self.current_tick_number]
tick = tick()
#------------------------------------------------------------------------------
''' launch threads that work the chart'''
class Threads:
def __init__(self):
pass
def thread_launch_update_tick(self):
t1 = threading.Thread(target = tick.update_tick, args = ())
t1.start()
def thread_launch_get_data_from_update(self):
t2 = threading.Thread(target = get_data_from_update, args = ())
t2.start()
def get_data_from_update():
while True:
i = q.get()
item.set_data(i)
print(tick.tick_array[:tick.current_tick_number])
q.task_done()
#------------------------------------------------------------------------------
app = QtWidgets.QApplication([])
item = CandlestickItem()
item.set_data(tick.tick_array[:tick.current_tick_number + 1])
plt = pg.plot()
plt.addItem(item)
plt.setWindowTitle('pyqtgraph example: customGraphicsItem')
q = queue.Queue()
threads = Threads()
threads.thread_launch_get_data_from_update()
threads.thread_launch_update_tick()
#------------------------------------------------------------------------------
if __name__ == '__main__':
# window = Window()
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtWidgets.QApplication.instance().exec_()
The resulting array data looks like this
which is comprised of the [date in unix, last tick, current tick] aka time/open/close
[[0.00000000e+00 0.00000000e+00 0.00000000e+00]
[7.26202211e+12 0.00000000e+00 1.01348000e+00]
[7.26202211e+12 1.01348000e+00 1.01349000e+00]
[7.26202211e+12 1.01349000e+00 1.01348000e+00]
[7.26202211e+12 1.01348000e+00 1.01347000e+00]
[7.26202211e+12 1.01347000e+00 1.01348000e+00]
[7.26202211e+12 1.01348000e+00 1.01347000e+00]
[7.26202211e+12 1.01347000e+00 1.01348000e+00]
[7.26202211e+12 1.01348000e+00 1.01347000e+00]
[7.26202211e+12 1.01347000e+00 1.01346000e+00]
[7.26202211e+12 1.01346000e+00 1.01347000e+00]
[7.26202211e+12 1.01347000e+00 1.01346000e+00]]
Can anyone point out my flaw?
The get_data_from_update() function should update the chart but it simply doesn't work for reasons that are beyond me.

Related

Open multiple windows per time interval, using same long function, receiving different data with Python and PySimpleGui

I'm making a system that receives a list of login data (username and password), then opens each user and password in a window, consuming the same function that I receive as user and password parameters.
Below is a reduced example of what I'm doing.
from multiprocessing.dummy import Process
import PySimpleGUI as sg
from time import sleep
import threading
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import multiprocessing as mp
from multiprocessing import Process
import os, json
from datetime import datetime, timezone
import time
import random
HOME = './Dados'
def current_location(self):
geometry = self.TKroot.geometry()
location = geometry[geometry.find('+') + 1:].split('+')
x, y = int(location[0]), int(location[1])
return(x, y+-30)
sg.Window.current_location = current_location
def function_thread(usuario, senha):
global window
window['output'].print(f'Login {usuario} e {senha}')
for i in range(5):
#print(f'Tarefas no perfil {usuario}')
#window['output'].update(str('Tarefas no perfil {usuario}'))
time.sleep(random.randint(1,2))
horaatual = datetime.today().strftime('[%H:%M:%S:%f] - ')
window['output'].print(horaatual+': Tarefas: '+ str(i))
#sleep(2)
#window.write_event_value('output', i)
#window.write_event_value('output', usuario)
window['output'].print(f'Completado {usuario}')
#sleep(1)
x=-7
y=515
def abrir_abas():
global x,y, usuario_insta, senha_insta
window2 = None
cont = 0
#list_user = [('perfil1','senha1'),('perfil2', 'senha2'),('perfil3', 'senha3'),('perfil4', 'senha4'),('perfil5','senha5'),('perfil6', 'senha6'),('perfil7', 'senha7'),('perfil8', 'senha8')]
'''
executor = concurrent.futures.ProcessPoolExecutor(8)
futures = [executor.submit(function_thread(group[0],group[1])for group in list_user)]
concurrent.futures.wait(futures)
'''
with open(os.path.join(HOME, f'Config/abas.txt'), 'r') as f:
abas = f.readlines()
lista_conta = []
for index_lista in range(len(abas)):
abas[index_lista] = abas[index_lista].rstrip('\n')
if abas[index_lista] != '' or None:
lista_conta.append(abas[index_lista])
#with concurrent.futures.ProcessPoolExecutor() as executor:
for i_lista_conta in range(len(lista_conta)):
qtd_lista_conta = int(len(lista_conta)/2)
if i_lista_conta % 2 == 0:
usuario = lista_conta[i_lista_conta].split(' ')
usuario_insta = usuario[0]
senha_insta = usuario[1]
#self.output(usuario_insta, senha_insta)
titulo = 'TAB '+usuario_insta
window2 = output(usuario_insta, senha_insta, loc=(x,y))
window2.TKroot.title(titulo)
t1 = threading.Thread(target=function_thread(usuario_insta, senha_insta), daemon=True)
t1.start()
y=y+-30
y=y
cont = cont + 1
'''
processos = []
for i_lista_conta in range(len(lista_conta)):
qtd_lista_conta = int(len(lista_conta)/2)
if i_lista_conta % 2 == 0:
usuario_insta = lista_conta[i_lista_conta]
senha_insta = lista_conta[i_lista_conta+1]
#self.output(usuario_insta, senha_insta)
titulo = 'TAB '+usuario_insta
window2 = output(loc=(x,y))
window2.TKroot.title(titulo)
#threading.Thread(target=self.thread_acao(usuario_insta, senha_insta), daemon=True).start()
p = Process(target=function_thread(usuario_insta, senha_insta))
p.start()
processos.append(p)
y=y+-30
y=y
cont = cont + 1
'''
#for process in processos:
# process.join()
'''
for i in list_user:
titulo = 'TAB '+i[0]
#with concurrent.futures.ProcessPoolExecutor() as executor:
# window2 = executor.map(output(loc=(x,y)))
window2 = output(loc=(x,y))
window2.TKroot.title(titulo)
#with concurrent.futures.ProcessPoolExecutor() as executor:
# executor.map(function_thread(i[0],i[1]))
#threading.Thread(target=function_thread(i[0],i[1]), daemon=True).start()
Process(target=function_thread(i[0],i[1])).start()
y=y+-30
y=y
cont = cont + 1
sleep(2)
'''
def output(login, senha, loc=(-7, 515))->sg.Window:
global window
voltar = 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsTAAALEwEAmpwYAAAF8WlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDggNzkuMTY0MDM2LCAyMDE5LzA4LzEzLTAxOjA2OjU3ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHhtcDpDcmVhdGVEYXRlPSIyMDIwLTAzLTAzVDA5OjUwOjM0LTAzOjAwIiB4bXA6TW9kaWZ5RGF0ZT0iMjAyMC0wNS0wMlQxNzo1Nzo0Mi0wMzowMCIgeG1wOk1ldGFkYXRhRGF0ZT0iMjAyMC0wNS0wMlQxNzo1Nzo0Mi0wMzowMCIgZGM6Zm9ybWF0PSJpbWFnZS9wbmciIHBob3Rvc2hvcDpDb2xvck1vZGU9IjMiIHBob3Rvc2hvcDpJQ0NQcm9maWxlPSJzUkdCIElFQzYxOTY2LTIuMSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo0MGZlY2VlMS0yYmE4LTI4NGQtYTk4NC0zYjQxNTIxNzA1ZGQiIHhtcE1NOkRvY3VtZW50SUQ9ImFkb2JlOmRvY2lkOnBob3Rvc2hvcDo2MmQ1MzYwYi0zMTQ5LTI2NDAtYWZhMC03YzdiNjkzNDFkY2YiIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDowZDY2MTNmZi0wMGIzLWMzNDQtYTgzOS1jMTUwZTc4ZGZlODEiPiA8eG1wTU06SGlzdG9yeT4gPHJkZjpTZXE+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJjcmVhdGVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjBkNjYxM2ZmLTAwYjMtYzM0NC1hODM5LWMxNTBlNzhkZmU4MSIgc3RFdnQ6d2hlbj0iMjAyMC0wMy0wM1QwOTo1MDozNC0wMzowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDIxLjAgKFdpbmRvd3MpIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo0MGZlY2VlMS0yYmE4LTI4NGQtYTk4NC0zYjQxNTIxNzA1ZGQiIHN0RXZ0OndoZW49IjIwMjAtMDUtMDJUMTc6NTc6NDItMDM6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyMS4wIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8L3JkZjpTZXE+IDwveG1wTU06SGlzdG9yeT4gPC9yZGY6RGVzY3JpcHRpb24+IDwvcmRmOlJERj4gPC94OnhtcG1ldGE+IDw/eHBhY2tldCBlbmQ9InIiPz4M/LCOAAABD0lEQVQ4y43SMSvFURjH8XPvVchCScpwB2VRykIpG7u7KLeMZGGQd6BMXoDZwAtQUgaTkZKNRalPSgYZrAbn1On0v/f+Tz2d0/P0fM/z/J4nIBTWyN5rOMU5zrCD4RhrIvRKHsEFXmLiIU5wjycsJUioqCDgCtcYrYgd+D/zZQWteO/FX3J/I5acKjzGTVULAXfYiu+hHi2O4w0LZWAy/j5XIWhpl+iUgGk8YLYGYAJjVYDHmoCQNGhmApaAVpp3P0BuJSAMqiRgFSvRMRM1SCIuYr0fJKCDTyzHNX3GFNr4wG6+ur1a2MY7urjFBl5x1C85FzFBvvGFn7iyoa6IaQqb+MV+4a81hSRSe1BSbn+7uNbAq7k3vAAAAABJRU5ErkJggg=='
pause = 'iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAACXBIWXMAABYlAAAWJQFJUiTwAAAGvmlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQiPz4gPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iQWRvYmUgWE1QIENvcmUgNS42LWMxNDggNzkuMTY0MDM2LCAyMDE5LzA4LzEzLTAxOjA2OjU3ICAgICAgICAiPiA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPiA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIgeG1sbnM6cGhvdG9zaG9wPSJodHRwOi8vbnMuYWRvYmUuY29tL3Bob3Rvc2hvcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHhtcDpDcmVhdGVEYXRlPSIyMDIwLTA1LTAyVDE3OjQ4OjI0LTAzOjAwIiB4bXA6TW9kaWZ5RGF0ZT0iMjAyMC0wNS0wM1QxMTowNjo0NC0wMzowMCIgeG1wOk1ldGFkYXRhRGF0ZT0iMjAyMC0wNS0wM1QxMTowNjo0NC0wMzowMCIgZGM6Zm9ybWF0PSJpbWFnZS9wbmciIHBob3Rvc2hvcDpDb2xvck1vZGU9IjMiIHBob3Rvc2hvcDpJQ0NQcm9maWxlPSJzUkdCIElFQzYxOTY2LTIuMSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo0ODFlYzZjZS1jYTNhLWIxNDgtYjlmYy0yZmZmYjVhNzI3ZTEiIHhtcE1NOkRvY3VtZW50SUQ9ImFkb2JlOmRvY2lkOnBob3Rvc2hvcDo1NmE4ZDEzNC0wNmIzLWVkNGMtOWYwYi1kMjk4YmU1YTlmYzAiIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDo0ZmQ2MzhkYy05YzYyLTFjNGUtYjg2Ny03ZDNmOTdlMmE5YTgiPiA8eG1wTU06SGlzdG9yeT4gPHJkZjpTZXE+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJjcmVhdGVkIiBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOjRmZDYzOGRjLTljNjItMWM0ZS1iODY3LTdkM2Y5N2UyYTlhOCIgc3RFdnQ6d2hlbj0iMjAyMC0wNS0wMlQxNzo0ODoyNC0wMzowMCIgc3RFdnQ6c29mdHdhcmVBZ2VudD0iQWRvYmUgUGhvdG9zaG9wIDIxLjAgKFdpbmRvd3MpIi8+IDxyZGY6bGkgc3RFdnQ6YWN0aW9uPSJzYXZlZCIgc3RFdnQ6aW5zdGFuY2VJRD0ieG1wLmlpZDo0N2MwYWRlMy02NmVjLWVjNDEtYjE1YS0zNmI2ZmZmOGEzNWYiIHN0RXZ0OndoZW49IjIwMjAtMDUtMDJUMTc6NTI6MTYtMDM6MDAiIHN0RXZ0OnNvZnR3YXJlQWdlbnQ9IkFkb2JlIFBob3Rvc2hvcCAyMS4wIChXaW5kb3dzKSIgc3RFdnQ6Y2hhbmdlZD0iLyIvPiA8cmRmOmxpIHN0RXZ0OmFjdGlvbj0ic2F2ZWQiIHN0RXZ0Omluc3RhbmNlSUQ9InhtcC5paWQ6NDgxZWM2Y2UtY2EzYS1iMTQ4LWI5ZmMtMmZmZmI1YTcyN2UxIiBzdEV2dDp3aGVuPSIyMDIwLTA1LTAzVDExOjA2OjQ0LTAzOjAwIiBzdEV2dDpzb2Z0d2FyZUFnZW50PSJBZG9iZSBQaG90b3Nob3AgMjEuMCAoV2luZG93cykiIHN0RXZ0OmNoYW5nZWQ9Ii8iLz4gPC9yZGY6U2VxPiA8L3htcE1NOkhpc3Rvcnk+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+sKbvRwAAAWBJREFUOI3t1L2KFFEQBeCvZ8afBZENFmEEBU0MxGQfQBE00jfYh9DUFxBBQzMx8A00MlVMDQVRFARxXDN1BWfH6W2DPpe5jLA7xnqS+1NV51Rfuk4zmUwE6ziR/R5G+IFJznJ3CkfwK+c5PmE6Ho+NLHAW59CF4BC+YgffkrOBTRzFLDltYu+L4nrInuNYFGuhO7iV4ge4XuW0GOItLmF7hDEuhGwLr1I8x31cDvEaLuIR7uWzZxG4jdOFUJQ6vMDHqrsPOJN9F4I3ES04mXUOg7zXHpq8UY3jJTGEbe5qbFRxA/ujOyD+R85BhH+N/4T/ImGzQs2+/+ZAP9xlBD8vxXdwuBIrDlSj1AxZOEbxuy29cwxzv2lhXcXSruC13sKmuFZ3PtKbQasf+LtL6jPcyH6Kh7iJq0t5j/GuEP7EFzzBd+ymmzVs42WKWjzD+XS3m44bPM3zaLpulflfHb8BEgRaa6It0wUAAAAASUVORK5CYII='
col_v = sg.Column(
[
[sg.Button(image_data=pause, size=(None,20), key='Executar',button_color=(sg.theme_background_color(), sg.theme_background_color()), border_width=0)],
[sg.Text('')],
[sg.Button(image_data=voltar, key='voltar',tooltip="Voltar",button_color=(sg.theme_background_color(), sg.theme_background_color()), border_width=0)],
], k='c_v', visible=True)
main_layout =[
#[sg.Titlebar(title = texto, key = 'titulo')],
[sg.StatusBar('O Rei Dos BOTs',justification='center', font='Arial 12')],
[col_v,sg.Multiline(size=(20,2),expand_x=True, expand_y=True, key='output')],
[sg.StatusBar('INICIANDO', k='status+', auto_size_text=True, justification='center', font=('Arial', 12, 'bold'), text_color='#FFFFFF', background_color=None, s=(None, 1))],
]
window = sg.Window('ABA', size=(253,181), finalize=True, location=loc, background_color=None, auto_size_text=True, font='Arial 7', resizable=True, auto_size_buttons=False,right_click_menu = False, margins=(1, 1)).Layout(main_layout).Finalize()
#with concurrent.futures.ProcessPoolExecutor() as executor:
# executor.map(function_thread(login, senha))
return window
def main_output():
global window, usuario_insta, senha_insta
sg.theme('Topanga')
layout = [ [sg.Text('Janela')],
[sg.Text(size=(10,1), key='-OUT-')],
[sg.Button('Executar'), sg.Button('Sair')],
[sg.StatusBar('INICIANDO', k='status', auto_size_text=True, justification='center', font='Arial 12', text_color='#FFFFFF', background_color=None, s=(None, 1))],
]
window = sg.Window('Janela principal', layout, size=(253,181), use_default_focus=False,finalize=True, use_custom_titlebar=False, no_titlebar=False)
while True:
event, values = window.read()
if event == sg.WIN_CLOSED or event == 'Sair':
break
if event == 'Executar':
window.hide()
#with ThreadPoolExecutor() as executor:
# executor.map(abrir_abas())
#threading.Thread(target=abrir_abas(), daemon=True).start()
abrir_abas()
#threading.Thread(target=abrir_abas(), args=(window,), daemon=False).start()
#with concurrent.futures.ProcessPoolExecutor() as executor:
# executor.map(abrir_abas())
if event == 'Ok':
print('Completado')
if event == 'voltar':
print('Voltar')
window.close()
#threading.Thread(target=main_output(), args=(window,), daemon=True).start()
main_output()
I can only do this by opening the first window and waiting for the function to complete and then opening the second window and so on.
The function is looping and can take more than 24 hours.

Have a pytorch class produce 2 separate datasets

Does anyone have any suggestions for creating 2 datasets at once? When I run dataset = ImbCircuitDataset('test'), I want it to save 2 separate sets, even if I have to go all the way back to self.data and split it there, I just want the class to produce 2 separate datasets.
Basically run set1, set2 = ImbCircuitDataset('test')
I started doing it at the bottom of the def process(self) but I have a feeling its wrong. I am thinking in def processed_file_names(self) I need to return 'set1.pt', 'set2.pt'.
from scipy.io import loadmat
import pandas as pd
from torch_geometric.data import Data, InMemoryDataset
import torch
import neptune.new as neptune
import numpy as np
import torch_geometric
from tqdm import tqdm
from typing import Tuple, Union
from torch import Tensor
from collections.abc import Sequence
from torch_geometric.utils import from_networkx
import networkx as nx
import matplotlib.pyplot as plt
IndexType = Union[slice, Tensor, np.ndarray, Sequence]
print(f"Torch version: {torch.__version__}")
print(f"Cuda available: {torch.cuda.is_available()}")
print(f"Torch geometric version: {torch_geometric.__version__}")
class ImbCircuitDataset(InMemoryDataset):
def __init__(self, root, transform=None, pre_transform=None,
pre_filter=None):
super().__init__(root, transform, pre_transform, pre_filter)
self.data, self.slices = torch.load(self.processed_paths[0])
#property
def raw_file_names(self):
return 'shuffled_data.mat'
#property
def processed_file_names(self):
return 'data.pt'
def download(self):
pass
def process(self):
self.raw_data = loadmat(self.raw_paths[0], squeeze_me=True)
self.data = pd.DataFrame(self.raw_data['Graphs'])
self.data = self.data.sort_values('Labels', ascending=True, ignore_index=True)
for i in range(len(self.data)):
self.data['Ln'][i] = dict(enumerate(self.data['Ln'][i]))
data_list = []
for index, cir in tqdm(self.data.iterrows(), total=self.data.shape[0]):
nxg = nx.Graph(self.data['A'][index])
nx.set_node_attributes(nxg, self.data['Ln'][index], 'component')
pt_graph = self._get_graph_object(nxg)
pt_graph.x = self._get_node_features(nxg, self.data['Ln'][index])
pt_graph.performance = torch.tensor(self.data['Labels'][index], dtype=torch.float)
data_list.append(pt_graph)
if self.pre_filter is not None:
data_list = [d for d in data_list if self.pre_filter(d)]
if self.pre_transform is not None:
data_list = [self.pre_transform(d) for d in data_list]
split = 0.5
N = len(data_list)
set_1 = data_list[:N*split]
set_2 = data_list[N*split:]
data1, slices1 = self.collate(set_1)
data2, slices2 = self.collate(set_2)
torch.save((data1, slices1), self.processed_paths[0])
torch.save((data2,slices2), self.processed_paths[0])
def _get_node_features(self, nxgraph, node_labels):
betweenness = list(nx.betweenness_centrality(nxgraph).values())
eigenvector = list(nx.eigenvector_centrality(
nxgraph, max_iter=600).values())
mapping_dict = {'C': 0, 'G': 1, 'I': 2, 'O': 3, 'P': 4, 'R': 5}
component_labels = []
for value in node_labels.values():
if value in mapping_dict:
component_labels.append(mapping_dict[value])
all_feats = [component_labels, betweenness, eigenvector]
all_feats = np.asarray(all_feats).transpose()
return torch.tensor(all_feats, dtype=torch.float)
def _get_graph_object(self, nx_graph):
nxg = from_networkx(nx_graph)
return nxg
#property
def num_node_features(self) -> int:
return 3
#property
def num_classes(self) -> int:
return 2

Script Multiprocessing dont finish all task, and also i get 100 cpu?

i need to ask if part of my script is correct, working "i think fine" but i think really i have somethink wrong, because still i get CPU 100% and so many time dont finish all task but after 50/100 task is like frozen.
Any info how to edit it ? Or Maybe just tell me where is the error ?
Thank you
Ps. I have inserted all the modules that the script requires and only the part that should be of interest for multiprocessing and also just firt part of the script.
Many Thanks
from __future__ import print_function
import sys
import os
import easygui
import pyautogui as py
import datetime
import pwinput
import json
from collections import Counter
import random
import string
import threading
import subprocess
import multiprocessing
import queue
from multiprocessing import cpu_count
from multiprocessing import Value, Lock, Process, Queue, current_process
import numpy as np
import grequests
import requests
from requests.exceptions import ConnectionError
from requests.exceptions import HTTPError
import time
from time import sleep
number_of_processes = cpu_count()
class Counter(object):
def __init__(self, initval=0):
self.val = Value('i', initval)
self.lock = Lock()
def increment(self):
with self.lock:
self.val.value += 1
def value(self):
with self.lock:
return self.val.value
def updateTitle(number_of_processes,number_of_task,counterhits,counterdone,countersl,countml,username):
while True:
hits = int(counterhits.value())
done = int(counterdone.value())
shtot = int(countersl.value())
maitot = int(countml.value())
remain_scan = number_of_task - hits
elapsed = time.strftime('%H:%M:%S', time.gmtime(time.time() - start))
ctypes.windll.kernel32.SetConsoleTitleW(f'Site Valid For: {number_of_task} | Started: {hits} | Complete: {done} | Remain: {remain_scan} | SL Found: {shtot} | ML Found: {maitot} | Threads: {number_of_processes} | Time elapsed: {elapsed} ! Licensed at: {username}')
sleep(0.3)
def worker_main(tasks_to_do,tasks_finished,counterhits,counterdone,countersl,countml):
while True:
try:
site = tasks_to_do.get_nowait()
if site is None:
break
except Queue.Empty:
break
except Queue.Full:
sleep(0.5)
continue
counterhits.increment()
do_work(site,counterhits,counterdone,countersl,countml)
tasks_finished.put(site + current_process().name)
counterdone.increment()
return True
def main():
global username
number_of_task = int(len(filter_data))
counterhits = Counter(0)
counterdone = Counter(0)
countersl = Counter(0)
countml = Counter(0)
tasks_to_do = Queue()
tasks_finished = Queue()
processes1 = []
prefix = ['http://']
# creating processes
for w in range(number_of_processes):
p1 = Process(target=worker_main, args=(tasks_to_do,tasks_finished,counterhits,counterdone,countersl,countml))
processes1.append(p1)
p1.start()
procs = [Process(target=updateTitle, args=(number_of_processes,number_of_task,counterhits,counterdone,countersl,countml,username), daemon=True) for i in range(1)]
for p in procs: p.start()
for site_il in filter_data:
site_or = site_il.rstrip("\n")
if (site_or.startswith("http://")) :
site_or = site_or.replace("http://","")
elif (site_or.startswith("https://")) :
site_or = site_or.replace("https://","")
site_or = site_or.rstrip()
site_or = site_or.split('/')[0]
if ('www.' in site_or) :
site_or = site_or.replace("www.", "")
sitexx = [sub + site_or for sub in prefix]
for site in sitexx:
tasks_to_do.put(site)
# completing process
for p1 in processes1:
p1.join()
for p in procs: p.join()
# print the output
while not tasks_finished.empty():
print(tasks_finished.get())
os.system('pause>nul')
return True
if __name__ == '__main__':
if sys.platform.startswith('win'):
# On Windows calling this function is necessary.
multiprocessing.freeze_support()
main()

multiprocessing TypeError: can't pickle _thread.lock objects

when i run it on python3 ,i get error:TypeError: can't pickle _thread.lock objects
example:
import time
import multiprocessing
from multiprocessing import Process,freeze_support,Manager,Queue,Pool
q=Queue()
pool=Pool(3)
class A(object):
def init(self,varam):
self.varam = varam
class class_one(object):
def __init__(self):
self.var = {}
self.list = []
self.get()
def get_result(self, varam):
q.put(A(varam))
def get(self):
for i in [6,7,8,9]:
self.list.append(pool.apply_async(self.get_result, (i, )))
#p = Process(target=self.get_result, args=(i, ))
#process_list.append(p)
#p.start()
pool.close()
pool.join()
if __name__ == '__main__':
freeze_support()
process_num =2
process_list = []
g = class_one()
for i in g.list:
print(i.get())
# for j in process_list:
# j.join()
# print(q.qsize())
# if q.empty():
# print("error")
# while (not q.empty()):
# print(q.get().varam)
# g.var[q.get().varam]=q.get()

big raw video file seeking in python gstreamer

I'm working on program, that should display big raw video, seek in it that I'd be able to set from and to times cuts in it, set black borders sizes to hide shaked borders of image. The crucial part of this project is seeking. I've tried 5min file cutoff and when I seek at the start of the video, it's OK but after middle something goes wrong. Since it seems there is no much examples and documentation I'm using this:
self.pipe.seek_simple(
Gst.Format.TIME,
Gst.SeekFlags.FLUSH | Gst.SeekFlags.KEY_UNIT,
destSeek
)
My actual (non-mature) code:
import sys
import gi
gi.require_version('Gst', '1.0')
gi.require_version('Gtk', '3.0')
gi.require_version('GdkX11', '3.0')
gi.require_version('GstVideo', '1.0')
from gi.repository import GObject, Gst, Gtk, GdkX11, GstVideo, Gdk
GObject.threads_init()
Gst.init(None)
testGrab = "testRAW.mkv"
class VideoDec(Gst.Bin):
def __init__(self):
super().__init__()
# elements
q1 = Gst.ElementFactory.make('queue', None)
videoparse = Gst.ElementFactory.make('videoparse', None)
q2 = Gst.ElementFactory.make('queue', None)
self.add(q1)
self.add(videoparse)
self.add(q2)
videoparse.set_property('width', 720)
videoparse.set_property('height', 576)
videoparse.set_property('format', 4)
# link
q1.link(videoparse)
videoparse.link(q2)
# Add Ghost Pads
self.add_pad(
Gst.GhostPad.new('sink', q1.get_static_pad('sink'))
)
self.add_pad(
Gst.GhostPad.new('src', q2.get_static_pad('src'))
)
class AudioDec(Gst.Bin):
def __init__(self):
super().__init__()
# elements
q1 = Gst.ElementFactory.make('queue', None)
audioparse = Gst.ElementFactory.make('audioparse', None)
q2 = Gst.ElementFactory.make('queue', None)
#sink = Gst.ElementFactory.make('autoaudiosink', None)
self.add(q1)
self.add(audioparse)
self.add(q2)
#self.add(sink)
# link
q1.link(audioparse)
audioparse.link(q2)
#audioparse.link(sink)
# Add Ghost Pads
self.add_pad(
Gst.GhostPad.new('sink', q1.get_static_pad('sink'))
)
self.add_pad(
Gst.GhostPad.new('src', q2.get_static_pad('src'))
)
class Player(object):
def __init__(self):
self.fps = 25
self.window = Gtk.Window()
self.window.connect('destroy', self.quit)
self.window.set_default_size(800, 600)
self.drawingarea = Gtk.DrawingArea()
#hbox
self.hbox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
self.window.add(self.hbox)
Gtk.Box.pack_start(self.hbox, self.drawingarea, True, True, 0)
self.setPipeline()
self.setGUI()
self.setShortcuts()
self.playing = False
def setPipeline(self):
self.pipe = Gst.Pipeline.new('player')
# Create bus to get events from GStreamer pipeline
self.bus = self.pipe.get_bus()
self.bus.add_signal_watch()
self.bus.connect('message::eos', self.on_eos)
self.bus.connect('message::error', self.on_error)
# This is needed to make the video output in our DrawingArea:
self.bus.enable_sync_message_emission()
self.bus.connect('sync-message::element', self.on_sync_message)
self.src = Gst.ElementFactory.make('filesrc', None)
self.src.set_property("location", testGrab)
self.dec = Gst.ElementFactory.make('decodebin', None)
self.video = VideoDec()
self.audio = AudioDec()
self.glimagesink = Gst.ElementFactory.make('glimagesink', None)
self.audiosink = Gst.ElementFactory.make('autoaudiosink', None)
self.pipe.add(self.src)
self.pipe.add(self.dec)
self.pipe.add(self.video)
self.pipe.add(self.audio)
self.pipe.add(self.glimagesink)
self.pipe.add(self.audiosink)
#self.pipe.add(self.autovideosink)
# Connect signal handlers
self.dec.connect('pad-added', self.on_pad_added)
# link
self.src.link(self.dec)
self.video.link(self.glimagesink)
self.audio.link(self.audiosink)
def on_pad_added(self, element, pad):
string = pad.query_caps(None).to_string()
print('on_pad_added():', string)
if string.startswith('audio/'):
pad.link(self.audio.get_static_pad('sink'))
elif string.startswith('video/'):
pad.link(self.video.get_static_pad('sink'))
def setGUI(self):
vbox = Gtk.Box(Gtk.Orientation.HORIZONTAL, 0)
vbox.set_margin_top(3)
vbox.set_margin_bottom(3)
Gtk.Box.pack_start(self.hbox, vbox, False, False, 0)
self.playButtonImage = Gtk.Image()
self.playButtonImage.set_from_stock("gtk-media-play", Gtk.IconSize.BUTTON)
self.playButton = Gtk.Button.new()
self.playButton.add(self.playButtonImage)
self.playButton.connect("clicked", self.playToggled)
Gtk.Box.pack_start(vbox, self.playButton, False, False, 0)
self.slider = Gtk.HScale()
self.slider.set_margin_left(6)
self.slider.set_margin_right(6)
self.slider.set_draw_value(False)
self.slider.set_range(0, 100)
self.slider.set_increments(1, 10)
Gtk.Box.pack_start(vbox, self.slider, True, True, 0)
self.label = Gtk.Label(label='0:00')
self.label.set_margin_left(6)
self.label.set_margin_right(6)
Gtk.Box.pack_start(vbox, self.label, False, False, 0)
def setShortcuts(self):
accel = Gtk.AccelGroup()
accel.connect(Gdk.KEY_space, Gdk.ModifierType.CONTROL_MASK, 0, self.playToggled)
accel.connect(Gdk.KEY_Right, Gdk.ModifierType.CONTROL_MASK, 0, self.seekFW0)
accel.connect(Gdk.KEY_Right, Gdk.ModifierType.CONTROL_MASK|Gdk.ModifierType.SHIFT_MASK, 0, self.seekFW10s)
accel.connect(Gdk.KEY_Right, Gdk.ModifierType.SHIFT_MASK, 0, self.seekFW2)
accel.connect(Gdk.KEY_Right, Gdk.ModifierType.MOD1_MASK, 0, self.seekFW10) # alt key
self.window.add_accel_group(accel)
def seekFW0(self, *args):
self.seekTime = 2 * Gst.SECOND // self.fps
self.seekFW()
def seekFW10s(self, *args):
self.seekTime = Gst.SECOND * 10
self.seekFW()
def seekFW2(self, *args):
self.seekTime = Gst.SECOND * 60 * 2
self.seekFW()
def seekFW10(self, *args):
self.seekTime = Gst.SECOND * 60 * 10
self.seekFW()
def seekFW(self, *args):
nanosecs = self.pipe.query_position(Gst.Format.TIME)[1]
destSeek = nanosecs + self.seekTime
self.pipe.seek_simple(
Gst.Format.TIME,
Gst.SeekFlags.FLUSH | Gst.SeekFlags.KEY_UNIT,
destSeek
)
def play(self):
self.pipe.set_state(Gst.State.PLAYING)
GObject.timeout_add(1000, self.updateSlider)
def stop(self):
self.pipe.set_state(Gst.State.PAUSED)
def playToggled(self, *w):
if(self.playing == False):
self.play()
else:
self.stop()
self.playing=not(self.playing)
self.updateButtons()
def updateSlider(self):
try:
nanosecs = self.pipe.query_position(Gst.Format.TIME)[1]
duration_nanosecs = self.pipe.query_duration(Gst.Format.TIME)[1]
# block seek handler so we don't seek when we set_value()
# self.slider.handler_block_by_func(self.on_slider_change)
duration = float(duration_nanosecs) / Gst.SECOND
position = float(nanosecs) / Gst.SECOND
self.slider.set_range(0, duration)
self.slider.set_value(position)
self.label.set_text ("%d" % (position / 60) + ":%02d" % (position % 60))
#self.slider.handler_unblock_by_func(self.on_slider_change)
except Exception as e:
# pipeline must not be ready and does not know position
print(e)
pass
return True
def updateButtons(self):
if(self.playing == False):
self.playButtonImage.set_from_stock("gtk-media-play", Gtk.IconSize.BUTTON)
else:
self.playButtonImage.set_from_stock("gtk-media-pause", Gtk.IconSize.BUTTON)
def run(self):
self.window.show_all()
# You need to get the XID after window.show_all(). You shouldn't get it
# in the on_sync_message() handler because threading issues will cause
# segfaults there.
self.xid = self.drawingarea.get_property('window').get_xid()
#self.pipeline.set_state(Gst.State.PLAYING)
Gtk.main()
def quit(self, window):
self.pipe.set_state(Gst.State.NULL)
Gtk.main_quit()
def on_sync_message(self, bus, msg):
if msg.get_structure().get_name() == 'prepare-window-handle':
print('prepare-window-handle')
msg.src.set_window_handle(self.xid)
def on_eos(self, bus, msg):
#print('on_eos(): seeking to start of video')
print('on_eos(): pausing video')
self.stop()
#self.pipeline.seek_simple(
# Gst.Format.TIME,
# Gst.SeekFlags.FLUSH | Gst.SeekFlags.KEY_UNIT,
# 0
#)
#self.playing = False
#self.slider.set_value(0)
#self.label.set_text("0:00")
#self.updateButtons()
def on_error(self, bus, msg):
print('on_error():', msg.parse_error())
p = Player()
p.run()
If somebody could share wisdom or documentation/example links I'd be happy.
I no longer remember quite why I gave up using seek_simple but I do recall that it caused me no end of grief.
Use instead a standard seek:
Gplayer.seek(self.rate, Gst.Format.TIME,
(Gst.SeekFlags.FLUSH | Gst.SeekFlags.ACCURATE),
Gst.SeekType.SET, seek_time , Gst.SeekType.NONE, -1)
Gstreamer1.0 seek
Late Edit:
I believe the issue was getting accurate and consistent timestamps, when I was using the scaletempo element. The scaletempo element caters for adjusting the speed at which playback occurs and I was getting inconsistent times from the player if I varied the rate of play. I moved from seek_simple to seek to resolve the issue.
Note: self.rate above is a user defined variable, which for normal playback would be 1.00, dropping below 1.00 for slower playback and above 1.00 for faster playback.

Resources