I am doing a graphics project and have to use pixel plotting function only. I came across pygame but it doesnt seem to provide its calls to be used in threads. Please suggest me workaround this or any other python library that can be used with threads.
#! /usr/bin/env python
import threading
import time
class plotterthread(threading.Thread):
def __init__(self,screen,queue):
threading.Thread.__init__(self)
self.screen = screen
self.__queue__ = queue
self._colorset = False
self.kill = False
def setcolor(self,r,g,b):
self._r = r
self._g = g
self._b = b
self._colorset = True
def run(self):
try:
while not self.__queue__.empty() and not self.kill:
(x,y) = self.__queue__.get(timeout=1)
if not self._colorset:
#this is where error occures
#this calls pygame.set_at() method
self.screen.plotpixel((x,y))
else:
self.screen.plotpixel((x,y),(self._r,self._g,self._b))
except:
pass
class plotter:
def __init__(self,screen,min,max,queue,ppt=50):
self.screen = screen
self.min = min
self.max = max
self.ppt = ppt # pixels per thread
self.coords = queue
def plot(self):
noOfcoords = self.coords.qsize()
noOfthreads = int(noOfcoords/self.ppt)
if noOfthreads > self.max:
noOfthreads = self.max
elif noOfthreads < self.min:
noOfthreads = self.min
self.threadpool = []
for i in range(noOfthreads):
pthread = plotterthread(screen = self.screen,queue = self.coords)
self.threadpool.append(pthread)
pthread.start()
while True:
try:
livethreads = 0
for t in self.threadpool:
if t is not None and t.isAlive():
livethreads += 1
t.join(timeout=1)
if livethreads == 0:
break
except KeyboardInterrupt:
for threads in self.threadpool:
if threads is not None and threads.isAlive():
threads.kill = True
break
Here, screen is the instance of the window on which I am trying to draw.
queue is the Queue which contains all the calculated coordinates to be plotted.
All I wanted to do was first calculate all the pixels and push them into a queue and then share the queue with threads to plot them in parallel to speed up the process.
Related
I am facing an issue.
So I have a GUI, and because i have to update constantly labels and a polargraph, i used threads. the code works just fine, the labels get updated correctly and so on. the problem comes when i try to update the graph at the same time with the labels. first, i tried to put the plotting code inside the worker, where i got the error "timers cannot be started from another thread", even tho i use no timer at all. after, i tried to put the plotting code inside my main window. there, the variables resulted from the worker function are not seen. meaning that i get the error: "worker has no attribute "ex".
since i am looped between these two errors, i am requesting help. i am attaching the main and worker classes below.
class Worker(QObject):
finished = pyqtSignal()
progress = pyqtSignal(int)
def cart2pol(self,x, y):
rho = np.sqrt(x**2 + y**2)
phi = np.arctan2(y, x)
return(rho, phi)
def pol2cart(self,rho, phi):
x = rho * np.cos(phi)
y = rho * np.sin(phi)
return(x, y)
#functieon
def run(self):
"""Long-running task."""
for i in range(5):
self.progress.emit(i + 1)
self.finished.emit()
ser = serial.Serial('COM1', 19200, timeout=5.0)
sio = io.TextIOWrapper(io.BufferedRWPair(ser, ser))
f1 = open('da.txt')
f2 = open('vn.txt')
while 1:
try:
line = sio.readline()
msg = pynmea2.parse(line)
if line.startswith('$WIMWV'):
first.valdirvant.setText(str(msg.wind_angle))
first.valvantapar.setText(str(msg.wind_speed))
da = f1.readline()
vn = f2.readline()
first.valda.setText(da)
first.valvitnav.setText(vn)
xav = []
yav = []
xbv = []
ybv = []
av = []
bv = []
xa = float(vn)
ya = float(da)
xb = -(float(msg.wind_speed))
yb = -(math.radians(float(msg.wind_angle))) #math.radians
a = xa+xb
b = ya+yb
xav.append(xa)
yav.append(ya)
xbv.append(xb)
ybv.append(yb)
av.append(a)
bv.append(b)
#first.graphWidget.plot(xav, yav)
#first.graphWidget.plot(xbv, ybv)
#first.graphWidget.plot(av, bv)
except serial.SerialException as e:
print('Device error: {}'.format(e))
break
except pynmea2.ParseError as e:
print('Parse error: {}'.format(e))
continue
class MainWindow(QDialog):
def __init__(self):
super(MainWindow, self).__init__()
loadUi("meteo.ui", self)
#butoane
self.on.clicked.connect(self.runLongTask)
#customizaregrafic
self.graphWidget.setBackground('white')
# Add polar grid lines
self.graphWidget.addLine(x=0, pen=0.2)
self.graphWidget.addLine(y=0, pen=0.2)
for r in range(2, 20, 2):
circle = pg.QtGui.QGraphicsEllipseItem(-r, -r, r * 2, r * 2)
circle.setPen(pg.mkPen(0.2))
self.graphWidget.addItem(circle)
self.graphWidget.plot(Worker.xav, Worker.yav)
self.graphWidget.plot(QObject.xbv, QObject.ybv)
self.graphWidget.plot(QObject.av, QObject.bv)
def runLongTask(self):
self.thread = QThread()
self.worker = Worker()
self.worker.moveToThread(self.thread)
self.thread.started.connect(self.worker.run)
self.thread.start()
as i said earlier, i tried putting the plotting code in the run function, inside the worker class, and i got that timer error, which i understood that is because of the graphic plotting, meaning that i will have to do the plotting inside my main function. and because of that, when i put the plotting code in my main function, it gives me an error because it cannot see the variables calculated in the run function.
i am trying to run a scenario where i have a producer which is capturing frames from webcam and putting it in a queue.
and then consumer reads image from input queue and does some processing and puts o/p image in outgoing queue.
Issue is, consumer read from queue is not blocking. Ideally it should be, also when it reads value from queue, size is always constant 128, which is wrong. I am sure size of image that I am putting in queue is far greater.
from __future__ import print_function
import multiprocessing
import time
import logging
import sys
import cv2
class Consumer(multiprocessing.Process):
def __init__(self, incoming_q, outgoing_q):
multiprocessing.Process.__init__(self)
self.outgoing_q = outgoing_q
self.incoming_q = incoming_q
def run(self):
proc_name = self.name
print(f"{proc_name} - inside process_feed..starting")
while True:
#print(f"size of incoming_q=>{self.incoming_q.qsize()}")
try:
#print(f"{proc_name} - size of B incoming_q=>{self.incoming_q.qsize()}")
image_np = self.incoming_q.get(True)
size_of_img = sys.getsizeof(image_np)
#print(f"{proc_name} - size of A incoming_q=>{self.incoming_q.qsize()}")
if size_of_img > 128:
print(f"{proc_name} - size image=>{size_of_img}")
time.sleep(1)
self.outgoing_q.put_nowait(image_np)
except:
pass
print("inside process_feed..ending")
class Producer(multiprocessing.Process):
def __init__(self, incoming_q, outgoing_q):
multiprocessing.Process.__init__(self)
self.incoming_q = incoming_q
self.outgoing_q = outgoing_q
def run(self):
proc_name = self.name
print("inside capture_feed")
stream = cv2.VideoCapture(0)
try:
counter = 0
while True:
counter += 1
if counter == 1:
if not self.incoming_q.full():
(grabbed, image_np) = stream.read()
size_of_img = sys.getsizeof(image_np)
print(f"{proc_name}........B.......=>{self.incoming_q.qsize()}")
print(f"{proc_name} - size image=>{size_of_img}")
self.incoming_q.put(image_np)
print(f"{proc_name}........A.......=>{self.incoming_q.qsize()}")
counter = 0
try:
image_np = self.outgoing_q.get_nowait()
logging.info("reading value for o/p")
cv2.imshow('object detection', image_np)
except:
pass
if cv2.waitKey(25) & 0xFF == ord('q'):
break
finally:
stream.release()
cv2.destroyAllWindows()
print("inside capture_feed..ending")
if __name__ == '__main__':
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
stream = cv2.VideoCapture(0)
incoming_q = multiprocessing.Queue(maxsize=100)
outgoing_q = multiprocessing.Queue(maxsize=100)
logging.info("before start of thread")
max_process = 1
processes = []
processes.append(Producer(incoming_q, outgoing_q))
for i in range(max_process):
p = Consumer(incoming_q, outgoing_q)
p.daemon = True
processes.append(p)
logging.info("inside main thread..middle")
for p in processes:
p.start()
logging.info("inside main thread..ending")
logging.info("waiting in main thread too....")
logging.info("waiting in main thread finished....")
for p in processes:
p.join()
logging.info("inside main thread..ended")
I was able to figure out issue with my approach. I missed whole concept of pickle (serialization).
I changed my code to serialize numpy array before writing to queue and deserialize after reading it. Code started working as expected.
also printing 128 as sizeof np array is fine, i was misinterpreting that number.
def serialize_ndarray(arr:np.ndarray):
serialized = pickle.dumps(arr)
return serialized
def deserialize_ndarray(string):
data = pickle.loads(string)
return data
I am afraid that I'm not doing the multithreading thing the right way, so I came here in search of wisdom. I have two arrays of addresses and I have to check if the address of the first array exists in the second array and in case it doesn't look for the most similar address in array 2.
The array that has the "oficial" addresses is called directory and the array that I need to validate is called look_address.
The code goes as follows:
import pandas as pd
import numpy as np
from fuzzywuzzy import fuzz
from fuzzywuzzy import process
from datetime import datetime,timedelta
import threading
import queue
class myThread(threading.Thread):
def __init__(self,threadID,name,q):
threading.Thread.__init__(self)
self.threadID = threadID
self.name=name
self.q = q
def run(self):
print(f"starting {self.name}")
process_data(self.name,self.q)
print(f"ending {self.name}")
locs = []
ratios={}
def process_data(threadName,q):
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
d = q.get()
queueLock.release()
d = d.strip()
if directory.isin([d]).any():
locs.append(d)
else:
pos = process.extract(d,directory.values,scorer=fuzz.ratio,limit=50)
ratios[d] = pos
else:
queueLock.release()
threadlist = ["T-1","T-2","T-3","T-4","T-5","T-6","T-7","T-8","T-9","T-10"]
nameList = look_address
queueLock = threading.Lock()
workQueue = queue.Queue(len(nameList)+1)
threads=[]
threadID=1
exitFlag=0
for name in threadlist:
thread = myThread(threadID,name,workQueue)
thread.start()
threads.append(thread)
threadID+=1
queueLock.acquire()
for addr in nameList:
workQueue.put(addr)
queueLock.release()
total_steps = len(workQueue.queue)
tot_sec = 0
t0 = datetime.now()
while not workQueue.empty():
total_seconds =(datetime.now()-t0).total_seconds()
if total_seconds == 0:
total_seconds = 1e-8
progress = 1-len(workQueue.queue)/total_steps
tot_sec+=total_seconds
print("\rProgreso: {pr:.2f}% || Buenas/Errores: {gb}/{bd}".format(
pr = progress*100,
its = 1/total_seconds,
elap = timedelta(seconds=np.round(tot_sec)),
gb=len(locs),
bd=len(errors),
eta = timedelta(seconds=np.round(total_seconds*(total_steps-len(workQueue.queue))))),end="",flush=True)
exitFlag = 1
for t in threads:
t.join()
print("\nExiting Main Thread")
Each request in process.extract takes around 25s (did a %timeit). Now, with the script above it doesn't seems to speed up the data processing. It has been running for like 2 hours and it has progressed by around 4.29%.
My two questions are:
Is the implementation of multithreading correct?
How can I speed up the data processing? Maybe run this on a VPS on amazon or google?
I want to understand why this is so slow and how I can speed things up.
EDIT: Changed from:
if not workQueue.empty():
d = q.get()
d = d.strip()
if directory.isin([d]).any():
locs.append(d)
else:
pos = process.extract(d,directory.values,scorer=fuzz.ratio,limit=50)
ratios[d] = pos
queueLock.release()
to:
if not workQueue.empty():
d = q.get()
queueLock.release()
d = d.strip()
if directory.isin([d]).any():
locs.append(d)
else:
pos = process.extract(d,directory.values,scorer=fuzz.ratio,limit=50)
ratios[d] = pos
I want to run a program that after being load will refresh tkinter canvas automatically.
I want the other process to be independent from tkinter main loop
In pseudo code i want it to behave something like this:
if __name__ == "__main__":
app = loadTkinterWindowsWithCanvas()
app.mainloop()
manager = Manager.Manager()
dataProcess = Process(target = manager.start())
dataProcess.start()
oldX = None
x = None
while True:
time.sleep(1)
oldX, x = x , manager.x
if oldX is not x:
app.updateCanvas(x)
...
#in Manager Class
def start(self):
self.x = 0
while True:
self.x += 1
I was trying out a lot of examples from stackoverflow or tutorials, with process threads,lock and event but i cant get behavior i desire. My program either doesn't load or doesn't update. What is best approach ?
This is what my code looks right now
if __name__ == "__main__":
lock = mp.RLock()
app = MainFrame()
listOfCities = [(0,121),(112,5),(14,201),(45,88),(141,231),(1,8),(22,11),(101,84),(90,231)]
pop = Population.Population(200,listOfCities)
manager = EvolutionManager.EvolutionManager(100,pop,lock)
pro = mp.Process(target = manager.startTraining())
pro.start()
app.mainloop()
lastBest = None
best = None
while True:
print("x")
time.sleep(0.2)
lastBest, best = best,manager.population.bestSalesman
if lastBest is not best:
app.getCurrentTopFrame().updateFrame(best.dna.getAsListOfTuple())
Behavior is that while second process start window does not load until the process ends
UPDATE
This started to work.
def genethicAlgorithmPart(event,manager):
manager.startTraining(event)
def addChangerListiner(manager,app,event):
thread = t.Thread(target = changeListiner, args = (manager,app,event,))
thread.start()
def changeListiner(manager,app,event):
lastBest = None
best = None
print("Starting listiner thread")
while True:
print("x")
event.wait()
lastBest, best = best,manager.population.bestSalesman
if lastBest is not best:
app.getCurrentTopFrame().updateFrame(best.dna.getAsListOfTuple())
event.clear()
if __name__ == "__main__":
# listOfCities = [(631, 44), (612, 137), (441, 266), (447, 173), (52, 243), (104, 148), (333, 70), (474, 182), (419, 221), (238, 291), (264, 340), (290, 213), (332, 97), (473, 294), (188, 198), (180, 258), (433, 382), (394, 139)]
listOfCities = rmg.genRandomListOfPoints(15,800,400)
pop = Population.Population(400,listOfCities)
manager = EvolutionManager.EvolutionManager(100,pop)
event = mp.Event()
pro = t.Thread(target = genethicAlgorithmPart, args = (event, manager,))
app = MainFrame.MainFrame()
app.getCurrentTopFrame().updateFrame(manager.population.bestSalesman.dna.getAsListOfTuple())
app.after(111, addChangerListiner(manager, app, event))
pro.start()
app.mainloop()
Hi I'm new at programming in python and gtk.
I'm writing a program to do some measurement.
For plotting the measurement, I use matplotlib.
The program will have a function to turn a heater on and off and to make the measurement.
I want to use separate threads for the heater and the measurement.
For now the communication with the hardware hasn't been implemented yet in this program.
The problem is when I click the "measurebutton", the "progressbar" doesn't work anymore.
I get a message:
gtk.ProgressBar object at 0x29b8460 (uninitialized at 0x0)
When I only use the heaterbutton, the progressbar keeps working
What am I doing wrong ?
This is the code
#!/usr/bin/env python
import pygtk
pygtk.require('2.0')
import gtk
import time
import gobject
import threading
import matplotlib
import numpy as np
from matplotlib.backends.backend_gtkagg import FigureCanvasGTKAgg as FigureCanvas
class measure:
# callback to quit
def delete_event(self, widget, event, data = None):
gtk.main_quit()
return False
def heater_helper(self, widget, heater, progressbar):
print "starting heater thread"
threading.Thread(target=self.heater_cb, args=(widget, heater, progressbar)).start()
def heater_cb(self, widget, heater, progressbar):
heaterstring = "6.3"
heater = eval(heaterstring)
stap = 1
j = 0.1
heatervalue = widget.get_active()
print heatervalue
progressbar.set_fraction(0.1)
while (stap <= 10 ):
if widget.get_active():
print widget.get_active()
fraction = j * stap
print fraction
progressbar.set_fraction(fraction)
stap = stap + 1
time.sleep(1)
else:
stap = 11
progressbar.set_fraction(0.0)
break
def do_measurement_helper(self, widget, fig):
print " Start measurement thread"
threading.Thread(target=self.do_measurement, args=(widget, fig)).start()
def do_measurement(self, widget, fig):
fig.clear()
ax = fig.add_subplot(111)
x = np.arange(0, 5*np.pi, 0.01)
y = np.sin(x**2)*np.exp(-x)
ax.plot(x, y)
fig.canvas.draw()
def __init__(self):
# Create new Window
self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
self.window.connect("delete_event", self.delete_event)
self.window.show()
mainbox = gtk.HBox(False, 0)
self.window.add(mainbox)
mainbox.show()
leftvbox = gtk.VBox(False, spacing = 10)
mainbox.pack_start(leftvbox, expand = False, fill = False, padding = 0)
leftvbox.show()
rightvbox = gtk.VBox(False, spacing = 10)
mainbox.pack_start(rightvbox, expand = False, fill = False, padding =0)
rightvbox.show()
heaterprogressbar = gtk.ProgressBar()
leftvbox.pack_start(heaterprogressbar, expand = False, fill = False, padding = 0)
heaterprogressbar.show()
heaterbutton = gtk.ToggleButton("Heater")
leftvbox.pack_start(heaterbutton, expand = True, fill = False, padding = 0)
heaterbutton.show()
heaterbutton.connect("toggled", self.heater_helper, heaterbutton, heaterprogressbar)
fig = matplotlib.figure.Figure(figsize=(5,4), dpi=64)
canvas = FigureCanvas(fig)
rightvbox.pack_start(canvas, expand = True, fill = True, padding = 0 )
canvas.show()
measurebutton = gtk.Button("Measure")
rightvbox.pack_start(measurebutton, expand = False, fill = False, padding = 0)
measurebutton.show()
measurebutton.connect("clicked", self.do_measurement_helper, fig)
def main():
gtk.main()
return(0)
if __name__ == "__main__":
gtk.gdk.threads_init()
measure()
main()
gtk.gdk.threads_leave()
Kind regards,
Joris Weijters
Combining threads, Matplotlib, and the GTK main loop is probably not supported and difficult to debug exactly what is going on. My advice is not to do any GUI calls from threads, but instead schedule them using gobject.idle_add().
Threads, Matplotlib and the GTK main loop can be combined, if you keep in mind some things:
I use gobject.threads_init() instead of gtk.gdk.threads_init(), the gtk.gdk variant did not work for me in combination with Matplotlib. I think you can also omit the gtk.gdk.threads_leave().
As ptomato mentioned, you should let the main gtk thread handle anything that has to do with gtk widgets by calling the gobject.idle_add() and gobject.timeout_add() functions.
I usually make a helper function to periodically update the statusbar from a float variable:
def do_measurement(self):
self.data = []
self.progress = 0
self.abort = threading.Event()
gobject.timeout_add(100, self.update_progressbar)
for point in some_generator_yielding_100_values():
if self.abort.is_set():
break
self.data.append(point)
self.progress += 0.01
self.progress = None
def update_progressbar(self):
if self.progress is None:
self.progressbar.set_fraction(0) # reset bar
return False # do not run again
self.progressbar.set_fraction(self.progress)
return True # run again after 100ms
def start_measurement(self):
threading.Thread(target=self.do_measurement).start()
def stop_measurement(self):
self.abort.set()
But you can of course also just call gobject.idle_add(self.progressbar.set_fraction, x) to set the new value x asynchroneously.