How to run two python scripts in parallel on Windows - multithreading

I currently have two python scripts, one is for writing data into a file and the other one is for reading data continuously from that file at the same time and do plotting. I want to create a third script which can automatically run these two scripts at the same time ( actually one slightly ahead of another because the file needs to be created for the other script to read from it).
So here is my code:
import serial
import sys
import Queue
import threading
import scipy.io
import numpy as num
from mpl_toolkits.mplot3d import *
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm
import matplotlib.animation as animation
import time
from time import sleep
AngleText = open ("data2.txt" , "w") # open file for writing data
a= 1
b= 1
c =1
for x in range (0,20):
count = 0
sleep(0.5)
AngleText.writelines (str(a)+',')
AngleText.writelines (str(b)+',')
AngleText.writelines (str(c)+'\n')
count = count +1
a= a + 1
b= b + 2
c= c + 3
AngleText.flush()
And the plotting script:
from mpl_toolkits.mplot3d import *
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
import matplotlib
import threading as thrd
import numpy as np
fig = plt.figure()
ax = fig.add_subplot(111, projection = '3d')
def animate(i):
pullData = open ('data2.txt','r').read()
dataArray = pullData.split('\n')
xar = []
yar = []
zar = []
for eachLine in dataArray:
if len(eachLine)>1:
x,y,z = eachLine.split(',')
xar.append(float(x))
yar.append(float(y))
zar.append(float(z))
tx = xar.pop(0)
Floatx= float(tx)
ty = yar.pop(0)
Floaty= float(ty)
tz = zar.pop(0)
Floatz= float(tz)
x1 = [164, 94, 0, -100.5]
x2 = [164, 94, 0, -100.5]
y1= [-72.5, -103.5, -103.5, -134.5]
y2= [72.5, 103.5, 103.5, 134.5]
z1 = [112, 60, 3, 3]
z2 = [112, 60, 3, 3]
ax.plot(x1, y1, z1, color = 'b')
plt.hold(True)
ax.plot(x2, y2, z2, color = 'r')
plt.hold(True)
ax.scatter(Floatx, Floaty, Floatz)
plt.hold(False)
ani = animation.FuncAnimation(fig,animate, interval = 25)
ax.set_xlabel('x label')
ax.set_ylabel('y label')
ax.set_zlabel('z label')
plt.show()
Any advice would be appreciated!
My current solution is:
import os
from subprocess import *
import time
from time import sleep
p = Popen([r'testing.py'], shell=True, stdin=PIPE, stdout=PIPE)
output = p.communicate()
print output[0]
sleep(0.5)
#run child script 2
p = Popen([r'realPlot2.py'], shell=True, stdin=PIPE, stdout=PIPE)
output = p.communicate()
print output[0]
By running the above code, the graph won't be able to show until the writing function is finished.

use the multiprocessing class. you can start a new process to write to the file, then call p.join() to make the main function wait for the first process to finish, then start the second process. see here:
http://docs.python.org/2/library/multiprocessing.html

Related

Is there a way in an external .py to catch data populated by a kivy button (row by row) and then get len() of that dataset as well?

How do I catch tuples generated by clicks on the kivy button in file 1 and the corresponding number of rows, i.e. len(), of that number of rows in file 2? Any support out there is much, much appreciated....
View explanation below...
I created a kivy app delivering a row of tupled values every time I click my button. That works fine. Now I want to pick up, e.g. a dataset of five clicks on button, i.e. 5 rows of tuples. Below is what I did in file 1:
file 1.py:
kv = '''
<Launch>:
BoxLayout:
Button:
size:(80,80)
size_hint:(None,None)
text:"..."
on_press: root.build()
'''
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
warnings.simplefilter(action='ignore', category=DeprecationWarning)
warnings.simplefilter(action='ignore', category=RuntimeWarning)
warnings.simplefilter(action='ignore', category=UserWarning)
import pandas as pd
import numpy as np
from kivy.app import App
from kivy.uix.button import Button
def test(t):
size = t
a = pd.DataFrame(columns=['col1', 'col2', 'col3'])
a['col1'] = pd.DataFrame(np.random.randint(1, 50, size))
a['col2'] = pd.DataFrame(np.random.randint(1, 50, size))
a['col3'] = pd.DataFrame(np.random.rand(size))
t = a
return t
def vars_n(self):
a = test(t=1)
# Define objects for dataframe and col inputs
self.a_num = pd.DataFrame(test(1))
self.a_limit = a[(a.col3) < 1 & (a.col3 > 0.8)]
self.a_col1 = a['col1']
self.a_col2 = a['col2']
self.a_col3 = a['col3']
cols = self.a_col1, self.a_col2, self.a_col3
lst = []
self.a_col1, self.a_col2, self.a_col3 = 'src', 'dest', 'col3'
for a in range(1):
lst.append([self.a_col1, self.a_col2, self.a_col3])
self.a_col1, self.a_col2, self.a_col3 = \
np.random.randint(1, 40, size=1), np.random.randint(1, 40, size=1), np.random.rand(1)
df = pd.DataFrame(lst, columns=cols)
tuple1 = self.a_col1
tuple2 = self.a_col2
tuple3 = self.a_col3
q = tuple(zip(tuple1, tuple2, tuple3))
return q
class MyDf(App, object):
def __init__(self):
super().__init__()
def test_def(self):
msg = test(1)
print(msg)
def test_vars_n(self):
msg = vars_n(test(t=1))
print(msg)
def length(self):
result = len(vars_n(test(t=1)))
print(result)
# Define output for activation of kivy button
def press(self, instance):
print(vars_n(test(t=1)))
# Define kivy button configuration
def build(self):
butt=Button(text="...")
butt.bind(on_press=self.press)
return butt
MyDf().run()
Result after e.g. five clicks, could generate below dataset:
((6, 22, 0.8525529856428397),)
((12, 7, 0.3912468711230911),)
((30, 14, 0.979806646854341),)
((21, 27, 0.618131650972481),)
((8, 20, 0.9164440407619223),)
So, in file 2, I'd like to pull above five lines in the dataset above and, at the same time, get the len of that dataset, i.e. 5. Tried this, but it does not seem to catch the output of file 1:
file 2.py:
import pandas as pd
import numpy as np
my_instance = MyDf()
interactions = my_instance.test_vars_n()
interactions = np.array(interactions)
print(len(interactions)) # testing result
Got this error:
Traceback (most recent call last):
File "G:\...\...\...\....\file2.py", line 38, in <module>
print(len(interactions))
TypeError: len() of unsized object
EDITING with example I was inspired by in above attempts:
File 1
import pandas as pd
import numpy as np
def test(t):
size = t
a = pd.DataFrame(columns=['col1', 'col2', 'col3'])
a['col1'] = pd.DataFrame(np.random.randint(1, 50, size))
a['col2'] = pd.DataFrame(np.random.randint(1, 50, size))
a['col3'] = pd.DataFrame(np.random.rand(size))
t = a
return t
class ClassTest(object):
def test_def(self):
msg = test(1)
print(msg)
File 2:
from Call_an_outside_function_from_class_file_1 import ClassTest
my_new_instance = ClassTest()
ClassTest().test_def()
Got this result, and without using the App.get_running_app() replacement:
col1 col2 col3
0 48 3 0.514489
Process finished with exit code 0

How can the `_property_values` of an element of a bokeh `figure.renderers` be changed directly?

How can the _property_values of an element of a bokeh figure.renderers be changed directly? I learned that the lements of renderers have an id, so I expect to do something like renderers['12345']. But as it is a list (a PropertyValueList to be more precise), this doesn't work. Instead, the only solution I found is to iterate over the list, storing the correct element in a new pointer (?), modifying the pointer and thus modifying the original element.
Here is my toy example where a vertical line in a histogram is updated based on some widget's value:
import hvplot.pandas
import ipywidgets as widgets
import numpy as np
from bokeh.io import push_notebook, show, output_notebook
from bokeh.models import Span
from bokeh.plotting import figure
%matplotlib inline
hist, edges = np.histogram([1, 2, 2])
p = figure()
r = p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:])
vline = Span(location=0, dimension='height')
p.renderers.extend([vline])
def update_hist(x):
myspan = [x for x in p.renderers if x.id==vline.id][0]
myspan._property_values['location'] = x
show(p, notebook_handle=True)
widgets.interact(update_hist, x = widgets.FloatSlider(min=1, max=2))
Bigreddot pointed me into the right direction: I don't have to update p directly, but the elements used to generate p (here the Span). By this I found the this question where the code bears the solution: update vline.location.
Full code:
import hvplot.pandas
import ipywidgets as widgets
import numpy as np
from bokeh.io import push_notebook, show, output_notebook
from bokeh.models import Span
from bokeh.plotting import figure
%matplotlib inline
hist, edges = np.histogram([1, 2, 2])
p = figure()
r = p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:])
vline = Span(location=0, dimension='height')
p.renderers.extend([vline])
show(p, notebook_handle=True)
def update_hist(x):
vline.location = x
push_notebook()
widgets.interact(update_hist, x = widgets.FloatSlider(min=1, max=2, step = 0.01))
As a Python beginner, I still often oversee, that Python does not have variables. So we can change an element x by changing y.
x = ['alice']
y = x
y[0] = 'bob'
x # is now ['bob] too

Cartopy wmts KeyError issue : TileMatrixLimits

Since a while, I am using the wmts function of cartopy but since last week this function does not works returning this error:
KeyError: 'TileMatrixLimits with tileMatrix "1" already exists'
The run code is coming from the cartopy web doc.
below the code:
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
def main():
url = 'https://map1c.vis.earthdata.nasa.gov/wmts-geo/wmts.cgi'
layer = 'VIIRS_CityLights_2012'
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
ax.add_wmts(url, layer)
ax.set_extent([-15, 25, 35, 60], crs=ccrs.PlateCarree())
ax.set_title('Suomi NPP Earth at night April/October 2012')
plt.show()
if __name__ == '__main__':
main()
I do not understand what is wrong.
Thank for help.
You can solve it commenting the lines 654 to 657 of (your Python dir)/Lib/site-packages/owslib/wmts.py which check the existence of the indexes in the static function from_elements of the TileMatrixSetLink class
Like this:
for limits_element in link_element.findall(path):
tml = TileMatrixLimits(limits_element)
if tml.tilematrix:
#if tml.tilematrix in tilematrixlimits:
# msg = ('TileMatrixLimits with tileMatrix "%s" '
# 'already exists' % tml.tilematrix)
# raise KeyError(msg)
tilematrixlimits[tml.tilematrix] = tml
links.append(TileMatrixSetLink(uri, tilematrixlimits))
For portability effects you can replace that function in your code before you use it
Like this:
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
### START REPLACEMENT ###
from owslib.wmts import TileMatrixSetLink, TileMatrixLimits, _TILE_MATRIX_SET_TAG, _TILE_MATRIX_SET_LIMITS_TAG, _TILE_MATRIX_LIMITS_TAG
def custom_from_elements(link_elements):
links = []
for link_element in link_elements:
matrix_set_elements = link_element.findall(_TILE_MATRIX_SET_TAG)
if len(matrix_set_elements) == 0:
raise ValueError('Missing TileMatrixSet in %s' % link_element)
elif len(matrix_set_elements) > 1:
set_limits_elements = link_element.findall(
_TILE_MATRIX_SET_LIMITS_TAG)
if set_limits_elements:
raise ValueError('Multiple instances of TileMatrixSet'
' plus TileMatrixSetLimits in %s' %
link_element)
for matrix_set_element in matrix_set_elements:
uri = matrix_set_element.text.strip()
links.append(TileMatrixSetLink(uri))
else:
uri = matrix_set_elements[0].text.strip()
tilematrixlimits = {}
path = '%s/%s' % (_TILE_MATRIX_SET_LIMITS_TAG,
_TILE_MATRIX_LIMITS_TAG)
for limits_element in link_element.findall(path):
tml = TileMatrixLimits(limits_element)
if tml.tilematrix:
tilematrixlimits[tml.tilematrix] = tml
links.append(TileMatrixSetLink(uri, tilematrixlimits))
return links
TileMatrixSetLink.from_elements = custom_from_elements
### END REPLACEMENT ###
def main():
url = 'https://map1c.vis.earthdata.nasa.gov/wmts-geo/wmts.cgi'
layer = 'VIIRS_CityLights_2012'
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
ax.add_wmts(url, layer)
ax.set_extent([-15, 25, 35, 60], crs=ccrs.PlateCarree())
ax.set_title('Suomi NPP Earth at night April/October 2012')
plt.show()
if __name__ == '__main__':
main()

Update plot in for loop in function

I'm trying to call a function like in the example below, and plot while running the code. The real values that I get as y-data are not really random numbers, but the point is that I would like it to get updated real-time. The plot in my example code below is just empty though and isn't getting updated.
import numpy as np
import matplotlib.pyplot as plt
import random as rnd
import time
initial_time = time.time()
def multiple_runs(number_of_runs):
x_data, y_data = [], []
fig, ax = plt.subplots()
sc = ax.scatter(x_data, y_data)
plt.draw()
for i in range(0, number_of_runs):
x_data.append(i+1)
y_data.append(rnd.randint(0,100))
sc.set_offsets(np.c_[x_data, y_data])
fig.canvas.draw_idle()
plt.pause(0.1)
print ('Total time after run number ' + str(i+1) + ': ' + str(time.time() - initial_time))
multiple_runs(100)
UPDATE:
Thanks #ImportanceOfBeingErnest , I got the code to work. However my problem right now is that the figure closes down as soon as it's finished, is there anyway to keep it open? I tried using plt.waitforbuttonpress() but I get a strange error from QTimer, not sure how or why. This is my working example code;
import numpy as np
import matplotlib.pyplot as plt
import random as rnd
import time
initial_time = time.time()
def multiple_runs(number_of_runs):
x_data, y_data = [], []
x_data2, y_data2 = [], []
fig, ax = plt.subplots(2, sharex = True)
sc = ax[0].scatter(x_data, y_data)
sc2 = ax[1].scatter(x_data2, y_data2)
ax[0].set(xlim=(0,100), ylim=(0,100))
ax[1].set(xlim=(0,100), ylim=(0,100))
plt.draw()
for i in range(0, number_of_runs):
x_data.append(i+1)
y_data.append(rnd.randint(0,100))
x_data2.append(i+1)
y_data2.append(rnd.randint(0,100))
sc.set_offsets(np.c_[x_data, y_data])
sc2.set_offsets(np.c_[x_data2, y_data2])
fig.canvas.draw_idle()
plt.pause(0.1)
print ('Total time after run number ' + str(i+1) + ': ' + str(time.time() - initial_time))
multiple_runs(100)
UPDATE2:
I tried using FuncAnimation, but getting the error TypeError: update() missing 2 required positional arguments: 'y' and 'y2'. I still need to use the for-loop because in my real code I'm using the previous values of y, to calculate the next values of y. This is my example code which is giving me the error;
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import random as rnd
import time
initial_time = time.time()
def multiple_runs(number_of_runs):
x_data, y_data = [], []
x_data2, y_data2 = [], []
fig, ax = plt.subplots(2, sharex = True)
sc = ax[0].scatter(x_data, y_data)
sc2 = ax[1].scatter(x_data2, y_data2)
ax[0].set(xlim=(0,100), ylim=(0,100))
ax[1].set(xlim=(0,100), ylim=(0,100))
def update(i, y, y2):
x_data.append(i+1)
y_data.append(y)
x_data2.append(i+1)
y_data2.append(y2)
sc.set_offsets(np.c_[x_data, y_data])
sc2.set_offsets(np.c_[x_data2, y_data2])
print ('Total time after run number ' + str(i+1) + ': ' + str(time.time() - initial_time))
for i in range(0, number_of_runs):
y = rnd.randint(0,100)
y2 = rnd.randint(0,100)
update(i,y,y2)
ani = FuncAnimation(fig, update, frames=number_of_runs, interval=100, repeat=False)
plt.show()
multiple_runs(100)
As commented, I would recommend to use FuncAnimation. This would look as follows in your case. Note that in order to close the window, one would need to press q or close it with the mouse.
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import random as rnd
import time
initial_time = time.time()
def multiple_runs(number_of_runs):
x_data, y_data = [], []
x_data2, y_data2 = [], []
fig, ax = plt.subplots(2, sharex = True)
sc = ax[0].scatter(x_data, y_data)
sc2 = ax[1].scatter(x_data2, y_data2)
ax[0].set(xlim=(0,100), ylim=(0,100))
ax[1].set(xlim=(0,100), ylim=(0,100))
def get_ydata(i):
y = rnd.randint(0,100)
y2 = rnd.randint(0,100)
return y, y2
def update(i):
y, y2 = get_ydata(i)
x_data.append(i+1)
y_data.append(y)
x_data2.append(i+1)
y_data2.append(y2)
sc.set_offsets(np.c_[x_data, y_data])
sc2.set_offsets(np.c_[x_data2, y_data2])
ani = FuncAnimation(fig, update, frames=number_of_runs, interval=100, repeat=False)
plt.show()
multiple_runs(100)

Main thread not in main loop error in threading module

import time
import datetime as dt
import urllib.request
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
import matplotlib.animation as Animation
from matplotlib import style
import matplotlib
import csv
import threading
style.use('fivethirtyeight')
fig = plt.figure()
def usd_in_bitcoin():
try:
resp = urllib.request.urlopen("https://bitcoinwisdom.com/")
except Exception as e:
print(e)
text = resp.read()
soup = BeautifulSoup(text, 'html.parser')
intermediate = soup.find('tr', {"id": "o_btcusd"})
ans = intermediate.find('td', {'class': 'r'})
return ans.contents[0]
def write_to_file(interval):
while True:
value = str(usd_in_bitcoin())
unix_time = str(time.time())
print(unix_time, value)
with open('bitcoin_usd.csv', 'a+') as file:
file.write(unix_time)
file.write("," + str(value))
file.write('\n')
time.sleep(interval)
def animate(i):
with open('bitcoin_usd.csv') as csv_file:
readcsv = csv.reader(csv_file, delimiter=',')
xs = []
ys = []
for row in readcsv:
if len(row) > 1:
x, y = [float(s) for s in row]
xs.append(dt.datetime.fromtimestamp(x))
ys.append(y)
print(len(xs))
dates = matplotlib.dates.date2num(xs)
# print(dates)
fig.clear()
plt.plot_date(dates, ys)
def plotting():
ani = Animation.FuncAnimation(fig, animate, interval=1000)
plt.show()
def main():
# plotting()
b = threading.Thread(name='making graph', target=plotting)
# a = threading.Thread(name='updating_csv', target=write_to_file, args=(5,))
# a.start()
b.start()
if __name__ == '__main__':
main()
In the above block of code, I am trying to plot the value of a bitcoin in usd by using scraping and then putting the value in a csv file.
Then I read the csv file to plot the graph.
Both plotting and scraping seem to work fine but if I do both of them simultaneously, I am getting an error saying main thread not in main loop. I searched a lot but was not able to solve this problem
The problem here is with the sequence of lines in main()
Try this:
def main():
a = threading.Thread(name='updating_csv', target=write_to_file, args=(5,))
a.start()
b = threading.Thread(name='making graph', target=plotting)
b.start()
plotting()

Resources