import time
import datetime as dt
import urllib.request
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt
import matplotlib.animation as Animation
from matplotlib import style
import matplotlib
import csv
import threading
style.use('fivethirtyeight')
fig = plt.figure()
def usd_in_bitcoin():
try:
resp = urllib.request.urlopen("https://bitcoinwisdom.com/")
except Exception as e:
print(e)
text = resp.read()
soup = BeautifulSoup(text, 'html.parser')
intermediate = soup.find('tr', {"id": "o_btcusd"})
ans = intermediate.find('td', {'class': 'r'})
return ans.contents[0]
def write_to_file(interval):
while True:
value = str(usd_in_bitcoin())
unix_time = str(time.time())
print(unix_time, value)
with open('bitcoin_usd.csv', 'a+') as file:
file.write(unix_time)
file.write("," + str(value))
file.write('\n')
time.sleep(interval)
def animate(i):
with open('bitcoin_usd.csv') as csv_file:
readcsv = csv.reader(csv_file, delimiter=',')
xs = []
ys = []
for row in readcsv:
if len(row) > 1:
x, y = [float(s) for s in row]
xs.append(dt.datetime.fromtimestamp(x))
ys.append(y)
print(len(xs))
dates = matplotlib.dates.date2num(xs)
# print(dates)
fig.clear()
plt.plot_date(dates, ys)
def plotting():
ani = Animation.FuncAnimation(fig, animate, interval=1000)
plt.show()
def main():
# plotting()
b = threading.Thread(name='making graph', target=plotting)
# a = threading.Thread(name='updating_csv', target=write_to_file, args=(5,))
# a.start()
b.start()
if __name__ == '__main__':
main()
In the above block of code, I am trying to plot the value of a bitcoin in usd by using scraping and then putting the value in a csv file.
Then I read the csv file to plot the graph.
Both plotting and scraping seem to work fine but if I do both of them simultaneously, I am getting an error saying main thread not in main loop. I searched a lot but was not able to solve this problem
The problem here is with the sequence of lines in main()
Try this:
def main():
a = threading.Thread(name='updating_csv', target=write_to_file, args=(5,))
a.start()
b = threading.Thread(name='making graph', target=plotting)
b.start()
plotting()
Related
I created a code (using Tkinter, Python3 and matplotlid) that could read data from different serial ports, save them to csv, then create graphs and finally preview data in GUI. The code was splited in two different scripts. The main script contained reading data, save data to csv an priview of data and the other script contained the graph creation.
Today I rewrote the code using the answer of #user2464430 here. The code is working, but I can't update the GUI. Opens once and then no refresh with new data.
The following code is a part of total code.
My code is:
from PIL import ImageTk, Image
import tkinter as Tk
import multiprocessing
from queue import Empty, Full
from time import strftime
import serial
import numpy as np
import matplotlib.pyplot as plt
from drawnow import *
from pylab import *
import pandas as pd
from datetime import timedelta
from datetime import datetime
import plotly.graph_objs as go
from plotly.subplots import make_subplots
import locale
import os
class GuiApp(object):
def __init__(self, image):
self.root = Tk.Tk()
self.root.resizable(width=False, height=False)
self.root.geometry("1600x800+0+0")
C = Canvas(self.root, bg="black", width=1600, height=800)
def BasicLabels():
....... # in this stage create multiple axis labels
Î¥AxisLabels()
BasicLabels()
def ValueLabels():
....... # Read and munipulate datas from CSV file and print in in labels
ValueLabels()
C.pack()
def GenerateData(q): #Read Serial Ports and store data to CSV file
file_exists = os.path.isfile("BigData.csv")
header = [["Daytime,T1"]]
if not file_exists:
with open("BigData.csv", "a+") as csvfile:
np.savetxt(csvfile, header, delimiter=",", fmt="%s", comments="")
while True:
try:
ser1 = serial.Serial(port="COM4", baudrate=9600)
read_ser1 = ser1.readline()
if read_ser1 == "":
read_ser1 = "Missing Value"
else:
read_ser1 = ser1.readline()
read_ser1 = str(read_ser1[0 : len(read_ser1)].decode("utf-8"))
# print("COM4:", read_ser1)
ser1.close()
except:
print("Failed 1")
read_ser1 = "9999,9999,9999,9999,9999"
daytime = strftime(" %d-%m-%Y %H:%M:%S")
rows = [
daytime
+ ","
+ read_ser1.strip()
]
with open("BigData.csv", "a+") as csvfile:
np.savetxt(csvfile, rows, delimiter=",", fmt="%s", comments="")
CreateGraphs()
def CreateGraphs():
#Code to generate graph. Called every time i have new line in CSV.
if __name__ == "__main__":
# Queue which will be used for storing Data
q = multiprocessing.Queue()
q.cancel_join_thread() # or else thread that puts data will not term
gui = GuiApp(q)
t1 = multiprocessing.Process(target=GenerateData, args=(q,))
t1.start()
gui.root.mainloop()
t1.join()
The graphs are generating after while True in GenerateData.
All datas for labels and graphs are coming from CSV file and not directly from serial port.
Is it possible to update GUI with latest datas from CSV and created graphs?
Thank for your time.
I want to limit my graph from left and set (the current time - 2 hours) to xlim. I tried to add to the "update" function this
self.ax.set_xlim(left=max(self.data.iloc[0, 0], self.data.iloc[-1, 0] - pd.Timedelta(hours=2)))
But this is doesn't work. Could anyone help me do this, please?
from matplotlib import pyplot as plt
from matplotlib.animation import FuncAnimation
from datetime import datetime
import pandas as pd
def to_pd(wt, wf):
p = pd.DataFrame({"Date": [wt], "Cost": [wf]})
p["Date"] = pd.to_datetime(p["Date"], format='%Y-%m-%d %H:%M:%S')
return p
fig = plt.figure(figsize=(18,8), dpi=90)
class Graph():
def __init__(self):
self.ax = fig.add_subplot()
self.start_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.data = to_pd(self.start_time,0)
self.line, = self.ax.plot(self.data.Date,self.data.Cost)
def update(self,i):
self.current_time = (self.data.iloc[-1,0]+pd.Timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
self.data = self.data.append(to_pd(self.current_time,(-1)**i))
self.line.set_data(self.data.Date, self.data.Cost)
self.ax.relim()
self.ax.autoscale_view()
return self.line,
object = Graph()
def animate(i):
return object.update(i)
anim = FuncAnimation(fig, animate, frames=200,interval=100, blit=True)
plt.show()
Solved, but I think there is more convenient way. But, here what am done, just added it into the 'update' function, it's clear all old data:
if self.data.iloc[-1, 0] - pd.Timedelta(hours=2) >= self.data.iloc[0, 0]:
self.data = self.data[self.data.Date > self.data.iloc[0, 0]]
Can we able to store our own trained dataset in xml format using pickle?
import numpy as np
import random
import pickle
import matplotlib
gui_env = [i for i in matplotlib.rcsetup.interactive_bk]
non_gui_backends = matplotlib.rcsetup.non_interactive_bk
print ("Non Gui backends are:", non_gui_backends)
print ("Gui backends I will test for", gui_env)
for gui in gui_env:
print ("testing", gui)
try:
matplotlib.use(gui,warn=False, force=True)
from matplotlib import pyplot as plt
print (" ",gui, "Is Available")
plt.plot([1.5,2.0,2.5])
fig = plt.gcf()
fig.suptitle(gui)
print ("Using ..... ",matplotlib.get_backend())
except:
print (" ",gui, "Not found")
import os
import cv2
from tqdm import tqdm
DATADIR = "Datasets/PetImages"
CATEGORIES = ["Dog", "Cat"]
training_data = []
IMG_SIZE = 50
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR,category)
class_num = CATEGORIES.index(category)
for img in tqdm(os.listdir(path)):
try:
img_array = cv2.imread(os.path.join(path,img) ,cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
training_data.append([new_array, class_num])
except Exception as e:
pass
random.shuffle(training_data)
X = []
y = []
for features,label in training_data:
X.append(features)
y.append(label)
print(X[0].reshape(-1, IMG_SIZE, IMG_SIZE, 1))
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
pickle_out = open("X.xml","wb")
pickle.dump(X, pickle_out)
pickle_out.close()
pickle_out = open("y.xml","wb")
pickle.dump(y, pickle_out)
pickle_out.close()
create_training_data()
print(len(training_data))
Here's my part of little program for testing (will be in further used)
main.py
import requests
import re
import os
import random
from bs4 import BeautifulSoup
from flask import Flask, request, abort
#from imgurpython import ImgurClient
from argparse import ArgumentParser
from linebot import (
LineBotApi, WebhookHandler
)
from linebot.exceptions import (
InvalidSignatureError
)
from linebot.models import *
from apple_news import apple_newss
a=input("plz:")
if a == "applenews":
content0 = apple_newss_content0
content1 = apple_newss_content1
content2 = apple_newss_content2
print (content0)
print (content1)
print (content2)
and my sub program
apple_news.py
def apple_newss( apple_newss_content0, apple_newss_content1, apple_newss_content2):
target_url = 'http://www.appledaily.com.tw/realtimenews/section/new/'
print('Start parsing appleNews....')
rs = requests.session()
res = rs.get(target_url, verify=False)
soup = BeautifulSoup(res.text, 'html.parser')
list_links = [] # Create empty list
for a in soup.select("div[class='abdominis rlby clearmen']")[0].findAll(href=True): # find links based on div
if a['href']!= None and a['href'].startswith('https://'):
list_links.append(a['href']) #append to the list
print(a['href']) #Check links
#for l in list_links: # print list to screen (2nd check)
# print(l)
print("\n")
random_list = [] #create random list if needed..
random.shuffle(list_links) #random shuffle the list
apple_newss_content0 = ''
apple_newss_content1 = ''
apple_newss_content2 = ''
for i in range(3): # specify range (5 items in this instance)
res = list_links.pop(random.randint(1, len(list_links))) # pop of each item randomly based on the size of the list
random_list.append(res)
#print(res)
#print(random_list)
print("\n")
apple_newss_content0=random_list[0]
apple_newss_content1=random_list[1]
apple_newss_content2=random_list[2]
I just can't figure out how to return
apple_newss_content0 apple_newss_content1 apple_newss_content2
to main.py
(for some purpose it must be in separated form which is 3 link actually)
And make main.py print those 3 individually
(I've stuck this question for month.....)
Thanks!!!
if i understand your question, you can simple do
def apple_newss():
apple_newss_content0 = 0
apple_newss_content1, apple_newss_content2 = 1, 2
return apple_newss_content0, apple_newss_content1, apple_newss_content2
then in main.py:
content0, content1, content2 = apple_newss()
eg.
>>> def apple_newss():
... apple_newss_content0 = 0
... apple_newss_content1, apple_newss_content2 = 1, 2
... return apple_newss_content0, apple_newss_content1, apple_newss_content2
...
>>> content0, content1, content2 = apple_newss()
>>> content0, content1, content2
(0, 1, 2)
>>> print(content0)
0
>>> print(content1)
1
>>> print(content2)
2
import requests
from requests import Session
from bs4 import BeautifulSoup
import re
from multiprocessing.dummy import Pool as ThreadPool
#s = Session()
def get_photo_from_page():
tut = []
r = requests.get('https://vk.com/uporols_you').text
soup = BeautifulSoup(r, 'lxml')
im = soup.find_all('img', class_="ph_img")
for a in im:
s = a.get('data-src_big').split('|')[0]
tut.append(s)
y = "img%s.jpg"
for t, im in tut, [y % i for i in range(1,5)]:
p = requests.get(t)
out = open(im, "wb")
out.write(p.content)
out.close()
def main():
get_photo_from_page()
if __name__ == '__main__':
main()
error from cmd for t, im in tut, [y % i for i in range(1,5)]:
ValueError: too many values to unpack (expected 2)
> I need to list with a 1 to 1 accrue to URL, and on passage possylke,
and save all images with the new name, in separate cycles, it always
takes the last available reference and stores it as the number of
times indicated in the cycle.
import requests
from requests import Session
from bs4 import BeautifulSoup
import re
from multiprocessing.dummy import Pool as ThreadPool
#s = Session()
def get_photo_from_page():
tut = []
r = requests.get('https://m.vk.com/uporols_you').text
soup = BeautifulSoup(r, 'lxml')
im = soup.find_all('img', class_="ph_img")
try:
for a in im:
s = a.get('data-src_big').split('|')[0]
tut.append(s)
print(tut)
except:
print('no have any links)')
for num, link in enumerate(tut, start=1):
p = requests.get(link)
out = open("img%s.jpg" % (num), 'wb')
out.write(p.content)
out.close()
def main():
get_photo_from_page()
if __name__ == '__main__':
main()