Streaming Live Websocket stream data and updating plotly graph with dash within a class - python-3.x

Apologies if this is a stupid question, but I'm wondering if it would be possible to create a dash app and stream live data straight into the graph?
I've been working on a trading bot which has a number of elements. I am currently trying to stream data that updates every 2000ms from a websocket and feed that straight into the graph.
I'm relatively new to programming so could use some assistance and criticism where neccesary!
So, as this live graph will ultimately be incorporated as part of the GUI and bot functionality, I am trying to make this priceTicker class encompass both initialising the dash app and also live updating the graph with the websocket stream that is received.
I am able to stream the data without problem and could parse the data into json, or csv or anything really. I am also able to initialise the graph in the dash app, but it isnt updating the graph with new data.
I have looked at a few SO posts around the general area but not found anything that explains it for my case unfortunately - sorry if there is any posts out there that i've missed!
Below is my full code:
from binance.client import Client
from binance.websockets import BinanceSocketManager
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Output, Input
import plotly
import plotly.graph_objs as go
from collections import deque
import pandas as pd
import json
class priceTicker:
def __init__(self, api_key, secret_key):
self.app = dash.Dash(__name__)
self.ticker_time = deque(maxlen=200)
self.last_price = deque(maxlen=200)
self.app.layout = html.Div(
[
dcc.Graph(id = 'live-graph', animate = True),
dcc.Interval(
id = 'graph-update',
interval = 2000,
n_intervals = 0
)
]
)
client = Client(api_key, secret_key)
socket = BinanceSocketManager(client)
socket.start_kline_socket('BTCUSDT', self.on_message, '1m')
socket.start()
def on_message(self, message):
app = self.app
price = {'time':message['E'], 'price':message['k']['c']}
self.ticker_time.append(message['E'])
self.last_price.append(message['k']['c'])
ticker_time = self.ticker_time
last_price = self.last_price
#print(self.ticker_time, self.last_price)
#app.callback(
Output('live-graph', 'figure'),
[Input('graph-update', 'n_intervals')])
def update_graph(ticker_time,last_price):
data = go.Scatter(
x = list(ticker_time),
y = list(last_price),
name ='Scatter',
mode = 'lines+markers'
)
return {'data': [data],
'layout': go.Layout(xaxis=dict(range=[min(ticker_time), max(ticker_time)]),
yaxis=dict(range=[min(last_price), max(last_price)]),)}
def Main():
api_key = ''
secret_key = ''
ticker = priceTicker(api_key, secret_key)
ticker.app.run_server(debug=True, host='127.0.0.1', port=16552)
if __name__ == '__main__':
#app.run_server(debug=False, host='127.0.0.1', port=10010)
Main()
I'm not sure where I'm going wrong but it doesn't seem like the call back function is being called. Sorry if i've missed something obvious!
I'm running on 3.7.6 and macos Big Sur
If anyone has any advice would be greatly appreciated.
cheers

For anyone that's interested or trying to do a similar thing to me.
I managed to fix the problem by calling the app.callback within init and assigning the callback to a function within the class:
from binance.client import Client
from binance.websockets import BinanceSocketManager
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Output, Input
import plotly
import plotly.graph_objs as go
from collections import deque
import pandas as pd
import json
class priceTicker:
def __init__(self, api_key, secret_key):
self.ticker_time = deque(maxlen=200)
self.last_price = deque(maxlen=200)
client = Client(api_key, secret_key)
socket = BinanceSocketManager(client)
socket.start_kline_socket('BTCUSDT', self.on_message, '1m')
socket.start()
self.app = dash.Dash()
self.app.layout = html.Div(
[
dcc.Graph(id = 'live-graph', animate = True),
dcc.Interval(
id = 'graph-update',
interval = 2000,
n_intervals = 0
)
]
)
self.app.callback(
Output('live-graph', 'figure'),
[Input('graph-update', 'n_intervals')])(self.update_graph)
#app.run_server(debug=True, host='127.0.0.1', port=16452)
def on_message(self, message):
#price = {'time':message['E'], 'price':message['k']['c']}
self.ticker_time.append(message['E'])
self.last_price.append(message['k']['c'])
#print(self.ticker_time, self.last_price)
def update_graph(self, n):
data = go.Scatter(
x = list(self.ticker_time),
y = list(self.last_price),
name ='Scatter',
mode = 'lines+markers'
)
return {'data': [data],
'layout': go.Layout(xaxis=dict(range=[min(self.ticker_time), max(self.ticker_time)]),
yaxis=dict(range=[min(self.last_price), max(self.last_price)]),)}
def Main():
api_key = ''
secret_key = ''
ticker = priceTicker(api_key, secret_key)
ticker.app.run_server(debug=True, host='127.0.0.1', port=16452)
if __name__ == '__main__':
#app.run_server(debug=False, host='127.0.0.1', port=10010)
Main()
Output is as expected with the app updating the last price every 2 sec.
Cheers

Related

Matplotlib, Tkinter, Serial Multiprocessing

I created a code (using Tkinter, Python3 and matplotlid) that could read data from different serial ports, save them to csv, then create graphs and finally preview data in GUI. The code was splited in two different scripts. The main script contained reading data, save data to csv an priview of data and the other script contained the graph creation.
Today I rewrote the code using the answer of #user2464430 here. The code is working, but I can't update the GUI. Opens once and then no refresh with new data.
The following code is a part of total code.
My code is:
from PIL import ImageTk, Image
import tkinter as Tk
import multiprocessing
from queue import Empty, Full
from time import strftime
import serial
import numpy as np
import matplotlib.pyplot as plt
from drawnow import *
from pylab import *
import pandas as pd
from datetime import timedelta
from datetime import datetime
import plotly.graph_objs as go
from plotly.subplots import make_subplots
import locale
import os
class GuiApp(object):
def __init__(self, image):
self.root = Tk.Tk()
self.root.resizable(width=False, height=False)
self.root.geometry("1600x800+0+0")
C = Canvas(self.root, bg="black", width=1600, height=800)
def BasicLabels():
....... # in this stage create multiple axis labels
Î¥AxisLabels()
BasicLabels()
def ValueLabels():
....... # Read and munipulate datas from CSV file and print in in labels
ValueLabels()
C.pack()
def GenerateData(q): #Read Serial Ports and store data to CSV file
file_exists = os.path.isfile("BigData.csv")
header = [["Daytime,T1"]]
if not file_exists:
with open("BigData.csv", "a+") as csvfile:
np.savetxt(csvfile, header, delimiter=",", fmt="%s", comments="")
while True:
try:
ser1 = serial.Serial(port="COM4", baudrate=9600)
read_ser1 = ser1.readline()
if read_ser1 == "":
read_ser1 = "Missing Value"
else:
read_ser1 = ser1.readline()
read_ser1 = str(read_ser1[0 : len(read_ser1)].decode("utf-8"))
# print("COM4:", read_ser1)
ser1.close()
except:
print("Failed 1")
read_ser1 = "9999,9999,9999,9999,9999"
daytime = strftime(" %d-%m-%Y %H:%M:%S")
rows = [
daytime
+ ","
+ read_ser1.strip()
]
with open("BigData.csv", "a+") as csvfile:
np.savetxt(csvfile, rows, delimiter=",", fmt="%s", comments="")
CreateGraphs()
def CreateGraphs():
#Code to generate graph. Called every time i have new line in CSV.
if __name__ == "__main__":
# Queue which will be used for storing Data
q = multiprocessing.Queue()
q.cancel_join_thread() # or else thread that puts data will not term
gui = GuiApp(q)
t1 = multiprocessing.Process(target=GenerateData, args=(q,))
t1.start()
gui.root.mainloop()
t1.join()
The graphs are generating after while True in GenerateData.
All datas for labels and graphs are coming from CSV file and not directly from serial port.
Is it possible to update GUI with latest datas from CSV and created graphs?
Thank for your time.

Different sessions show different streaming data with a single bokeh server, how to solve it?

I'm working on a simulated osilloscope where the server PC collects data and ultimately will publish the streaming plot online. Below is a working script that can do the job. However, when I open multiple browsers, the streaming plots exhibit different data. (Although they are using the same data source). The example 'ohlc' seems to have the same problem. So, what is the right way to do this? I'm considering to write data to a file, but that will bring some issues like file i/o delay and disk storage limitation etc. Thank you for any help.
from bokeh.server.server import Server
from bokeh.models import ColumnDataSource, Label
from bokeh.plotting import figure
from bokeh.layouts import column
import numpy as np
import datetime as dt
from functools import partial
import time
# this will be replaced with the real data collector in the end
def f_emitter(p=0.1):
v = np.random.rand()
return (dt.datetime.now(), 0. if v>p else v)
def make_document(doc, functions, labels):
def update():
for index, func in enumerate(functions):
data = func()
sources[index].stream(new_data=dict(time=[data[0]], data=[data[1]]), rollover=1000)
annotations[index].text = f'{data[1]: .3f}'
sources = [ColumnDataSource(dict(time=[], data=[])) for _ in range(len(functions))]
figs = []
annotations = []
for i in range(len(functions)):
figs.append(figure(x_axis_type='datetime',
y_axis_label=labels[i], toolbar_location=None,
active_drag=None, active_scroll=None))
figs[i].line(x='time', y='data', source=sources[i])
annotations.append(Label(x=10, y=10, text='', text_font_size='40px', text_color='black',
x_units='screen', y_units='screen', background_fill_color='white'))
figs[i].add_layout(annotations[i])
# print(figs[i].plot_height)
doc.add_root(column([fig for fig in figs], sizing_mode='stretch_both'))
doc.add_periodic_callback(callback=update, period_milliseconds=100)
if __name__ == '__main__':
# list of functions and labels to feed into the scope
functions = [f_emitter]
labels = ['emitter']
server = Server({'/': partial(make_document, functions=functions, labels=labels)})
server.start()
server.io_loop.add_callback(server.show, "/")
try:
server.io_loop.start()
except KeyboardInterrupt:
print('keyboard interruption')
When you connect with a new client, by default Bokeh creates a new session. Each session has its own document, so the data source end up not being the same.

Getting Interactive Brokers API into Pandas

New to Python and IB API and stuck on this simple thing. This application works correctly and prints IB server reply. However, I cannot figure out how to get this data into a panda's dataframe or any other variable for that matter. How do you "get the data out?" Thanks!
Nothing on forums, documentation or youtube that I can find with a useful example. I think the answer must be to return accountSummary to pd.Series, but no idea how.
Expected output would be a data series or variable that can be manipulated outside of the application.
from ibapi import wrapper
from ibapi.client import EClient
from ibapi.utils import iswrapper #just for decorator
from ibapi.common import *
import pandas as pd
class TestApp(wrapper.EWrapper, EClient):
def __init__(self):
wrapper.EWrapper.__init__(self)
EClient.__init__(self, wrapper=self)
#iswrapper
def nextValidId(self, orderId:int):
print("setting nextValidOrderId: %d", orderId)
self.nextValidOrderId = orderId
# here is where you start using api
self.reqAccountSummary(9002, "All", "$LEDGER")
#iswrapper
def error(self, reqId:TickerId, errorCode:int, errorString:str):
print("Error. Id: " , reqId, " Code: " , errorCode , " Msg: " , errorString)
#iswrapper
def accountSummary(self, reqId:int, account:str, tag:str, value:str, currency:str):
print("Acct Summary. ReqId:" , reqId , "Acct:", account,
"Tag: ", tag, "Value:", value, "Currency:", currency)
#IB API data returns here, how to pass it to a variable or pd.series
#iswrapper
def accountSummaryEnd(self, reqId:int):
print("AccountSummaryEnd. Req Id: ", reqId)
# now we can disconnect
self.disconnect()
def main():
app = TestApp()
app.connect("127.0.0.1", 4001, clientId=123)
test = app.accountSummary
app.run()
if __name__ == "__main__":
main()
Hi had the same problem and collections did it for me. Here is my code for CFDs data. Maybe it will help somebody. You will have your data in app.df. Any suggestion for improvement are more than welcome.
import collections
import datetime as dt
from threading import Timer
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
import pandas as pd
# get yesterday and put it to correct format yyyymmdd{space}{space}hh:mm:dd
yesterday = str(dt.datetime.today() - dt.timedelta(1))
yesterday = yesterday.replace('-','')
IP = '127.0.0.1'
PORT = 7497
class App(EClient, EWrapper):
def __init__(self):
super().__init__(self)
self.data = collections.defaultdict(list)
def error(self, reqId, errorCode, errorString):
print(f'Error {reqId}, {errorCode}, {errorString}')
def historicalData(self, reqId, bar):
self.data['date'].append(bar.date)
self.data['open'].append(bar.open)
self.data['high'].append(bar.high)
self.data['low'].append(bar.low)
self.data['close'].append(bar.close)
self.data['volume'].append(bar.volume)
self.df = pd.DataFrame.from_dict(self.data)
def stop(self):
self.done = True
self.disconnect()
# create App object
app = App()
print('App created...')
app.connect(IP, PORT, 0)
print('App connected...')
# create contract
contract = Contract()
contract.symbol = 'IBDE30'
contract.secType = 'CFD'
contract.exchange = 'SMART'
contract.currency = 'EUR'
print('Contract created...')
# request historical data for contract
app.reqHistoricalData(reqId=1,
contract=contract,
endDateTime=yesterday,
durationStr='1 W',
barSizeSetting='15 mins',
whatToShow='ASK',
useRTH=0,
formatDate=1,
keepUpToDate=False,
chartOptions=[])
Timer(4, app.stop).start()
app.run()
I'd store the data to a dictionary, create a dataframe from the dictionary, and append the new dataframe to the main dataframe using the concat function. Here's an example:
def accountSummary(self, reqId:int, account:str, tag:str, value:str, currency:str):
acct_dict = {"account": account, "value": value, "currency": currency}
acct_df = pd.DataFrame([acct_dict], columns=acct_dict.keys())
main_df = pd.concat([main_df, acct_df], axis=0).reset_index()
For more information, you might like Algorithmic Trading with Interactive Brokers

Why is multiprocessing not working with python dash framework - Python3.6

I'm trying to implement multiprocessing library for splitting up a dataframe into parts, process it on multiple cores of CPU and then concatenate the results back into a final dataframe in a python dash application. The code works fine when I try it outside of the dash application (when I run the code standalone without enclosing it in a dash application). But when I enclose the same code in a dash application, I get an error. I have shown the code below:
I have tried the multiprocessing code out of the dash framework and it works absolutely fine.
import dash
from dash.dependencies import Input, Output, State
import dash_core_components as dcc
import dash_html_components as html
import flask
import dash_table_experiments as dt
import dash_table
import dash.dependencies
import base64
import time
import os
import pandas as pd
from docx import *
from docx.text.paragraph import Paragraph
from docx.text.paragraph import Run
import xml.etree.ElementTree as ET
import multiprocessing as mp
from multiprocessing import Pool
from docx.document import Document as doctwo
from docx.oxml.table import CT_Tbl
from docx.oxml.text.paragraph import CT_P
from docx.table import _Cell, Table
from docx.text.paragraph import Paragraph
import io
import csv
import codecs
import numpy as np
app = dash.Dash(__name__)
application = app.server
app.config.supress_callback_exceptions = True
app.layout = html.Div(children=[
html.Div([
html.Div([
html.H4(children='Reader'),
html.Br(),
],style={'text-align':'center'}),
html.Br(),
html.Br(),
html.Div([
dcc.Upload(html.Button('Upload File'),id='upload-data',style = dict(display = 'inline-block')),
html.Br(),
]
),
html.Div(id='output-data-upload'),
])
])
#app.callback(Output('output-data-upload', 'children'),
[Input('upload-data', 'contents')],
[State('upload-data', 'filename')])
def update_output(contents, filename):
if contents is not None:
content_type, content_string = contents.split(',')
decoded = base64.b64decode(content_string)
document = Document(io.BytesIO(decoded))
combined_df = pd.read_csv('combined_df.csv')
def calc_tfidf(input1):
input1 = input1.reset_index(drop=True)
input1['samplecol'] = 'sample'
return input1
num_cores = mp.cpu_count() - 1 #number of cores on your machine
num_partitions = mp.cpu_count() - 1 #number of partitions to split dataframe
df_split = np.array_split(combined_df, num_partitions)
pool = Pool(num_cores)
df = pd.concat(pool.map(calc_tfidf, df_split))
pool.close()
pool.join()
return len(combined_df)
else:
return 'No File uploaded'
app.css.append_css({'external_url': 'https://codepen.io/plotly/pen/EQZeaW.css'})
if __name__ == '__main__':
app.run_server(debug=True)
The above dash application takes as input any file. Upon uploading the file in the front end, a local CSV file (any file, in my case it is combined_df.csv) is loaded into a dataframe. Now I want to split the dataframe into parts using multiprocessing, process it and combine it back. But the above code results in the following error:
AttributeError: Can't pickle local object 'update_output..calc_tfidf'
What's wrong with this piece of code?
Okay I've figured it out now!. The problem is that the function calc_tfidf was not defined as a global function. I changed the function to be a global function and it worked perfect.
Simple checks when left unsolved at times might lead to days of redundant efforts! :(

I am unable to simultaneously stream the feed from my RBP3's camera and record to a file at the same time using python

I know how to save to a file using the code below (and timestamp the feed) and I know how to stream using uv4l but I am simply too bad to do it simultaneously.
import time
time.sleep(60)
import picamera
import datetime as dt
camera = picamera.PiCamera()
camera.resolution = (640, 480)
#camera.vflip = True
camera.led = False
x = 0
while True:
bideoname = "/media/pi/cam/" + dt.datetime.now().strftime('%Y-%m-%d-%H') + ".h264"
camera.annotate_background = picamera.Color('black')
camera.annotate_text = dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
#camera.start_preview()
camera.start_recording(bideoname)
start = dt.datetime.now()
while (dt.datetime.now() - start).seconds < 3600:
camera.annotate_text = dt.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
camera.wait_recording(0.2)
camera.stop_recording()
x = x+1
I imagine I would use flask to to create local website to stream the feed to.
I have looked up and down the internets and this example seems to be the closest solution by Dave Jones but I don't if socket can communicate with a browser:
https://raspberrypi.stackexchange.com/questions/27041/record-and-stream-video-from-camera-simultaneously
Also there is this code which streams the camera feed to a page but no mention of how to simultanously record as well:
from flask import Flask, render_template, Response
# Raspberry Pi camera module (requires picamera package, developed by Miguel Grinberg)
from camera_pi import Camera
app = Flask(__name__)
#app.route('/')
def index():
"""Video streaming home page."""
return render_template('index.html')
def gen(camera):
"""Video streaming generator function."""
while True:
frame = camera.get_frame()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
#app.route('/video_feed')
def video_feed():
"""Video streaming route. Put this in the src attribute of an img tag."""
return Response(gen(Camera()),
mimetype='multipart/x-mixed-replace; boundary=frame')
if __name__ == '__main__':
app.run(host='0.0.0.0', port =80, debug=True, threaded=True)
Or maybe this is all wrong and there is simpler solution to this?
Thanks for the help.

Resources