I am trying to post data to arangodb using grpc and using script below but while running the client getting above error , My server is running in a good way,unable to figure out where is the error
from inspect import trace
import re
from urllib import response
import grpc
import os
import first_pb2_grpc as pb2_grpc
import first_pb2 as pb2
import json
import grpc
from typing import Dict, List
from google.protobuf import json_format
from first_pb2_grpc import*
import traceback
test_data_file_name = "data.json"
curr_dir = os.path.dirname(os.path.realpath(__file__))
test_data_file = os.path.join(curr_dir,test_data_file_name)
def read_json_file(file):
with open(file, encoding="utf8") as f:
data = json.load(f)
return data
test_data = read_json_file(test_data_file)
channel = grpc.insecure_channel('localhost:31024')
stub = pb2_grpc.UnaryStub(channel)
def test(request):
try:
response = stub.GetServerResponse(request)
print(response , 'AAAAA')
return response
except Exception as e:
return str(e)
def test_add_name(message):
try:
request = pb2.Message(
message=message
)
test_response = test(request)
return test_response
except Exception as e:
traceback.print_exc()
return str(e)
if __name__ == "__main__":
message = test_data["message"]
#attribute_val = json_format.Parse(json.dumps(name) , message='hi')
api_response = test_add_name(message)
print(api_response)
Please tell me how can i resolve this
New to Python and IB API and stuck on this simple thing. This application works correctly and prints IB server reply. However, I cannot figure out how to get this data into a panda's dataframe or any other variable for that matter. How do you "get the data out?" Thanks!
Nothing on forums, documentation or youtube that I can find with a useful example. I think the answer must be to return accountSummary to pd.Series, but no idea how.
Expected output would be a data series or variable that can be manipulated outside of the application.
from ibapi import wrapper
from ibapi.client import EClient
from ibapi.utils import iswrapper #just for decorator
from ibapi.common import *
import pandas as pd
class TestApp(wrapper.EWrapper, EClient):
def __init__(self):
wrapper.EWrapper.__init__(self)
EClient.__init__(self, wrapper=self)
#iswrapper
def nextValidId(self, orderId:int):
print("setting nextValidOrderId: %d", orderId)
self.nextValidOrderId = orderId
# here is where you start using api
self.reqAccountSummary(9002, "All", "$LEDGER")
#iswrapper
def error(self, reqId:TickerId, errorCode:int, errorString:str):
print("Error. Id: " , reqId, " Code: " , errorCode , " Msg: " , errorString)
#iswrapper
def accountSummary(self, reqId:int, account:str, tag:str, value:str, currency:str):
print("Acct Summary. ReqId:" , reqId , "Acct:", account,
"Tag: ", tag, "Value:", value, "Currency:", currency)
#IB API data returns here, how to pass it to a variable or pd.series
#iswrapper
def accountSummaryEnd(self, reqId:int):
print("AccountSummaryEnd. Req Id: ", reqId)
# now we can disconnect
self.disconnect()
def main():
app = TestApp()
app.connect("127.0.0.1", 4001, clientId=123)
test = app.accountSummary
app.run()
if __name__ == "__main__":
main()
Hi had the same problem and collections did it for me. Here is my code for CFDs data. Maybe it will help somebody. You will have your data in app.df. Any suggestion for improvement are more than welcome.
import collections
import datetime as dt
from threading import Timer
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
import pandas as pd
# get yesterday and put it to correct format yyyymmdd{space}{space}hh:mm:dd
yesterday = str(dt.datetime.today() - dt.timedelta(1))
yesterday = yesterday.replace('-','')
IP = '127.0.0.1'
PORT = 7497
class App(EClient, EWrapper):
def __init__(self):
super().__init__(self)
self.data = collections.defaultdict(list)
def error(self, reqId, errorCode, errorString):
print(f'Error {reqId}, {errorCode}, {errorString}')
def historicalData(self, reqId, bar):
self.data['date'].append(bar.date)
self.data['open'].append(bar.open)
self.data['high'].append(bar.high)
self.data['low'].append(bar.low)
self.data['close'].append(bar.close)
self.data['volume'].append(bar.volume)
self.df = pd.DataFrame.from_dict(self.data)
def stop(self):
self.done = True
self.disconnect()
# create App object
app = App()
print('App created...')
app.connect(IP, PORT, 0)
print('App connected...')
# create contract
contract = Contract()
contract.symbol = 'IBDE30'
contract.secType = 'CFD'
contract.exchange = 'SMART'
contract.currency = 'EUR'
print('Contract created...')
# request historical data for contract
app.reqHistoricalData(reqId=1,
contract=contract,
endDateTime=yesterday,
durationStr='1 W',
barSizeSetting='15 mins',
whatToShow='ASK',
useRTH=0,
formatDate=1,
keepUpToDate=False,
chartOptions=[])
Timer(4, app.stop).start()
app.run()
I'd store the data to a dictionary, create a dataframe from the dictionary, and append the new dataframe to the main dataframe using the concat function. Here's an example:
def accountSummary(self, reqId:int, account:str, tag:str, value:str, currency:str):
acct_dict = {"account": account, "value": value, "currency": currency}
acct_df = pd.DataFrame([acct_dict], columns=acct_dict.keys())
main_df = pd.concat([main_df, acct_df], axis=0).reset_index()
For more information, you might like Algorithmic Trading with Interactive Brokers
I am trying to understand multiprocessing by a simple task of multiprocessing using pathos module but I am getting following error
AttributeError: Can't get attribute 'SomeClass' on mp_main' from
'path\to\packageConstructor.py'
Here is my full code for review:
from random import randint
import os
import pathos
mp = pathos.helpers.mp
prc = pathos.helpers.mp.Process
class SomeClass:
def m1(self):
self.objAttr = randint(20000,40000)
self.selfID = id(self)
self.m2()
def m2(self):
print(os.getpid(), self.objAttr,self.selfID)
def checkMultiprocessing(self):
for c in range(10):
exec(f"p{c} = prc(target=self.m1)")
exec(f"p{c}.start()")
for c in range(10):
exec(f"p{c}.join()")
if __name__ == "__main__":
mp.freeze_support()
SomeClass().checkMultiprocessing()
What am I missing?
I have some problem with unpickling data recived from logger. Given udp_server:
import pickle
import logging
import logging.handlers
import socketserver
import struct
class MyUDPHandler(socketserver.BaseRequestHandler):
def handle(self):
data = self.request[0].strip()
socket = self.request[1]
print("{} wrote:".format(self.client_address[0]))
print(self.unPickle(data)) # here is problem
socket.sendto(data.upper(), self.client_address)
def unPickle(self, data):
return pickle.loads(data)
class LogRecordSocketReceiver(socketserver.UDPServer):
allow_reuse_address = True
def __init__(self, host='localhost',
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
handler=MyUDPHandler):
socketserver.UDPServer.__init__(self, (host, port), handler)
self.abort = 0
self.timeout = 1
self.logname = None
def serve_until_stopped(self):
import select
abort = 0
while not abort:
rd, wr, ex = select.select([self.socket.fileno()],
[], [],
self.timeout)
if rd:
self.handle_request()
abort = self.abort
if __name__ == "__main__":
tcpserver = LogRecordSocketReceiver()
print('About to start UDP server...')
tcpserver.serve_until_stopped()
And udp_log_sender:
import logging, logging.handlers
rootLogger = logging.getLogger('')
rootLogger.setLevel(logging.DEBUG)
udp_handler = logging.handlers.DatagramHandler("localhost", logging.handlers.DEFAULT_TCP_LOGGING_PORT)
rootLogger.addHandler(udp_handler)
logging.info('Jackdaws love my big sphinx of quartz.')
When the server recives logging message and want to run unPickle the EOFError is thrown. What could cause such behaviour?
do not strip binary data
omit the first 4 bytes (i.e. use data[4:]) as they contain length of the dumped object
I didn't find this information in logging module documentation - sometimes one has go to the source (or just google harder).
After a lot of investigating, I found out that after serving hundreds of thousands of HTTP POST requests, there's a memory leak. The strange part is that the memory leak only occurs when using PyPy.
Here's an example code:
from twisted.internet import reactor
import tornado.ioloop
do_tornado = False
port = 8888
if do_tornado:
from tornado.web import RequestHandler, Application
else:
from cyclone.web import RequestHandler, Application
class MainHandler(RequestHandler):
def get(self):
self.write("Hello, world")
def post(self):
self.write("Hello, world")
if __name__ == "__main__":
routes = [(r"/", MainHandler)]
application = Application(routes)
print port
if do_tornado:
application.listen(port)
tornado.ioloop.IOLoop.instance().start()
else:
reactor.listenTCP(port, application)
reactor.run()
Here is the test code I am using to generate requests:
from twisted.internet import reactor, defer
from twisted.internet.task import LoopingCall
from twisted.web.client import Agent, HTTPConnectionPool
from twisted.web.iweb import IBodyProducer
from zope.interface import implements
pool = HTTPConnectionPool(reactor, persistent=True)
pool.retryAutomatically = False
pool.maxPersistentPerHost = 10
agent = Agent(reactor, pool=pool)
bid_url = 'http://localhost:8888'
class StringProducer(object):
implements(IBodyProducer)
def __init__(self, body):
self.body = body
self.length = len(body)
def startProducing(self, consumer):
consumer.write(self.body)
return defer.succeed(None)
def pauseProducing(self):
pass
def stopProducing(self):
pass
def callback(a):
pass
def error_callback(error):
pass
def loop():
d = agent.request('POST', bid_url, None, StringProducer("Hello, world"))
#d = agent.request('GET', bid_url)
d.addCallback(callback).addErrback(error_callback)
def main():
exchange = LoopingCall(loop)
exchange.start(0.02)
#log.startLogging(sys.stdout)
reactor.run()
main()
Note that this code does not leak with CPython nor with Tornado and Pypy! The code leaks only when using Twisted and Pypy together, and ONLY when using a POST request.
To see the leak, you have to send hundreds of thousands of requests.
Note that when setting PYPY_GC_MAX, the process eventually crashes.
What's going on?
Turns out that the cause of the leak is the BytesIO module.
Here's how to simulate the leak on Pypy.
from io import BytesIO
while True: a = BytesIO()
Here's the fix:
https://bitbucket.org/pypy/pypy/commits/40fa4f3a0740e3aac77862fe8a853259c07cb00b