Python threading causing issues with google api - python-3.x

I'm running through a list of locations and trying to find places along my route. This is my first attempt at threading, so any tips would be appreciated! When i run this it'll work fine for the first few iterations, but then i start getting a KeyError and the API response says route is not found (even though it should be). If I search along a shorter route, everything runs fine. When I extend the route past a couple of hours of drive time I start getting these errors. Is it possible that I'm overloading it or does my code look off?
import pandas as pd
from threading import Thread
import threading
import requests
start_input = input("start: ")
end_input = input("end: ")
out_way = input("out of the way: ")
out_way_secs = int(out_way) * 60
thread_local = threading.local()
def get_session():
if not getattr(thread_local, "session", None):
thread_local.session = requests.Session()
return thread_local.session
def get_routes(url, start, end, waypoint, idx):
session = get_session()
with session.get(url, params={'origins': f'{start}|{waypoint}', 'destinations': f'{start}|{end}',
'key': '# key'}) as response:
route = response.json()
if route['rows'][1]['elements'][0]['status'] != 'OK':
results[idx] = {'# info'}
else:
nonstop_route = route['rows'][0]['elements'][1]['duration']['value']
leg1 = route['rows'][1]['elements'][0]['duration']['value']
leg2 = route['rows'][1]['elements'][1]['duration']['value']
time_added = (leg1 + leg2) - nonstop_route
time_added_mins = str(datetime.timedelta(seconds=(leg1 + leg2) - nonstop_route))
more_time = time_added_mins.split(':')
added_time_str = str(f'{more_time[0]}:{more_time[1]}:{more_time[2]} away!')
if time_added < allowable_time:
results[idx] = {# info to return}
return results[idx]
if __name__ == "__main__":
start_time = time.time()
output_df = pd.DataFrame(columns=['Location', 'Added Time', 'Notes'])
threads = [None] * coords[0]
results = [None] * coords[0]
for i in range(len(threads)):
threads[i] = Thread(target=get_routes, args=('https://maps.googleapis.com/maps/api/distancematrix/json',
start_input, end_input, stops[i], i))
threads[i].start()
for i in range(len(threads)):
threads[i].join()
for x in range(len(results)):
output_df = output_df.append(results[x], ignore_index=True)
output_df = output_df.sort_values(['Added Time'], ascending=True)
output_df.to_csv('output.csv', index=False)
there are 3 errors that it will get, this first one pops up by itself and the last 2 will come together. The code is the same when I run it, so not sure why i'm getting different errors.
This is the most common error that comes by itself (the routing duration works fine when run individually):
Exception in thread Thread-171:
Traceback (most recent call last):
File "C:\Python37-32\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "C:\Python37-32\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "C:program.py", line 46, in get_routes
nonstop_route = route['rows'][0]['elements'][1]['duration']['value']
KeyError: 'duration'
The two below I get together and are less common:
Exception in thread Thread-436:
Traceback (most recent call last):
File "C:\Python37-32\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "C:\Python37-32\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "C:/program.py", line 40, in get_routes
route = response.json()
File "C:\requests\models.py", line 897, in json
return complexjson.loads(self.text, **kwargs)
File "C:\Python37-32\lib\json\__init__.py", line 348, in loads
return _default_decoder.decode(s)
File "C:\Python37-32\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Python37-32\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
second error:
Exception in thread Thread-196:
Traceback (most recent call last):
File "C:\site-packages\urllib3\response.py", line 360, in _error_catcher
yield
File "C:\urllib3\response.py", line 442, in read
data = self._fp.read(amt)
File "C:\Python37-32\lib\http\client.py", line 447, in read
n = self.readinto(b)
File "C:\Python37-32\lib\http\client.py", line 491, in readinto
n = self.fp.readinto(b)
File "C:\Python37-32\lib\socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "C:\Python37-32\lib\ssl.py", line 1052, in recv_into
return self.read(nbytes, buffer)
File "C:\Python37-32\lib\ssl.py", line 911, in read
return self._sslobj.read(len, buffer)
ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\site-packages\requests\models.py", line 750, in generate
for chunk in self.raw.stream(chunk_size, decode_content=True):
File "C:\site-packages\urllib3\response.py", line 494, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "C:\site-packages\urllib3\response.py", line 459, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "C:\Python37-32\lib\contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "C:\site-packages\urllib3\response.py", line 378, in _error_catcher
raise ProtocolError('Connection broken: %r' % e, e)
urllib3.exceptions.ProtocolError: ("Connection broken: ConnectionAbortedError(10053, 'An established connection was aborted by the software in your host machine', None, 10053, None)", ConnectionAbortedError(10053, 'An established connection was aborted by the software in your host machine', None, 10053, None))

Related

Tornado simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

I have a TornadoAPI for keras model, is it possible to check concurrency using python multiprocessing module, I tried with below code, but it throws error
from multiprocessing import Pool
import requests, json
url = 'http://localhost:8888/index/predict'
payload = { "colA":"some1", "colB":"some2",...….)
pl = json.dumps(payload)
def callAPI(x):
session = requests.Session()
r = session.post(url, json=json.loads(pl))
response = r.json()
return response
if __name__ == '__main__':
Pool(processes=15).map(callAPI, range(5))
Error
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "C:\Models\CallThreadsCheck.py", line 40, in callAPI
response = r.json()
File "C:\Users\HZ\Anaconda3\lib\site-packages\requests\models.py", line 897, in json
return complexjson.loads(self.text, **kwargs)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\__init__.py", line 525, in loads
return _default_decoder.decode(s)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "C:\Users\HZ\Anaconda3\lib\site-packages\simplejson\decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "CallThreadsCheck.py", line 49, in <module>
p.map(callAPI, range(calls))
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\HZ\Anaconda3\lib\multiprocessing\pool.py", line 657, in get
raise self._value
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
May I know what's this error about pls.

Python3 multiprocessing shared dictionary consume by all process

I'm beginner for multiprocessing,
i would like to use multiprocessing for parallel code running with streaming data.
To start good, I have coded below and got error.
Could you please tell me correct way to print on the screen.
Code:
import sys
from multiprocessing import Process, Manager
import time
def producer(dic, name):
for i in range(10000):
dic["A"] = i
time.sleep(2)
def consumer(dic, name):
for i in range(10000):
aval = dic.get("A")
#print(f" {name} - Val = {aval}")
sys.stdout.write(f" {name} - Val = {aval}")
sys.stdout.flush()
time.sleep(2.2)
if __name__ == '__main__':
manager = Manager()
dic = manager.dict()
Process(target=producer, args=(dic,"TT")).start()
time.sleep(1)
Process(target=consumer, args=(dic,"Con1")).start()
Process(target=consumer, args=(dic,"Con2")).start()
When I run the same in the windows console, I got below error, how can I print Consumer's print function in the console.Thanks
(base) PS D:\> python .\mulpro.py
Process Process-3:
Process Process-4:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 811, in
_callmethod
conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 297, in _
bootstrap
self.run()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 99, in ru
n
self._target(*self._args, **self._kwargs)
File "D:\mulpro.py", line 19, in consumer
aval = dic.get("A")
File "<string>", line 2, in get
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 815, in
_callmethod
self._connect()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 802, in
_connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 490, i
n Client
c = PipeClient(address)
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 691, i
n PipeClient
_winapi.WaitNamedPipe(address, 1000)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 811, in
_callmethod
conn = self._tls.connection
FileNotFoundError: [WinError 2] The system cannot find the file specified
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 297, in _
bootstrap
self.run()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 99, in ru
n
self._target(*self._args, **self._kwargs)
File "D:\mulpro.py", line 19, in consumer
aval = dic.get("A")
File "<string>", line 2, in get
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 815, in
_callmethod
self._connect()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 802, in
_connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 490, i
n Client
c = PipeClient(address)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 691, i
n PipeClient
_winapi.WaitNamedPipe(address, 1000)
FileNotFoundError: [WinError 2] The system cannot find the file specified
Process Process-2:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 297, in _
bootstrap
self.run()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 99, in ru
n
self._target(*self._args, **self._kwargs)
File "D:\mulpro.py", line 13, in producer
dic["A"] = i
File "<string>", line 2, in __setitem__
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 818, in
_callmethod
conn.send((self._id, methodname, args, kwds))
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 206, i
n send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 280, i
n _send_bytes
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
BrokenPipeError: [WinError 232] The pipe is being closed
The reason might be that the main process doesn't wait for two children process, so make your code like this could work:
def run():
manager = Manager()
dic = manager.dict()
Process(target=producer, args=(dic,"TT")).start()
time.sleep(1)
Process(target=consumer, args=(dic,"Con1")).start()
Process(target=consumer, args=(dic,"Con2")).start()
while True:
pass
if __name__ == '__main__':
run()
However it's very strange that if I append a dead loop in main instead of using another function, it still raise that exception. Anyway, the code above could help.

SyntaxError raised when using joblib with lxml on python 3.5

I am trying to parallelize the tasks of correcting texts on many documents with Python, so I naturally found "joblib". I want each task to be to correct a given document. Here is the structure of the code:
if __name__ == '__main__':
lexicon = build_compact_lexicon()
from joblib import Parallel, delayed
import multiprocessing
num_cores = multiprocessing.cpu_count()
results = Parallel(n_jobs=num_cores)(delayed(find_errors)('GDL', i, 1, lexicon) for i in range(1798, 1820))
I am using the function find_errors summed up here :
def find_errors(newspaper, year, month, lexicon):
# parse the input newspaper text data using etree parser from LXML
# detect errors in the text
return found_errors_type1, found_errors_type2, found_errors_type3
This does raise me a few errors
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 130, in __call__
return self.func(*args, **kwargs)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 72, in __call__
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 72, in <listcomp>
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "hellowordParallel.py", line 85, in find_errors
tree = etree.parse(xml_file_path)
File "src/lxml/lxml.etree.pyx", line 3427, in lxml.etree.parse (src/lxml/lxml.etree.c:79801)
File "src/lxml/parser.pxi", line 1805, in lxml.etree._parseDocument (src/lxml/lxml.etree.c:116293)
TypeError: cannot parse from 'NoneType'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 392, in find_cookie
line_string = line.decode('utf-8')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 24: invalid start byte
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/multiprocessing/pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 139, in __call__
tb_offset=1)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/format_stack.py", line 373, in format_exc
frames = format_records(records)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/format_stack.py", line 274, in format_records
for token in generate_tokens(linereader):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 514, in _tokenize
line = readline()
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/format_stack.py", line 265, in linereader
line = getline(file, lnum[0])
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/linecache.py", line 16, in getline
lines = getlines(filename, module_globals)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/linecache.py", line 47, in getlines
return updatecache(filename, module_globals)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/linecache.py", line 136, in updatecache
with tokenize.open(fullname) as fp:
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 456, in open
encoding, lines = detect_encoding(buffer.readline)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 433, in detect_encoding
encoding = find_cookie(first)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 397, in find_cookie
raise SyntaxError(msg)
File "<string>", line None
SyntaxError: invalid or missing encoding declaration for '/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/lxml/etree.so'
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "hellowordParallel.py", line 160, in <module>
results = Parallel(n_jobs=num_cores)(delayed(find_errors)('GDL', i, 1, lexicon) for i in range(1798, 1820))
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 810, in __call__
self.retrieve()
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 727, in retrieve
self._output.extend(job.get())
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/multiprocessing/pool.py", line 608, in get
raise self._value
SyntaxError: invalid or missing encoding declaration for '/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/lxml/etree.so'
I don't understand if this is due do something related with configs or if my function doesn't fit in a parallel implementation... (I guess it should...)
Did it happen to some of you before?
Hope my question is clear and there is enough information for someone to give me some help!

python3 sqlalchemy pymysql connect string

using python3, I can connect to mysql using pymysql. all works as expected. enclosed code works.
import pymysql
conn = pymysql.connect(host='127.0.0.1', unix_socket='/home/jhgong/mysql/tmp/mysql.sock', user='root', passwd='my_pass', db='my_db', port='3333')
cur = conn.cursor()
cur.execute('select user from auth_users')
for i in cur:
print(i)
trying to get sqlalchemy to connect with pymysql, the default example strings don't seem to work. the above example does not work unless I declare both the port number and a unix_socket.
below is what I've been using to try and get sqlalchemy to connect. I assume that the socket and port number are both needed. I used connect_args to feed in a hash with the extra unix_socket location. no joy.
enclosed the the snippit I've been using that creates the error.
conarg = {
'unix_socket':'/home/jhgong/mysql/tmp/mysql.sock',
'db' :'ice'
}
engine = create_engine('mysql+pymysql://root:my_pass#127.0.0.1:3333/my_db', connect_args = conarg, echo=True)
connection = engine.connect()
with or without the db in conarg hash i get the following error:
>>> connection = engine.connect()
2013-01-17 13:04:20,819 INFO sqlalchemy.engine.base.Engine b'SELECT DATABASE()'
2013-01-17 13:04:20,819 INFO sqlalchemy.engine.base.Engine ()
Traceback (most recent call last):
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 724, in _do_get
return self._pool.get(wait, self._timeout)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/util/queue.py", line 163, in get
raise Empty
sqlalchemy.util.queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 1574, in connect
return self._connection_cls(self, **kwargs)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 58, in __init__
self.__connection = connection or engine.raw_connection()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 1637, in raw_connection
return self.pool.unique_connection()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 182, in unique_connection
return _ConnectionFairy(self).checkout()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 398, in __init__
rec = self._connection_record = pool._do_get()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 744, in _do_get
con = self._create_connection()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 187, in _create_connection
return _ConnectionRecord(self)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 284, in __init__
exec_once(self.connection, self)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/event.py", line 362, in exec_once
self(*args, **kw)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/event.py", line 379, in __call__
fn(*args, **kw)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/strategies.py", line 168, in first_connect
dialect.initialize(c)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/dialects/mysql/base.py", line 2005, in initialize
default.DefaultDialect.initialize(self, connection)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/default.py", line 183, in initialize
self._get_default_schema_name(connection)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/dialects/mysql/base.py", line 1970, in _get_default_schema_name
return connection.execute('SELECT DATABASE()').scalar()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 645, in execute
params)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 791, in _execute_text
statement, parameters
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 854, in _execute_context
context)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/default.py", line 342, in do_execute
cursor.execute(statement, parameters)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/PyMySQL3-0.5-py3.2.egg/pymysql/cursors.py", line 105, in execute
query = query % escaped_args
TypeError: unsupported operand type(s) for %: 'bytes' and 'tuple'
it appears to be raising an error on empty pool queue. setting the size or type of pool queue has no effect.
any suggestions on how to figure this out?
Try this very simple example:
import sqlalchemy
from sqlalchemy.sql import select
from sqlalchemy import Table, MetaData
def init():
try:
server = 'xx'
db = 'xx'
login = 'xx'
passwd = 'xx'
engine_str = 'mysql+mysqlconnector://{}:{}#{}/{}'.format(login, passwd, server, db)
engine = sqlalchemy.create_engine(engine_str, echo=False, encoding='utf-8')
connection = engine.connect()
metadata = MetaData()
t_servers = Table('your_table_here', metadata, autoload=True, autoload_with=engine)
s = select([t_servers])
result = connection.execute(s)
for row in result:
print(row['the_field'])
except Exception:
raise
finally:
connection.close()
if __name__ == '__main__':
init()
Mysql connector for Python 3 : download here
I know this is late but the PyMySQL requirements mention CPython >= 2.6 or >= 3.3, it looks like you're using CPython 3.2 (the default version of python used), and that may be your problem.

sending chunks of data using webob

I tried to write a simple server-client program using webob.
1. The client send data using 'Transfer-Encoding', 'chunked'
2. the received data is then print in the server side.
The Server.py received the data correctly.
However, I got error a bunch of message from webob.
However, anyone kindly tell me what had happened or just
give a simple guide line for write such a simple program(sending chunk) with webob.
thanks!
the codes and error are as below:
server.py
from webob import Request
from webob import Response
class ChunkApp(object):
def __init__(self):
pass
def __call__(self, environ, start_response):
req = Request(environ)
for buf in self.chunkreadable(req.body_file, 65535):
print buf
resp = Response('chunk received')
return resp(environ, start_response)
def chunkreadable(self, iter, chunk_size=65536):
return self.chunkiter(iter, chunk_size) if \
hasattr(iter, 'read') else iter
def chunkiter(self, fp, chunk_size=65536):
while True:
chunk = fp.read(chunk_size)
if chunk:
yield chunk
else:
break
if __name__ == '__main__':
app = ChunkApp()
from wsgiref.simple_server import make_server
httpd = make_server('localhost', 8080, app)
print 'Serving on http://localhost:%s' % '8080'
try:
httpd.serve_forever()
except KeyboardInterrupt:
print '^C'
client.py
from httplib import HTTPConnection
conn = HTTPConnection("localhost:8080")
conn.putrequest('PUT', "/")
body = "1234567890"
conn.putheader('Transfer-Encoding', 'chunked')
conn.endheaders()
for chunk in body:
#print '%x\r\n%s\r\n' % (len(chunk), chunk)
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
conn.send('0\r\n\r\n')
Error
Traceback (most recent call last):
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 86, in run
self.finish_response()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 127, in finish_response
self.write(data)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 210, in write
self.send_headers()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 268, in send_headers
self.send_preamble()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 192, in send_preamble
'Date: %s\r\n' % format_date_time(time.time())
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 324, in write
self.flush()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 303, in flush
self._sock.sendall(view[write_offset:write_offset+buffer_size])
error: [Errno 32] Broken pipe
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 52575)
Traceback (most recent call last):
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 284, in _handle_request_noblock
self.process_request(request, client_address)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 310, in process_request
self.finish_request(request, client_address)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 323, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 641, in __init__
self.finish()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 694, in finish
self.wfile.flush()
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 303, in flush
self._sock.sendall(view[write_offset:write_offset+buffer_size])
error: [Errno 32] Broken pipe

Resources