The code below worked when using Python 2.7, but raises a StatementError when using Python 3.5. I haven't found a good explanation for this online yet.
Why doesn't sqlalchemy accept simple Python 3 string objects in this situation? Is there a better way to insert rows into a table?
from sqlalchemy import Table, MetaData, create_engine
import json
def add_site(site_id):
engine = create_engine('mysql+pymysql://root:password#localhost/database_name', encoding='utf8', convert_unicode=True)
metadata = MetaData()
conn = engine.connect()
table_name = Table('table_name', metadata, autoload=True, autoload_with=engine)
site_name = 'Buffalo, NY'
p_profile = {"0": 300, "1": 500, "2": 100}
conn.execute(table_name.insert().values(updated=True,
site_name=site_name,
site_id=site_id,
p_profile=json.dumps(p_profile)))
add_site(121)
EDIT The table was previously created with this function:
def create_table():
engine = create_engine('mysql+pymysql://root:password#localhost/database_name')
metadata = MetaData()
# Create table for updating sites.
table_name = Table('table_name', metadata,
Column('id', Integer, Sequence('user_id_seq'), primary_key=True),
Column('updated', Boolean),
Column('site_name', BLOB),
Column('site_id', SMALLINT),
Column('p_profile', BLOB))
metadata.create_all(engine)
EDIT Full error:
>>> scd.add_site(121)
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1073, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 610, in _init_compiled
for key in compiled_params
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 610, in <genexpr>
for key in compiled_params
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/sqltypes.py", line 834, in process
return DBAPIBinary(value)
File "/usr/local/lib/python3.5/dist-packages/pymysql/__init__.py", line 79, in Binary
return bytes(x)
TypeError: string argument without an encoding
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/user1/Desktop/server_algorithm/database_tools.py", line 194, in add_site
failed_acks=json.dumps(p_profile)))
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1078, in _execute_context
None, None)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1341, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/util/compat.py", line 185, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/base.py", line 1073, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 610, in _init_compiled
for key in compiled_params
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/engine/default.py", line 610, in <genexpr>
for key in compiled_params
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/sqltypes.py", line 834, in process
return DBAPIBinary(value)
File "/usr/local/lib/python3.5/dist-packages/pymysql/__init__.py", line 79, in Binary
return bytes(x)
sqlalchemy.exc.StatementError: (builtins.TypeError) string argument without an encoding [SQL: 'INSERT INTO table_name (updated, site_name, site_id, p_profile) VALUES (%(updated)s, %(site_name)s, %(site_id)s, %(p_profile)s)']
As univerio mentioned, the solution was to encode the string as follows:
conn.execute(table_name.insert().values(updated=True,
site_name=site_name,
site_id=site_id,
p_profile=bytes(json.dumps(p_profile), 'utf8')))
BLOBs require binary data, so we need bytes in Python 3 and str in Python 2, since Python 2 strings are sequences of bytes.
If we want to use Python 3 str, we need to use TEXT instead of BLOB.
You simply just need to convert your string to a byte string ex:
site_name=str.encode(site_name),
site_id=site_id,
p_profile=json.dumps(p_profile)))```
or
```site_name = b'Buffalo, NY'```
Related
I need to fetch data from google cloud datastore using python, when fetching all the collection it succeed, but when try to fetch specific value according to the key it failed, I provide the store data method and fetch data method:
def store_data(name, value):
entity = datastore.Entity(key=datastore_client.key('visit', name))
entity.update({
'name': name,
'value': value
})
datastore_client.put(entity)
def fetch_name(name):
query = datastore_client.query(kind='visit')
print("name is: " + name)
# query.add_filter("name", "=", name)
results = list(query.fetch())
# print("GET - param is results: " + results)
return jsonify(results)
which mean, if I uncommit the line:
# query.add_filter("name", "=", name)
will get this error:
Traceback (most recent call last):
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/flask/app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/flask/app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/flask/app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/flask/app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "/srv/main.py", line 107, in getVariable
times = fetch_times(name)
File "/srv/main.py", line 40, in fetch_times
results = list(query.fetch())
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/page_iterator.py", line 208, in _items_iter
for page in self._page_iter(increment=False):
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/page_iterator.py", line 244, in _page_iter
page = self._next_page()
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/datastore/query.py", line 610, in _next_page
response_pb = self.client._datastore_api.run_query(
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/datastore_v1/services/datastore/client.py", line 579, in run_query
response = rpc(
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/gapic_v1/method.py", line 154, in __call__
return wrapped_func(*args, **kwargs)
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry.py", line 283, in retry_wrapped_func
return retry_target(
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/retry.py", line 190, in retry_target
return target()
File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/api_core/grpc_helpers.py", line 74, in error_remapped_callable
raise exceptions.from_grpc_error(exc) from exc
google.api_core.exceptions.FailedPrecondition: 400 no matching index found.
and the response return: Internal server error. How can fix that ?
I tried to add indexing from the UI of google cloud page - add index to name property.
You'll want to check the mode of your database (https://cloud.google.com/sdk/gcloud/reference/alpha/firestore/databases/describe). It's likely your database is in Firestore native mode instead of Firestore in Datastore mode. If so, you'll want to use the Firestore client libraries instead of the Datastore client libraries, or switch your database type to Datastore mode.
This question already has answers here:
Why do I get AttributeError: 'NoneType' object has no attribute 'something'?
(11 answers)
Closed 1 year ago.
I'm getting the error when trying to get all the Urls for which the limited field is True. I've tried deleting the migrations and creating new migrations, but still getting the same error.
these are the installed dependencies:
asgiref==3.4.1
backports.zoneinfo==0.2.1
Django==4.0
django-cors-headers==3.10.1
django-shell-plus==1.1.7
djangorestframework==3.13.1
djongo==1.3.6
dnspython==2.1.0
gunicorn==20.1.0
pymongo==3.12.1
python-dotenv==0.19.2
pytz==2021.3
sqlparse==0.2.4
tzdata==2021.5
models.py:
class Urls(models.Model):
_id = models.ObjectIdField()
record_id = models.IntegerField(unique=True)
hash = models.CharField(unique=True, max_length=1000)
long_url = models.URLField()
created_at = models.DateField(auto_now_add=True)
expires_in = models.DurationField(default=timedelta(days=365*3))
expires_on = models.DateField(null=True, blank=True)
limited = models.BooleanField()
exhausted = models.BooleanField()
query:
Urls.objects.filter(limited=False)
error:
Traceback (most recent call last):
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 857, in parse
return handler(self, statement)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 933, in _select
return SelectQuery(self.db, self.connection_properties, sm, self._params)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 116, in __init__
super().__init__(*args)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 62, in __init__
self.parse()
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 152, in parse
self.where = WhereConverter(self, statement)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/converters.py", line 27, in __init__
self.parse()
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/converters.py", line 119, in parse
self.op = WhereOp(
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/operators.py", line 476, in __init__
self.evaluate()
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/operators.py", line 465, in evaluate
op.evaluate()
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/operators.py", line 258, in evaluate
self.rhs.negate()
AttributeError: 'NoneType' object has no attribute 'negate'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/cursor.py", line 51, in execute
self.result = Query(
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 784, in __init__
self._query = self.parse()
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/sql2mongo/query.py", line 885, in parse
raise exe from e
djongo.exceptions.SQLDecodeError:
Keyword: None
Sub SQL: None
FAILED SQL: SELECT "short_urls"."_id", "short_urls"."record_id", "short_urls"."hash", "short_urls"."long_url", "short_urls"."created_at", "short_urls"."expires_in", "short_urls"."expires_on", "short_urls"."limited", "short_urls"."exhausted" FROM "short_urls" WHERE NOT "short_urls"."limited" LIMIT 21
Params: ()
Version: 1.3.6
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/backends/utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/cursor.py", line 59, in execute
raise db_exe from e
djongo.database.DatabaseError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/models/query.py", line 256, in __repr__
data = list(self[:REPR_OUTPUT_SIZE + 1])
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/models/query.py", line 262, in __len__
self._fetch_all()
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/models/query.py", line 1354, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/models/query.py", line 51, in __iter__
results = compiler.execute_sql(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1202, in execute_sql
cursor.execute(sql, params)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/backends/utils.py", line 99, in execute
return super().execute(sql, params)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/backends/utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/django/db/backends/utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
File "/mnt/c/Users/pranjal/Desktop/urlhash/env/lib/python3.8/site-packages/djongo/cursor.py", line 59, in execute
raise db_exe from e
django.db.utils.DatabaseError
I would need to see your views.py where you do that query, Urls.objects.filter(limited=False) but first, perhaps the query is not finding any Urls objects that match the criteria, limited=False, thus the query returns None, which has no attributes. Second, I'm not familiar with negate. I don't see it in your model. Perhaps if I see the views.py where you make that query I will understand better.
I'm using Django 3 and Python 3.7. I have a model (MySql 8 backed table) that has integer primary keys. I have code that searches for such models like so
state = State.objects.get(pk=locality['state'])
The issue is if "locality['state']" contains an empty string, I get the below error
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/fields/__init__.py", line 1768, in get_prep_value
return int(value)
ValueError: invalid literal for int() with base 10: ''
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/davea/Documents/workspace/chicommons/maps/web/tests/test_serializers.py", line 132, in test_coop_create_with_incomplete_data
assert not serializer.is_valid(), serializer.errors
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/rest_framework/serializers.py", line 234, in is_valid
self._validated_data = self.run_validation(self.initial_data)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/rest_framework/serializers.py", line 433, in run_validation
value = self.to_internal_value(data)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/rest_framework/serializers.py", line 490, in to_internal_value
validated_value = field.run_validation(primitive_value)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/rest_framework/fields.py", line 565, in run_validation
value = self.to_internal_value(data)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/rest_framework/relations.py", line 519, in to_internal_value
return [
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/rest_framework/relations.py", line 520, in <listcomp>
self.child_relation.to_internal_value(item)
File "/Users/davea/Documents/workspace/chicommons/maps/web/directory/serializers.py", line 26, in to_internal_value
state = State.objects.get(pk=locality['state'])
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/manager.py", line 82, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/query.py", line 404, in get
clone = self._chain() if self.query.combinator else self.filter(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/query.py", line 904, in filter
return self._filter_or_exclude(False, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/query.py", line 923, in _filter_or_exclude
clone.query.add_q(Q(*args, **kwargs))
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/sql/query.py", line 1337, in add_q
clause, _ = self._add_q(q_object, self.used_aliases)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/sql/query.py", line 1362, in _add_q
child_clause, needed_inner = self.build_filter(
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/sql/query.py", line 1298, in build_filter
condition = self.build_lookup(lookups, col, value)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/sql/query.py", line 1155, in build_lookup
lookup = lookup_class(lhs, rhs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/lookups.py", line 22, in __init__
self.rhs = self.get_prep_lookup()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/lookups.py", line 72, in get_prep_lookup
return self.lhs.output_field.get_prep_value(self.rhs)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/django/db/models/fields/__init__.py", line 1770, in get_prep_value
raise e.__class__(
ValueError: Field 'id' expected a number but got ''.
Is there a more "Django" way to search for an object without an error being thrown if the object doesn't exist? I could do this
state = None if str(type(locality['state'])) != "<class 'int'>" else State.objects.get(pk=locality['state'])
but this seems unnecessarily wordy and not how Django was intended to be used.
I would choose Ask forgiveness not permission strategy
try:
state = State.objects.get(pk=int(locality['state']))
except ValueError:
state = None
You could use a logical AND to validate the dict value before using it to look up the data.
state = locality['state'] and State.objects.get(pk=locality['state'])
I am trying to parallelize the tasks of correcting texts on many documents with Python, so I naturally found "joblib". I want each task to be to correct a given document. Here is the structure of the code:
if __name__ == '__main__':
lexicon = build_compact_lexicon()
from joblib import Parallel, delayed
import multiprocessing
num_cores = multiprocessing.cpu_count()
results = Parallel(n_jobs=num_cores)(delayed(find_errors)('GDL', i, 1, lexicon) for i in range(1798, 1820))
I am using the function find_errors summed up here :
def find_errors(newspaper, year, month, lexicon):
# parse the input newspaper text data using etree parser from LXML
# detect errors in the text
return found_errors_type1, found_errors_type2, found_errors_type3
This does raise me a few errors
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 130, in __call__
return self.func(*args, **kwargs)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 72, in __call__
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 72, in <listcomp>
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "hellowordParallel.py", line 85, in find_errors
tree = etree.parse(xml_file_path)
File "src/lxml/lxml.etree.pyx", line 3427, in lxml.etree.parse (src/lxml/lxml.etree.c:79801)
File "src/lxml/parser.pxi", line 1805, in lxml.etree._parseDocument (src/lxml/lxml.etree.c:116293)
TypeError: cannot parse from 'NoneType'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 392, in find_cookie
line_string = line.decode('utf-8')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 24: invalid start byte
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/multiprocessing/pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 139, in __call__
tb_offset=1)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/format_stack.py", line 373, in format_exc
frames = format_records(records)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/format_stack.py", line 274, in format_records
for token in generate_tokens(linereader):
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 514, in _tokenize
line = readline()
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/format_stack.py", line 265, in linereader
line = getline(file, lnum[0])
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/linecache.py", line 16, in getline
lines = getlines(filename, module_globals)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/linecache.py", line 47, in getlines
return updatecache(filename, module_globals)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/linecache.py", line 136, in updatecache
with tokenize.open(fullname) as fp:
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 456, in open
encoding, lines = detect_encoding(buffer.readline)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 433, in detect_encoding
encoding = find_cookie(first)
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/tokenize.py", line 397, in find_cookie
raise SyntaxError(msg)
File "<string>", line None
SyntaxError: invalid or missing encoding declaration for '/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/lxml/etree.so'
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "hellowordParallel.py", line 160, in <module>
results = Parallel(n_jobs=num_cores)(delayed(find_errors)('GDL', i, 1, lexicon) for i in range(1798, 1820))
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 810, in __call__
self.retrieve()
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/joblib/parallel.py", line 727, in retrieve
self._output.extend(job.get())
File "/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/multiprocessing/pool.py", line 608, in get
raise self._value
SyntaxError: invalid or missing encoding declaration for '/home/mbl/anaconda3/envs/OCR_Correction/lib/python3.5/site-packages/lxml/etree.so'
I don't understand if this is due do something related with configs or if my function doesn't fit in a parallel implementation... (I guess it should...)
Did it happen to some of you before?
Hope my question is clear and there is enough information for someone to give me some help!
using python3, I can connect to mysql using pymysql. all works as expected. enclosed code works.
import pymysql
conn = pymysql.connect(host='127.0.0.1', unix_socket='/home/jhgong/mysql/tmp/mysql.sock', user='root', passwd='my_pass', db='my_db', port='3333')
cur = conn.cursor()
cur.execute('select user from auth_users')
for i in cur:
print(i)
trying to get sqlalchemy to connect with pymysql, the default example strings don't seem to work. the above example does not work unless I declare both the port number and a unix_socket.
below is what I've been using to try and get sqlalchemy to connect. I assume that the socket and port number are both needed. I used connect_args to feed in a hash with the extra unix_socket location. no joy.
enclosed the the snippit I've been using that creates the error.
conarg = {
'unix_socket':'/home/jhgong/mysql/tmp/mysql.sock',
'db' :'ice'
}
engine = create_engine('mysql+pymysql://root:my_pass#127.0.0.1:3333/my_db', connect_args = conarg, echo=True)
connection = engine.connect()
with or without the db in conarg hash i get the following error:
>>> connection = engine.connect()
2013-01-17 13:04:20,819 INFO sqlalchemy.engine.base.Engine b'SELECT DATABASE()'
2013-01-17 13:04:20,819 INFO sqlalchemy.engine.base.Engine ()
Traceback (most recent call last):
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 724, in _do_get
return self._pool.get(wait, self._timeout)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/util/queue.py", line 163, in get
raise Empty
sqlalchemy.util.queue.Empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 1574, in connect
return self._connection_cls(self, **kwargs)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 58, in __init__
self.__connection = connection or engine.raw_connection()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 1637, in raw_connection
return self.pool.unique_connection()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 182, in unique_connection
return _ConnectionFairy(self).checkout()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 398, in __init__
rec = self._connection_record = pool._do_get()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 744, in _do_get
con = self._create_connection()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 187, in _create_connection
return _ConnectionRecord(self)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/pool.py", line 284, in __init__
exec_once(self.connection, self)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/event.py", line 362, in exec_once
self(*args, **kw)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/event.py", line 379, in __call__
fn(*args, **kw)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/strategies.py", line 168, in first_connect
dialect.initialize(c)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/dialects/mysql/base.py", line 2005, in initialize
default.DefaultDialect.initialize(self, connection)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/default.py", line 183, in initialize
self._get_default_schema_name(connection)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/dialects/mysql/base.py", line 1970, in _get_default_schema_name
return connection.execute('SELECT DATABASE()').scalar()
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 645, in execute
params)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 791, in _execute_text
statement, parameters
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/base.py", line 854, in _execute_context
context)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/SQLAlchemy-0.8.0b1dev-py3.2.egg/sqlalchemy/engine/default.py", line 342, in do_execute
cursor.execute(statement, parameters)
File "/nfs/site/home/jhgongx/.local/lib/python3.2/site-packages/PyMySQL3-0.5-py3.2.egg/pymysql/cursors.py", line 105, in execute
query = query % escaped_args
TypeError: unsupported operand type(s) for %: 'bytes' and 'tuple'
it appears to be raising an error on empty pool queue. setting the size or type of pool queue has no effect.
any suggestions on how to figure this out?
Try this very simple example:
import sqlalchemy
from sqlalchemy.sql import select
from sqlalchemy import Table, MetaData
def init():
try:
server = 'xx'
db = 'xx'
login = 'xx'
passwd = 'xx'
engine_str = 'mysql+mysqlconnector://{}:{}#{}/{}'.format(login, passwd, server, db)
engine = sqlalchemy.create_engine(engine_str, echo=False, encoding='utf-8')
connection = engine.connect()
metadata = MetaData()
t_servers = Table('your_table_here', metadata, autoload=True, autoload_with=engine)
s = select([t_servers])
result = connection.execute(s)
for row in result:
print(row['the_field'])
except Exception:
raise
finally:
connection.close()
if __name__ == '__main__':
init()
Mysql connector for Python 3 : download here
I know this is late but the PyMySQL requirements mention CPython >= 2.6 or >= 3.3, it looks like you're using CPython 3.2 (the default version of python used), and that may be your problem.