How do I get $near to work in Flask-mongoalchemy - python-3.x

I have been trying to work on fetching events from my mongoDB database that are close to the current user's location
I have tried to reformat my Model scheme to contain [type: "Point"] and even arranged my longitude and latitude into a list.
I also tried adding "2dsphere" indexes using meta to my Model based on what I saw in mongo-alchemy documentation.
My Model
class Event(db.Document):
meta = {
'indexes': [
("*location.coordinates", "2dsphere")
]
}
user_id = db.StringField()
uuid = db.StringField()
name = db.StringField()
address = db.StringField()
start_time = db.DateTimeField(required=True, default=datetime.datetime.now())
end_time = db.DateTimeField(required=True, default=datetime.datetime.now())
location = db.DictField(db.AnythingField())
This is now my main query code
def get(self):
latitude = float(request.args.get('lat'))
longitude = float(request.args.get('long'))
print(longitude);
print(latitude);
event = Event.query.filter({"location" :
{ "$near" :
{
"$geometry" : {
"type" : "Point",
"coordinates" : [longitude, latitude] },
"$maxDistance" : 4000
}
}
}).first()
print(event);
[2019-01-15 23:20:59,797] ERROR in app: Exception on /v1/event [GET]
Traceback (most recent call last):
File "/home/creative_joe/.local/lib/python3.5/site-packages/flask/app.py", line 1813, in full_dispatch_request
rv = self.dispatch_request()
File "/home/creative_joe/.local/lib/python3.5/site-packages/flask/app.py", line 1799, in dispatch_request
return self.view_functionsrule.endpoint
File "/home/creative_joe/.local/lib/python3.5/site-packages/flask_restplus/api.py", line 325, in wrapper
resp = resource(*args, **kwargs)
File "/home/creative_joe/.local/lib/python3.5/site-packages/flask/views.py", line 88, in view
return self.dispatch_request(*args, **kwargs)
File "/home/creative_joe/.local/lib/python3.5/site-packages/flask_restplus/resource.py", line 44, in dispatch_request
resp = meth(*args, **kwargs)
File "/media/creative_joe/3004586c-9a2d-4cb0-8a5f-d41fe99afc05/home/creative_joe/MonkeyMusic.server/app/views/main.py", line 323, in get
"$maxDistance" : 4000
File "/home/creative_joe/.local/lib/python3.5/site-packages/mongoalchemy/query.py", line 139, in first
for doc in iter(self):
File "/home/creative_joe/.local/lib/python3.5/site-packages/mongoalchemy/query.py", line 412, in next
return self._next_internal()
File "/home/creative_joe/.local/lib/python3.5/site-packages/mongoalchemy/query.py", line 416, in _next_internal
value = next(self.cursor)
File "/home/creative_joe/.local/lib/python3.5/site-packages/mongoalchemy/py3compat.py", line 41, in next
return it.next()
File "/home/creative_joe/.local/lib/python3.5/site-packages/pymongo/cursor.py", line 1189, in next
if len(self.__data) or self._refresh():
File "/home/creative_joe/.local/lib/python3.5/site-packages/pymongo/cursor.py", line 1104, in _refresh
self.__send_message(q)
File "/home/creative_joe/.local/lib/python3.5/site-packages/pymongo/cursor.py", line 982, in __send_message
helpers._check_command_response(first)
File "/home/creative_joe/.local/lib/python3.5/site-packages/pymongo/helpers.py", line 155, in _check_command_response
raise OperationFailure(msg % errmsg, code, response)
pymongo.errors.OperationFailure: error processing query: ns=heroku_c9gg06k0.EventTree: GEONEAR field=location maxdist=4000 isNearSphere=0
Sort: {}
Proj: {}
planner returned error: unable to find index for $geoNear query

Related

Upload file from Django app to IBM Cloud Object Storage

I'm trying to connect a django app to IBM COS and having trouble. I'm capturing user video and want to save the file to IBM COS and the user info to Postgres also hosted on IBM. I'm able to connect from both the terminal and my app to IBM COS and move files around, but am having trouble getting the default storage configured properly. I'm using django-storages, trying to adapt the AWS configurations for IBM but I must be missing something.
This code will save the file to IBM COS, but makes no entries in the DB. The problem may be in the configuration?
Also, I am not able to manually upload a file form the django admin panel - I get a similar traceback. Thanks in advance for any help.
settings.py
# IBM STORAGE CONFIG
IBM_API_KEY_ID = 'IBM_API_KEY_ID'
IAM_SERVICE_ID = 'IAM_SERVICE_ID'
ENDPOINT = 'https://s3.us-east.cloud-object-storage.appdomain.cloud'
IBM_AUTH_ENDPOINT = 'https://iam.bluemix.net/oidc/token'
SERVICE_INSTANCE_ID = 'SERVICE_INSTANCE_ID'
IBM_STORAGE_BUCKET_NAME = 'cloud-object-storage-3u-cos-standard-77w'
AWS_S3_FILE_OVERWRITE = False
AWS_DEFAULT_ACL = None
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
models.py
class Video(models.Model):
user = models.ForeignKey(User, on_delete=models.CASCADE, null=True, blank=True)
created = models.DateTimeField(auto_now_add=True)
videofilename=models.CharField(max_length=500)
videofile=models.FileField(upload_to="video/", null=True, verbose_name="")
def __str__(self):
return str(self.videofile)
views.py
class upload_to_ibm_auto(LoginRequiredMixin, CreateView):
model = Video
context_object_name = 'Videos'
form_class = VideoForm
template_name = 'app_video/upload_to_ibm_auto.html'
success_url = reverse_lazy('video_upload_local')
def form_valid(self, form):
# create connection to IBM
cos_client = ibm_boto3.client(service_name='s3',
ibm_api_key_id=settings.IBM_API_KEY_ID,
ibm_service_instance_id=settings.IAM_SERVICE_ID,
ibm_auth_endpoint=settings.IBM_AUTH_ENDPOINT,
config=Config(signature_version='oauth'),
endpoint_url=settings.ENDPOINT)
# assign variables for upload to IBM
upload_name = str(form.cleaned_data['videofilename'])
local_file_name = 'C:/zjunk/' + str(form.cleaned_data['videofile'])
userid = str(self.request.user.id)
uploadtime=str(datetime.datetime.now())
key = userid + '-' + uploadtime + '-' + upload_name
#key='test'+form.cleaned_data['videofilename']
#local_file_name=form.cleaned_data['videofile']
#Bucket=str('cloud-object-storage-3u-cos-standard-77w')
bucket = settings.IBM_STORAGE_BUCKET_NAME
print(bucket)
try:
cos_client.upload_file(Filename=local_file_name, Bucket=bucket, Key=key)
except Exception as e:
print(Exception, e)
else:
print('File Uploaded to IBM')
return super(upload_to_ibm_auto, self).form_valid(form)
Traceback:
Environment:
Request Method: POST
Request URL: http://localhost:8000/app_video_upload_to_ibm_auto/
Django Version: 3.2.6
Python Version: 3.9.1
Installed Applications:
['django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'drf_spectacular',
'storages',
'app_crm',
'app_video',
'app_library']
Installed Middleware:
['django.middleware.security.SecurityMiddleware',
'whitenoise.middleware.WhiteNoiseMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware']
Traceback (most recent call last):
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\views\generic\base.py", line 70, in view
return self.dispatch(request, *args, **kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\contrib\auth\mixins.py", line 71, in dispatch
return super().dispatch(request, *args, **kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\views\generic\base.py", line 98, in dispatch
return handler(request, *args, **kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\views\generic\edit.py", line 172, in post
return super().post(request, *args, **kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\views\generic\edit.py", line 142, in post
return self.form_valid(form)
File "C:\Users\Kord\source\repos\storyline\dev\01\PythonDjangoAppLMSUS2021-08-23-SL-1\app_video\views.py", line 163, in form_valid
return super(upload_to_ibm_auto, self).form_valid(form)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\views\generic\edit.py", line 125, in form_valid
self.object = form.save()
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\forms\models.py", line 468, in save
self.instance.save()
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\base.py", line 726, in save
self.save_base(using=using, force_insert=force_insert,
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\base.py", line 763, in save_base
updated = self._save_table(
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\base.py", line 868, in _save_table
results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\base.py", line 906, in _do_insert
return manager._insert(
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\query.py", line 1270, in _insert
return query.get_compiler(using=using).execute_sql(returning_fields)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\sql\compiler.py", line 1415, in execute_sql
for sql, params in self.as_sql():
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\sql\compiler.py", line 1358, in as_sql
value_rows = [
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\sql\compiler.py", line 1359, in <listcomp>
[self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\sql\compiler.py", line 1359, in <listcomp>
[self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\sql\compiler.py", line 1310, in pre_save_val
return field.pre_save(obj, add=True)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\fields\files.py", line 302, in pre_save
file.save(file.name, file.file, save=False)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\db\models\fields\files.py", line 89, in save
self.name = self.storage.save(name, content, max_length=self.field.max_length)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\core\files\storage.py", line 53, in save
name = self.get_available_name(name, max_length=max_length)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\storages\backends\s3boto3.py", line 585, in get_available_name
return super().get_available_name(name, max_length)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\django\core\files\storage.py", line 87, in get_available_name
while self.exists(name) or (max_length and len(name) > max_length):
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\storages\backends\s3boto3.py", line 457, in exists
self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\client.py", line 386, in _api_call
return self._make_api_call(operation_name, kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\client.py", line 677, in _make_api_call
request_dict = self._convert_to_request_dict(
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\client.py", line 723, in _convert_to_request_dict
api_params = self._emit_api_params(
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\client.py", line 752, in _emit_api_params
self.meta.events.emit(
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\hooks.py", line 357, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\hooks.py", line 211, in _emit
response = handler(**kwargs)
File "C:\Users\Kord\source\repos\storyline\dev\01\venv2\lib\site-packages\botocore\handlers.py", line 235, in validate_bucket_name
if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket):
Exception Type: TypeError at /app_video_upload_to_ibm_auto/
Exception Value: expected string or bytes-like object
Thanks again.

How to filter Serializers in DjangoRestFramework based on user id?

I'm new to django,
I can't seem to find how to return an api based on the user who requested it and created the Transaction object, can you please guide me through? I have read this, but I can't find a clear guide in the docs to update the Serializer straightaway.
In laravel, I can do:
Transaction::where('user_id', Auth::user()->id)->get();
This is my serializer:
class TransactionSerializer(serializers.HyperlinkedModelSerializer):
user = UserSerializer(read_only=True)
tags = TagShortSerializer(read_only=True, many=True)
def get_fields(self, *args, **kwargs):
# Override this method so that the list of DatabaseUsers presented in the browseable API
# is restricted to the DatabaseUsers owned by the current user.
fields = super(TransactionSerializer, self).get_fields(*args, **kwargs)
view = self.context["view"]
user = view.request.user
return Transaction.objects.filter(user=user)
class Meta:
model = Transaction
fields = ["user", "title", "amount", "category", "tags"]
My viewset:
class TransactionViewSet(viewsets.ModelViewSet):
queryset = Transaction.objects.all()
serializer_class = TransactionSerializer
permission_classes = [permissions.IsAuthenticated, IsOwner]
Currently it returns:
Complete traceback:
Environment:
Request Method: GET
Request URL: http://127.0.0.1:8000/transactions/
Django Version: 3.2.5
Python Version: 3.9.5
Installed Applications:
['djmoney',
'rest_framework',
'finance.apps.FinanceConfig',
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles']
Installed Middleware:
['django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware']
Traceback (most recent call last):
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\django\views\decorators\csrf.py", line 54, in wrapped_view
return view_func(*args, **kwargs)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\viewsets.py", line 125, in view
return self.dispatch(request, *args, **kwargs)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\views.py", line 509, in dispatch
response = self.handle_exception(exc)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\views.py", line 469, in handle_exception
self.raise_uncaught_exception(exc)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\views.py", line 480, in raise_uncaught_exception
raise exc
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\views.py", line 506, in dispatch
response = handler(request, *args, **kwargs)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\mixins.py", line 43, in list
return self.get_paginated_response(serializer.data)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 745, in data
ret = super().data
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 246, in data
self._data = self.to_representation(self.instance)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 663, in to_representation
return [
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 664, in <listcomp>
self.child.to_representation(item) for item in iterable
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 500, in to_representation
for field in fields:
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 361, in _readable_fields
for field in self.fields.values():
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\django\utils\functional.py", line 48, in __get__
res = instance.__dict__[self.name] = self.func(instance)
File "E:\Development\Python\sharkware-api\.venv\lib\site-packages\rest_framework\serializers.py", line 349, in fields
for key, value in self.get_fields().items():
Exception Type: AttributeError at /transactions/
Exception Value: 'QuerySet' object has no attribute 'items'
That is not the correct usage for get_fields. That method returns the list of fields to be used when instantiating the serializer.
What you want is to override get_queryset on your viewset:
class TransactionViewSet(viewsets.ModelViewSet):
queryset = Transaction.objects.all()
serializer_class = TransactionSerializer
permission_classes = [permissions.IsAuthenticated, IsOwner]
def get_queryset(self):
if self.request.user.is_authenticated:
return self.queryset.filter(user=self.request.user)
return Transaction.objects.none()

How to pass multiple parameters for aiohttp request wit the same name?

The API from which I want to receive data takes several values with the same name as a parameter:
https://example.com/method?types=Retail&types=Office&types=Industrial......
I try to do it like this:
params_api_sale = {
'types': [
'Retail', 'Office', 'Industrial',
'Mixed Use', 'Development Site',
'Land', 'Special Purpose', 'Other'
],
'sortDirection': 'Descending',
'sortOrder': 'ActivatedOn'
}
async def get_items():
async with aiohttp.request(
method='GET', url=vars_.url_api_sale,
headers=vars_.headers, params=vars_.params_api_sale) as response:
...
But this gives me an error:
Traceback (most recent call last):
File "foo.py", line 85, in <module>
asyncio.get_event_loop().run_until_complete(main())
File "C:\Python37\lib\asyncio\base_events.py", line 583, in run_until_complete
return future.result()
File "foo.py", line 80, in main
await asyncio.create_task(get_items())
File "foo.py", line 44, in get_items
headers=vars_.headers, params=vars_.params_api_sale) as response:
File "D:\Dev\venv\lib\site-packages\aiohttp\client.py", line 1051, in __aenter__
self._resp = await self._coro
File "D:\Dev\venv\lib\site-packages\aiohttp\client.py", line 473, in _request
ssl=ssl, proxy_headers=proxy_headers, traces=traces)
File "D:\Dev\venv\lib\site-packages\aiohttp\client_reqrep.py", line 263, in __init__
url2 = url.with_query(params)
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 922, in with_query
new_query = self._get_str_query(*args, **kwargs)
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 886, in _get_str_query
quoter(k) + "=" + quoter(self._query_var(v)) for k, v in query.items()
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 886, in <genexpr>
quoter(k) + "=" + quoter(self._query_var(v)) for k, v in query.items()
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 864, in _query_var
"of type {}".format(v, type(v))
TypeError: Invalid variable type: value should be str or int, got ['Retail', 'Office', 'Industrial', 'Mixed Use', 'Development Site', 'Land', 'Special Purpose', 'Other'] of type <class 'list'>
I have not found a documented way to do this. All that comes to my mind is to generate a URL string with these parameters, of course, if there is no more correct way.
import requests
params = {
'types': [
'Retail', 'Office', 'Industrial',
'Mixed Use', 'Development Site',
'Land', 'Special Purpose', 'Other'
],
'sortDirection': 'Descending',
'sortOrder': 'ActivatedOn'
}
r = requests.get("http://www.test.com/", params=params)
print(r.url)
output:
https://www.test.com/?types=Retail&types=Office&types=Industrial&types=Mixed+Use&types=Development+Site&types=Land&types=Special+Purpose&types=Other&sortDirection=Descending&sortOrder=ActivatedOn
For aiohttp you can use MultiDict which is also included in the official document.
For sending data with multiple values for the same key MultiDict may be used as well.
As stated at aiohttp documentation, you can pass a list of tuples to params with the same key to achieve your goal:
Also it is possible to pass list of 2 items tuples as parameters, in that case you can specifiy multiple values for each key:
params = [
("key1", "value1"),
("key1", "value2"),
("key2", "value3"),
]

Scrapy error catching in scrapy/middleware.py file: TypeError: __init__() missing 1 required positional argument: 'uri'

I am catching this error while starting a crawl. I have searched for an answer in several forums, and looked at the code in scrapy/middleware.py (came standard with scrapy and I have not altered it) and cannot figure out why I am getting an error.
The scraper is using both an ImagesPipeline and S3FilesStore pipeline for storing a json file and downloaded images directly into different S3 folders. I am using Python 3.6.
Any help is appreciated. The error message and my scraper settings are below, please let me know if anything else would be useful.
Traceback (most recent call last):
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/crawler.py", line 77, in crawl
self.engine = self._create_engine()
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/crawler.py", line 102, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/middleware.py", line 58, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/middleware.py", line 40, in from_settings
mw = mwcls()
TypeError: __init__() missing 1 required positional argument: 'uri'
ITEM_PIPELINES = {
'scrapy.pipelines.files.S3FilesStore': 1,
'scrapy.pipelines.images.ImagesPipeline': 1
}
AWS_ACCESS_KEY_ID = 'xxxxxx'
AWS_SECRET_ACCESS_KEY= 'xxxxxx'
IMAGES_STORE = 's3 path'
FEED_URI = 's3 path'
FEED_FORMAT = 'jsonlines'
FEED_EXPORT_FIELDS = None
FEED_STORE_EMPTY = False
FEED_STORAGES = {}
FEED_STORAGES_BASE = {
'': 'scrapy.extensions.feedexport.FileFeedStorage',
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
}
FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jl': None,
'csv': None,
'xml': None,
'marshal': None,
'pickle': None,
}

Google Adwords Traffic Estimator Service and Python

I've downloaded a code sample that looks for particular keywords and pulls some metrics. I've noticed a lot of Google Adwords API examples are compliant with python 3.x so I'm wondering if there is an issue with that? See below for code sample:
from googleads import adwords
def main(client):
# Initialize appropriate service.
traffic_estimator_service = client.GetService(
'TrafficEstimatorService', version='v201609')
# Construct selector object and retrieve traffic estimates.
keywords = [
{'text': 'mars cruise', 'matchType': 'BROAD'},
{'text': 'cheap cruise', 'matchType': 'PHRASE'},
{'text': 'cruise', 'matchType': 'EXACT'}
]
negative_keywords = [
{'text': 'moon walk', 'matchType': 'BROAD'}
]
keyword_estimate_requests = []
for keyword in keywords:
keyword_estimate_requests.append({
'keyword': {
'xsi_type': 'Keyword',
'matchType': keyword['matchType'],
'text': keyword['text']
}
})
for keyword in negative_keywords:
keyword_estimate_requests.append({
'keyword': {
'xsi_type': 'Keyword',
'matchType': keyword['matchType'],
'text': keyword['text']
},
'isNegative': 'true'
})
# Create ad group estimate requests.
adgroup_estimate_requests = [{
'keywordEstimateRequests': keyword_estimate_requests,
'maxCpc': {
'xsi_type': 'Money',
'microAmount': '1000000'
}
}]
# Create campaign estimate requests.
campaign_estimate_requests = [{
'adGroupEstimateRequests': adgroup_estimate_requests,
'criteria': [
{
'xsi_type': 'Location',
'id': '2840' # United States.
},
{
'xsi_type': 'Language',
'id': '1000' # English.
}
],
}]
# Create the selector.
selector = {
'campaignEstimateRequests': campaign_estimate_requests,
}
# Optional: Request a list of campaign-level estimates segmented by
# platform.
selector['platformEstimateRequested'] = True
# Get traffic estimates.
estimates = traffic_estimator_service.get(selector)
campaign_estimate = estimates['campaignEstimates'][0]
# Display the campaign level estimates segmented by platform.
if 'platformEstimates' in campaign_estimate:
platform_template = ('Results for the platform with ID: "%d" and name: '
'"%s".')
for platform_estimate in campaign_estimate['platformEstimates']:
platform = platform_estimate['platform']
DisplayEstimate(platform_template % (platform['id'],
platform['platformName']),
platform_estimate['minEstimate'],
platform_estimate['maxEstimate'])
# Display the keyword estimates.
if 'adGroupEstimates' in campaign_estimate:
ad_group_estimate = campaign_estimate['adGroupEstimates'][0]
if 'keywordEstimates' in ad_group_estimate:
keyword_estimates = ad_group_estimate['keywordEstimates']
keyword_template = ('Results for the keyword with text "%s" and match '
'type "%s":')
keyword_estimates_and_requests = zip(keyword_estimates,
keyword_estimate_requests)
for keyword_tuple in keyword_estimates_and_requests:
if keyword_tuple[1].get('isNegative', False):
continue
keyword = keyword_tuple[1]['keyword']
keyword_estimate = keyword_tuple[0]
DisplayEstimate(keyword_template % (keyword['text'],
keyword['matchType']),
keyword_estimate['min'], keyword_estimate['max'])
def _CalculateMean(min_est, max_est):
if min_est and max_est:
return (float(min_est) + float(max_est)) / 2.0
else:
return None
def _FormatMean(mean):
if mean:
return '%.2f' % mean
else:
return 'N/A'
def DisplayEstimate(message, min_estimate, max_estimate):
"""Displays mean average cpc, position, clicks, and total cost for estimate.
Args:
message: str message to display for the given estimate.
min_estimate: sudsobject containing a minimum estimate from the
TrafficEstimatorService response.
max_estimate: sudsobject containing a maximum estimate from the
TrafficEstimatorService response.
"""
# Find the mean of the min and max values.
mean_avg_cpc = (_CalculateMean(min_estimate['averageCpc']['microAmount'],
max_estimate['averageCpc']['microAmount'])
if 'averageCpc' in min_estimate else None)
mean_avg_pos = (_CalculateMean(min_estimate['averagePosition'],
max_estimate['averagePosition'])
if 'averagePosition' in min_estimate else None)
mean_clicks = _CalculateMean(min_estimate['clicksPerDay'],
max_estimate['clicksPerDay'])
mean_total_cost = _CalculateMean(min_estimate['totalCost']['microAmount'],
max_estimate['totalCost']['microAmount'])
print (message)
print ('Estimated average CPC: %s' % _FormatMean(mean_avg_cpc))
print ('Estimated ad position: %s' % _FormatMean(mean_avg_pos))
print ('Estimated daily clicks: %s' % _FormatMean(mean_clicks))
print ('Estimated daily cost: %s' % _FormatMean(mean_total_cost))
if __name__ == '__main__':
# Initialize client object.
adwords_client = adwords.AdWordsClient.LoadFromStorage()
main(adwords_client)
Here is the error message:
(Money) not-found
path: "Money", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Location) not-found
path: "Location", not-found
(Language) not-found
path: "Language", not-found
<suds.sax.document.Document object at 0x03BF1D10>
Server raised fault in response.
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 82, in send
fp = self.u2open(u2request)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 132, in u2open
return url.open(u2request, timeout=tm)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 472, in open
response = meth(req, response)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 582, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 510, in error
return self._call_chain(*args)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 444, in _call_chain
result = func(*args)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 590, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 500: Internal Server Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 613, in send
reply = self.options.transport.send(request)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 94, in send
raise TransportError(e.msg, e.code, e.fp)
suds.transport.TransportError: Internal Server Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\adwords test - Copy (2).py", line 177, in <module>
main(adwords_client)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\adwords test - Copy (2).py", line 95, in main
estimates = traffic_estimator_service.get(selector)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\googleads\common.py", line 696, in MakeSoapRequest
raise e
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\googleads\common.py", line 692, in MakeSoapRequest
for arg in args])
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 521, in __call__
return client.invoke(args, kwargs)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 581, in invoke
result = self.send(soapenv)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 619, in send
description=tostr(e), original_soapenv=original_soapenv)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 670, in process_reply
raise WebFault(fault, replyroot)
suds.WebFault: Server raised fault: '[AuthenticationError.CLIENT_CUSTOMER_ID_IS_REQUIRED # ; trigger:'<null>']'
You should set client_customer_id in your googleads.yaml file . you can get client_customer_id from your manager account. go to your manager account and add client then copy your id from upright corner of screen. in googleads.yaml file paste that id to client_customer_id .

Resources