youtube_dl KeyError 'key' - python-3.x

Hello I am trying to download youtube videos with the following code
import youtube_dl
import tempfile
youtube_links = ['https://www.youtube.com/watch?v=668nUCeBHyY']
with tempfile.TemporaryDirectory() as tempdir:
opts = {
'format': 'best',
'outtmpl': f'{tempdir}/%(id)s.%(ext)s',
'noplaylist': True,
'postprocessors': [{
'preferredcodec': 'mp4'
}]
}
ydl = youtube_dl.YoutubeDL(opts)
try:
meta = ydl.extract_info(
youtube_links,
download=True
)
except Exception as e:
raise e
else:
print(f"Downloaded to {tempdir}/{meta['id']}.{meta['ext']}")
However this raises an error:
<path>>py -3.8 test.py
Traceback (most recent call last):
File "test.py", line 14, in <module>
ydl = youtube_dl.YoutubeDL(opts)
File "C:\Users\User\AppData\Local\Programs\Python\Python38\lib\site-packages\youtube_dl\YoutubeDL.py", line 429, in __init__
pp_class = get_postprocessor(pp_def_raw['key'])
KeyError: 'key'
And none of the examples told me anything about a key I have to pass, nor can I find anything about it, what am I doing wrong?

In a nutshell, try this:
'postprocessors': [{
'key':'FFmpegMetadata',
'preferredcodec': 'mp4'
}]
There are lots of keys, so if it's not working, check out this: https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/postprocessor/__init__.py
The trick is that the keys listed inside the file are not usable because you gotta drop the "PP" part!!
FYI:
It seems you are trying to get video? Then you should try the documented style: https://github.com/ytdl-org/youtube-dl/
You have to look at the "Functions" yourself and guess what parameters you should put here: https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/YoutubeDL.py In fact, somewhere in this link says, "check out the init.py for key information."

Related

Retrieve data from Tuya sdk

I use that guide to retrieve data from my Fingerbot: https://www.home-assistant.io/integrations/tuya/
I also use that repo to extract the data:
https://github.com/redphx/tuya-local-key-extractor
When I use the extract.py file I get that error:
Exception in thread Thread-1:
Traceback (most recent call last):
openapi <tuya_iot.openapi.TuyaOpenAPI object at 0x7f52fff924c0>
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
openmq <TuyaOpenMQ(Thread-1, started 139994399807232)>
self.run()
File "/home/xavier/.local/lib/python3.8/site-packages/tuya_iot/openmq.py", line 158, in run
self.__run_mqtt()
File "/home/xavier/.local/lib/python3.8/site-packages/tuya_iot/openmq.py", line 164, in __run_mqtt
mq_config = self._get_mqtt_config()
File "/home/xavier/.local/lib/python3.8/site-packages/tuya_iot/openmq.py", line 67, in _get_mqtt_config
"uid": self.api.token_info.uid,
AttributeError: 'NoneType' object has no attribute 'uid'
Traceback (most recent call last):
File "./extract.py", line 41, in <module>
device_manager.update_device_list_in_smart_home()
File "/home/xavier/.local/lib/python3.8/site-packages/tuya_iot/device.py", line 239, in update_device_list_in_smart_home
response = self.api.get(f"/v1.0/users/{self.api.token_info.uid}/devices")
AttributeError: 'NoneType' object has no attribute 'uid'
Here is the code I use:
import json
import logging
import os
from config import (
ENDPOINT,
APP,
EMAIL,
PASSWORD,
ACCESS_ID,
ACCESS_KEY,
)
from tuya_iot import (
TuyaOpenAPI,
AuthType,
TuyaOpenMQ,
TuyaDeviceManager,
)
openapi = TuyaOpenAPI(ENDPOINT, ACCESS_ID, ACCESS_KEY, AuthType.SMART_HOME)
openapi.connect(EMAIL, PASSWORD, country_code=84, schema=APP.value)
openmq = TuyaOpenMQ(openapi)
openmq.start()
print('openapi {}'.format(openapi))
print('openmq {}'.format(openmq))
device_manager = TuyaDeviceManager(openapi, openmq)
device_manager.update_device_list_in_smart_home()
devices = []
for tuya_device in device_manager.device_map.values():
device = {
'device_id': tuya_device.id,
'device_name': tuya_device.name,
'product_id': tuya_device.product_id,
'product_name': tuya_device.product_name,
'category': tuya_device.category,
'uuid': tuya_device.uuid,
'local_key': tuya_device.local_key,
}
try:
resp = openapi.get('/v1.0/iot-03/devices/factory-infos?device_ids={}'.format(tuya_device.id))
factory_info = resp['result'][0]
if 'mac' in factory_info:
mac = ':'.join(factory_info['mac'][i:i + 2] for i in range(0, 12, 2))
device['mac_address'] = mac
except Exception as e:
print(e)
devices.append(device)
print(json.dumps(devices, indent=2))
os._exit(0)
I am on Ubuntu 20.04 and I install the python sdk with tha t command:
pip3 install tuya-iot-py-sdk

How can i import keccak with python3?

i am trying to run the following python3 script however i am struggling to get keccak working:
import sha3
import requests
import time
SAFE_ADDRESS = '0x***'
OWNER_PRIVATE_KEY = '****'
DELEGATE_ADDRESS = '****'
TX_SERVICE_BASE_URL = 'https://safe-transaction.rinkeby.gnosis.io'
totp = int(time.time()) // 3600
hash_to_sign = keccak(text=DELEGATE_ADDRESS + str(totp))
account = Account.from_key(OWNER_PRIVATE_KEY)
signature = account.signHash(hash_to_sign)
add_payload = {
"safe": SAFE_ADDRESS,
"delegate": DELEGATE_ADDRESS,
"signature": signature.signature.hex(),
"label": "My new delegate2"
}
add_response = requests.post(f'{TX_SERVICE_BASE_URL}/api/v1/safes/{SAFE_ADDRESS}/delegates/', json=add_payload, headers = {'Content-type': 'application/json'})
print(add_response.text)
print(add_response.status_code)
i get the following error:
Traceback (most recent call last):
File "test.py", line 14, in <module>
hash_to_sign = keccak(text=DELEGATE_ADDRESS + str(totp))
NameError: name 'keccak' is not defined
additionally i have tried to replace import sha3 with from Crypto.Hash import keccak
but this time i get this error:
Traceback (most recent call last):
File "test.py", line 14, in <module>
hash_to_sign = keccak(text=DELEGATE_ADDRESS + str(totp))
TypeError: 'module' object is not callable
how can i fix this?

How to pass multiple parameters for aiohttp request wit the same name?

The API from which I want to receive data takes several values with the same name as a parameter:
https://example.com/method?types=Retail&types=Office&types=Industrial......
I try to do it like this:
params_api_sale = {
'types': [
'Retail', 'Office', 'Industrial',
'Mixed Use', 'Development Site',
'Land', 'Special Purpose', 'Other'
],
'sortDirection': 'Descending',
'sortOrder': 'ActivatedOn'
}
async def get_items():
async with aiohttp.request(
method='GET', url=vars_.url_api_sale,
headers=vars_.headers, params=vars_.params_api_sale) as response:
...
But this gives me an error:
Traceback (most recent call last):
File "foo.py", line 85, in <module>
asyncio.get_event_loop().run_until_complete(main())
File "C:\Python37\lib\asyncio\base_events.py", line 583, in run_until_complete
return future.result()
File "foo.py", line 80, in main
await asyncio.create_task(get_items())
File "foo.py", line 44, in get_items
headers=vars_.headers, params=vars_.params_api_sale) as response:
File "D:\Dev\venv\lib\site-packages\aiohttp\client.py", line 1051, in __aenter__
self._resp = await self._coro
File "D:\Dev\venv\lib\site-packages\aiohttp\client.py", line 473, in _request
ssl=ssl, proxy_headers=proxy_headers, traces=traces)
File "D:\Dev\venv\lib\site-packages\aiohttp\client_reqrep.py", line 263, in __init__
url2 = url.with_query(params)
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 922, in with_query
new_query = self._get_str_query(*args, **kwargs)
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 886, in _get_str_query
quoter(k) + "=" + quoter(self._query_var(v)) for k, v in query.items()
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 886, in <genexpr>
quoter(k) + "=" + quoter(self._query_var(v)) for k, v in query.items()
File "D:\Dev\venv\lib\site-packages\yarl\__init__.py", line 864, in _query_var
"of type {}".format(v, type(v))
TypeError: Invalid variable type: value should be str or int, got ['Retail', 'Office', 'Industrial', 'Mixed Use', 'Development Site', 'Land', 'Special Purpose', 'Other'] of type <class 'list'>
I have not found a documented way to do this. All that comes to my mind is to generate a URL string with these parameters, of course, if there is no more correct way.
import requests
params = {
'types': [
'Retail', 'Office', 'Industrial',
'Mixed Use', 'Development Site',
'Land', 'Special Purpose', 'Other'
],
'sortDirection': 'Descending',
'sortOrder': 'ActivatedOn'
}
r = requests.get("http://www.test.com/", params=params)
print(r.url)
output:
https://www.test.com/?types=Retail&types=Office&types=Industrial&types=Mixed+Use&types=Development+Site&types=Land&types=Special+Purpose&types=Other&sortDirection=Descending&sortOrder=ActivatedOn
For aiohttp you can use MultiDict which is also included in the official document.
For sending data with multiple values for the same key MultiDict may be used as well.
As stated at aiohttp documentation, you can pass a list of tuples to params with the same key to achieve your goal:
Also it is possible to pass list of 2 items tuples as parameters, in that case you can specifiy multiple values for each key:
params = [
("key1", "value1"),
("key1", "value2"),
("key2", "value3"),
]

Scrapy error catching in scrapy/middleware.py file: TypeError: __init__() missing 1 required positional argument: 'uri'

I am catching this error while starting a crawl. I have searched for an answer in several forums, and looked at the code in scrapy/middleware.py (came standard with scrapy and I have not altered it) and cannot figure out why I am getting an error.
The scraper is using both an ImagesPipeline and S3FilesStore pipeline for storing a json file and downloaded images directly into different S3 folders. I am using Python 3.6.
Any help is appreciated. The error message and my scraper settings are below, please let me know if anything else would be useful.
Traceback (most recent call last):
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/crawler.py", line 77, in crawl
self.engine = self._create_engine()
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/crawler.py", line 102, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/middleware.py", line 58, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/Users/user/anaconda/envs/python3/lib/python3.6/site-
packages/scrapy/middleware.py", line 40, in from_settings
mw = mwcls()
TypeError: __init__() missing 1 required positional argument: 'uri'
ITEM_PIPELINES = {
'scrapy.pipelines.files.S3FilesStore': 1,
'scrapy.pipelines.images.ImagesPipeline': 1
}
AWS_ACCESS_KEY_ID = 'xxxxxx'
AWS_SECRET_ACCESS_KEY= 'xxxxxx'
IMAGES_STORE = 's3 path'
FEED_URI = 's3 path'
FEED_FORMAT = 'jsonlines'
FEED_EXPORT_FIELDS = None
FEED_STORE_EMPTY = False
FEED_STORAGES = {}
FEED_STORAGES_BASE = {
'': 'scrapy.extensions.feedexport.FileFeedStorage',
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
}
FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',
'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
'jl': None,
'csv': None,
'xml': None,
'marshal': None,
'pickle': None,
}

Google Adwords Traffic Estimator Service and Python

I've downloaded a code sample that looks for particular keywords and pulls some metrics. I've noticed a lot of Google Adwords API examples are compliant with python 3.x so I'm wondering if there is an issue with that? See below for code sample:
from googleads import adwords
def main(client):
# Initialize appropriate service.
traffic_estimator_service = client.GetService(
'TrafficEstimatorService', version='v201609')
# Construct selector object and retrieve traffic estimates.
keywords = [
{'text': 'mars cruise', 'matchType': 'BROAD'},
{'text': 'cheap cruise', 'matchType': 'PHRASE'},
{'text': 'cruise', 'matchType': 'EXACT'}
]
negative_keywords = [
{'text': 'moon walk', 'matchType': 'BROAD'}
]
keyword_estimate_requests = []
for keyword in keywords:
keyword_estimate_requests.append({
'keyword': {
'xsi_type': 'Keyword',
'matchType': keyword['matchType'],
'text': keyword['text']
}
})
for keyword in negative_keywords:
keyword_estimate_requests.append({
'keyword': {
'xsi_type': 'Keyword',
'matchType': keyword['matchType'],
'text': keyword['text']
},
'isNegative': 'true'
})
# Create ad group estimate requests.
adgroup_estimate_requests = [{
'keywordEstimateRequests': keyword_estimate_requests,
'maxCpc': {
'xsi_type': 'Money',
'microAmount': '1000000'
}
}]
# Create campaign estimate requests.
campaign_estimate_requests = [{
'adGroupEstimateRequests': adgroup_estimate_requests,
'criteria': [
{
'xsi_type': 'Location',
'id': '2840' # United States.
},
{
'xsi_type': 'Language',
'id': '1000' # English.
}
],
}]
# Create the selector.
selector = {
'campaignEstimateRequests': campaign_estimate_requests,
}
# Optional: Request a list of campaign-level estimates segmented by
# platform.
selector['platformEstimateRequested'] = True
# Get traffic estimates.
estimates = traffic_estimator_service.get(selector)
campaign_estimate = estimates['campaignEstimates'][0]
# Display the campaign level estimates segmented by platform.
if 'platformEstimates' in campaign_estimate:
platform_template = ('Results for the platform with ID: "%d" and name: '
'"%s".')
for platform_estimate in campaign_estimate['platformEstimates']:
platform = platform_estimate['platform']
DisplayEstimate(platform_template % (platform['id'],
platform['platformName']),
platform_estimate['minEstimate'],
platform_estimate['maxEstimate'])
# Display the keyword estimates.
if 'adGroupEstimates' in campaign_estimate:
ad_group_estimate = campaign_estimate['adGroupEstimates'][0]
if 'keywordEstimates' in ad_group_estimate:
keyword_estimates = ad_group_estimate['keywordEstimates']
keyword_template = ('Results for the keyword with text "%s" and match '
'type "%s":')
keyword_estimates_and_requests = zip(keyword_estimates,
keyword_estimate_requests)
for keyword_tuple in keyword_estimates_and_requests:
if keyword_tuple[1].get('isNegative', False):
continue
keyword = keyword_tuple[1]['keyword']
keyword_estimate = keyword_tuple[0]
DisplayEstimate(keyword_template % (keyword['text'],
keyword['matchType']),
keyword_estimate['min'], keyword_estimate['max'])
def _CalculateMean(min_est, max_est):
if min_est and max_est:
return (float(min_est) + float(max_est)) / 2.0
else:
return None
def _FormatMean(mean):
if mean:
return '%.2f' % mean
else:
return 'N/A'
def DisplayEstimate(message, min_estimate, max_estimate):
"""Displays mean average cpc, position, clicks, and total cost for estimate.
Args:
message: str message to display for the given estimate.
min_estimate: sudsobject containing a minimum estimate from the
TrafficEstimatorService response.
max_estimate: sudsobject containing a maximum estimate from the
TrafficEstimatorService response.
"""
# Find the mean of the min and max values.
mean_avg_cpc = (_CalculateMean(min_estimate['averageCpc']['microAmount'],
max_estimate['averageCpc']['microAmount'])
if 'averageCpc' in min_estimate else None)
mean_avg_pos = (_CalculateMean(min_estimate['averagePosition'],
max_estimate['averagePosition'])
if 'averagePosition' in min_estimate else None)
mean_clicks = _CalculateMean(min_estimate['clicksPerDay'],
max_estimate['clicksPerDay'])
mean_total_cost = _CalculateMean(min_estimate['totalCost']['microAmount'],
max_estimate['totalCost']['microAmount'])
print (message)
print ('Estimated average CPC: %s' % _FormatMean(mean_avg_cpc))
print ('Estimated ad position: %s' % _FormatMean(mean_avg_pos))
print ('Estimated daily clicks: %s' % _FormatMean(mean_clicks))
print ('Estimated daily cost: %s' % _FormatMean(mean_total_cost))
if __name__ == '__main__':
# Initialize client object.
adwords_client = adwords.AdWordsClient.LoadFromStorage()
main(adwords_client)
Here is the error message:
(Money) not-found
path: "Money", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Location) not-found
path: "Location", not-found
(Language) not-found
path: "Language", not-found
<suds.sax.document.Document object at 0x03BF1D10>
Server raised fault in response.
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 82, in send
fp = self.u2open(u2request)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 132, in u2open
return url.open(u2request, timeout=tm)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 472, in open
response = meth(req, response)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 582, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 510, in error
return self._call_chain(*args)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 444, in _call_chain
result = func(*args)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 590, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 500: Internal Server Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 613, in send
reply = self.options.transport.send(request)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 94, in send
raise TransportError(e.msg, e.code, e.fp)
suds.transport.TransportError: Internal Server Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\adwords test - Copy (2).py", line 177, in <module>
main(adwords_client)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\adwords test - Copy (2).py", line 95, in main
estimates = traffic_estimator_service.get(selector)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\googleads\common.py", line 696, in MakeSoapRequest
raise e
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\googleads\common.py", line 692, in MakeSoapRequest
for arg in args])
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 521, in __call__
return client.invoke(args, kwargs)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 581, in invoke
result = self.send(soapenv)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 619, in send
description=tostr(e), original_soapenv=original_soapenv)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 670, in process_reply
raise WebFault(fault, replyroot)
suds.WebFault: Server raised fault: '[AuthenticationError.CLIENT_CUSTOMER_ID_IS_REQUIRED # ; trigger:'<null>']'
You should set client_customer_id in your googleads.yaml file . you can get client_customer_id from your manager account. go to your manager account and add client then copy your id from upright corner of screen. in googleads.yaml file paste that id to client_customer_id .

Resources