Getting HttpError 401 while generating google Admob network report using python script - python-3.x

Below is the code that I am using to generate the Admob network report
from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
import os
base_path=os.path.dirname(os.path.realpath(__file__))
scopes=['https://www.googleapis.com/auth/admob.report']
key_file_location = base_path+'/config/service_account.json'
credentials = ServiceAccountCredentials.from_json_keyfile_name(key_file_location, scopes)
account_id='accounts/pub-XXXXXXXXXXXXXXXX'
network_report_filter = {
'dateRange': {
'startDate': {'year': 2020, 'month': 1, 'day': 1},
'endDate': {'year': 2020, 'month': 2, 'day': 10}
},
'dimensions': ['DATE', 'APP', 'COUNTRY'],
'metrics': ['CLICKS', 'ESTIMATED_EARNINGS'],
'dimensionFilters': [
{
'dimension': 'COUNTRY',
'matchesAny': {'values': [{'value': 'US', 'value': 'CN'}]}
}
],
'sortConditions': [
{'dimension':'APP', 'order': 'ASCENDING'},
{'metric':'CLICKS', 'order': 'DESCENDING'}
],
'localizationSettings': {
'currencyCode': 'USD',
'languageCode': 'en-US'
}
}
# Build the service object.
admob = build('admob', 'v1', credentials=credentials)
admob._resourceDesc=network_report_filter
accounts=admob.accounts()
network_report=accounts.networkReport().generate(parent=account_id)
data=network_report.execute()
It throws the below error
*** HttpError: https://admob.googleapis.com/v1/accounts/pub-XXXXXXXXXXXXXXXX/networkReport:generate?alt=json returned "Request is missing required authentication credential. Expected OAuth 2 access token, login cookie or other valid authentication credential. See https://developers.google.com/identity/sign-in/web/devconsole-project.">
I have generated the service account credentials with Admob API is enabled.
But not able to figure out why there is authentication error.

The main issue, that code above tries to use the service account to query the api. But, it's not supported. It could be queried with OAuth2.0 Client Id.
The steps to generate OAth2.0 Client ID:
Open the credential page of your project (https://console.cloud.google.com/apis/credentials?project=REPLACE_WITH_YOUR_PROJECT_ID);
Generate OAuth2.0 Client ID;
Download the generated json file;
Use it with code below;
The following works well for me:
Libs:
pip3 install --upgrade google-api-python-client --user
pip3 install --upgrade oauth2client --user
Code example:
import csv
import sys
from googleapiclient import discovery
from googleapiclient.http import build_http
from oauth2client import tools
from oauth2client.file import Storage
from oauth2client.client import OAuth2WebServerFlow
class AdMobAPI:
def __init__(self):
scope = 'https://www.googleapis.com/auth/admob.report'
name = 'admob'
version = 'v1'
flow = OAuth2WebServerFlow(client_id='<todo: replace with a client_id from the secret json>',
client_secret='<todo: replace with a secret from the secret json>',
scope=scope)
storage = Storage(name + '.dat')
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = tools.run_flow(flow, storage)
http = credentials.authorize(http=build_http())
self.admob = discovery.build(name, version, http=http)
# Convert to the list of dictionaries
def report_to_list_of_dictionaries(self, response):
result = []
for report_line in response:
if report_line.get('row'):
print(report_line)
row = report_line.get('row')
dm = {}
if row.get('dimensionValues'):
for key, value in row.get('dimensionValues').items():
if value.get('value') and value.get('displayLabel'):
dm.update({key: value.get('value')})
dm.update({key + '_NAME': value.get('displayLabel')})
else:
dm.update({key: next(filter(None, [value.get('value'), value.get('displayLabel')]))})
if row.get('metricValues'):
for key, value in row.get('metricValues').items():
dm.update({key: next(filter(None, [value.get('value'), value.get('microsValue'), value.get('integerValue')]))})
result.append(dm)
return result
def generate_report(self, publisher_id):
date_range = {'startDate': {'year': 2020, 'month': 4, 'day': 1},
'endDate': {'year': 2020, 'month': 4, 'day': 1}}
dimensions = ['DATE', 'APP', 'PLATFORM', 'COUNTRY']
metrics = ['ESTIMATED_EARNINGS', 'IMPRESSIONS', 'CLICKS',
'AD_REQUESTS', 'MATCHED_REQUESTS']
sort_conditions = {'dimension': 'DATE', 'order': 'DESCENDING'}
report_spec = {'dateRange': date_range,
'dimensions': dimensions,
'metrics': metrics,
'sortConditions': [sort_conditions]}
request = {'reportSpec': report_spec}
return self.admob.accounts().networkReport().generate(
parent='accounts/{}'.format(publisher_id),
body=request).execute()
api = AdMobAPI()
raw_report = api.generate_report('<todo: replace with publisher id, smth like pub-[0-9]+>')
report_as_list_of_dictionaries = api.report_to_list_of_dictionaries(raw_report)
# Convert to CSV
dict_writer = csv.DictWriter(sys.stdout, report_as_list_of_dictionaries[0].keys())
dict_writer.writeheader()
dict_writer.writerows(report_as_list_of_dictionaries)

Currently, google admob api does not support service accounts
for more details, see here enter link description here

Related

Elasticsearch "register repo" command returns 500 error code but syntax matches the doc (I think)

I have this Python code to register a Google Cloud Storage (GCS) repository:
import requests
from grabconfig import grabconfig
(HOSTS, ign) = grabconfig()
reqHeaders = {'content-type' : 'application/json'}
for h in HOSTS:
url = f'http://{h}:9200'
r = requests.put(f'{url}/_snapshot/prod_backup2',
'''{ \"type\" : \"gcs\" }, { \"settings\" : { \"client\" : \"secondary\", \"bucket\" : \"prod_backup2\" },
{ \"compress\" : \"true\" }}''',
headers=reqHeaders)
print(r)
print(r.json())
r2 = requests.get(f'{url}/_cat/snapshots')
print(r2)
print(r2.json())
The configuration file I am using is the prod.py one:
HOSTS = ['10.x.x.x']
BACKUP_REPO = ['prod_backup2']
But when I run the code I get this error, always:
<Response [500]>
{'error': {'root_cause': [{'type': 'repository_exception', 'reason': '[prod_backup2] repository type [gcs] does not exist'}], 'type': 'repository_exception', 'reason': '[prod_backup2] repository type [gcs] does not exist'}, 'status': 500}
I think I found it: the gcs plugin was not installed on the server I was targeting.
That's supposed to be fixed by Monday, so I'm on to the next task.

Why the error occured when i tried to send a POST request with QWebEngineHttpRequest (PyQt5)?

I try to get JSON Object from source, but getting only errors.
Now I'm trying to understand what I'm doing wrong.
This code works well with other resources.
Maybe It`s not problem with code, but with web-source.
import sys
import json
from PyQt5.QtCore import QByteArray, QUrl
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWebEngineCore import QWebEngineHttpRequest
from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineSettings
from pprint import pprint
def on_load_finished():
"""Handle response"""
engine.toPlainText(handle_to_html) # get json
def handle_to_html(html):
"""Handle html"""
print(html)
QApplication.quit()
if __name__ == "__main__":
app = QApplication(sys.argv)
engine = QWebEnginePage()
url = "https://sportsapi.betway.com/api/Events/V2/GetEvents"
request = QWebEngineHttpRequest(url=QUrl(url), method=QWebEngineHttpRequest.Post)
request.setHeader(QByteArray(b"Content-Type"), QByteArray(b"application/json"))
payload = {
"LanguageId": 1,
"ClientTypeId": 2,
"BrandId": 3,
"JurisdictionId": 1,
"ClientIntegratorId": 1,
"ExternalIds": [9032262, 9038528, 9037778],
"MarketCName": "win-draw-win",
"ScoreboardRequest": {"ScoreboardType": 3, "IncidentRequest": {}},
"BrowserId": 3,
"OsId": 3,
"ApplicationVersion": "",
"BrowserVersion": "97.0.4692.99",
"OsVersion": "NT 10.0",
"SessionId": "null",
"TerritoryId": 227,
"CorrelationId": "06779075-21e2-4ba8-8e91-d71a981621fe",
"VisitId": "d1088cdf-13a8-42d0-be90-b34fd1332c36",
"ViewName": "sports",
"JourneyId": "833a4c0c-3354-499f-9d52-949df6d159f9",
}
request.setPostData(bytes(json.dumps(payload), "utf-8"))
engine.load(request)
engine.loadFinished.connect(on_load_finished)
app.exec_()
Errors are looks like
This XML file does not appear to have any style information associated with it. The document tree is shown below.
<Error>
<Message>An error has occurred.</Message>
</Error>
Maybe It`s not problem with code, but with web-source.

AttributeError: module 'google.cloud.dialogflow' has no attribute 'types'

I am building a telegram-bot and using Dialogflow in it, I am getting the following error :
2021-11-19 23:26:46,936 - __main__ - ERROR - Update '{'message': {'entities': [],
'delete_chat_photo': False, 'text': 'hi', 'date': 1637344606, 'new_chat_members': [],
'channel_chat_created': False, 'message_id': 93, 'photo': [], 'group_chat_created':
False, 'supergroup_chat_created': False, 'new_chat_photo': [], 'caption_entities': [],
'chat': {'id': 902424541, 'type': 'private', 'first_name': 'Akriti'},
'from': {'first_name': 'Akriti', 'id': 902424541, 'is_bot': False, 'language_code': 'en'}
}, 'update_id': 624230278}' caused error 'module 'google.cloud.dialogflow' has no
attribute 'types''
It appears there is some issue with the Dialogflow attribute "types", but I don't know what I am doing wrong.
Here is the code where I am using it:
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "client.json"
from google.cloud import dialogflow
dialogflow_session_client = dialogflow.SessionsClient()
PROJECT_ID = "get-informed-ufnl"
from gnewsclient import gnewsclient
client = gnewsclient.NewsClient()
def detect_intent_from_text(text, session_id, language_code='en'):
session = dialogflow_session_client.session_path(PROJECT_ID, session_id)
text_input = dialogflow.types.TextInput(text=text, language_code=language_code)
query_input = dialogflow.types.QueryInput(text=text_input)
response = dialogflow_session_client.detect_intent(session=session, query_input=query_input)
return response.query_result
def get_reply(query, chat_id):
response = detect_intent_from_text(query, chat_id)
if response.intent.display_name == 'get_news':
return "get_news", dict(response.parameters)
else:
return "small_talk", response.fulfillment_text
def fetch_news(parameters):
client.language = parameters.get('language')
client.location = parameters.get('geo-country')
client.topic = parameters.get('topic')
return client.get_news()[:5]
topics_keyboard = [
['Top Stories', 'World', 'Nation'],
['Business', 'Technology', 'Entertainment'],
['Sports', 'Science', 'Health']
]
I figured it out, the problem lies in the import statement. The correct module name should be:
import google.cloud.dialogflow_v2 as dialogflow
I recommend to deactivate your current error handler or use one similar to this example such that you can see the full traceback of the exception :)
Disclaimer: I'm currently the maintainer of python-telegram-bot

issue while creating VM instance using python code in GCP

I am trying to write a code which will read values from excel file and will create VMs in Google Cloud. I am facing problem at two locations, while creating tags if I use 'items': [tag] or while creating service account scope it starts giving me error.
import os, json
import googleapiclient.discovery
from google.oauth2 import service_account
import csv
credentials = service_account.Credentials.from_service_account_file('G:/python/json/mykids-280210.json')
compute = googleapiclient.discovery.build('compute', 'v1', credentials=credentials)
def create_instance(compute, vm_name, image_project, image_family, machinetype, startupscript, zone, network,
subnet, project, scope, tag):
# Get the latest Debian Jessie image.
image_response = compute.images().getFromFamily(
project=image_project, family=image_family).execute()
source_disk_image = image_response['selfLink']
# Configure the machine
machine_type = "zones/" + zone + "/machineTypes/" + machinetype
startup_script = startupscript
config = {
'name': vm_name,
'machineType': machine_type,
'description': 'This VM was created with python code',
'tags': {
'items': ['external', 'home', 'local'] #'items': [tag] <~~~~~~~~~~~
},
'deletionProtection': False,
'labels': {'env': 'dev', 'server': 'mytower', 'purpose': 'personal'},
# Specify the boot disk and the image to use as a source.
'disks': [
{
'boot': True,
'autoDelete': True,
'initializeParams': {
'sourceImage': source_disk_image,
}
}
],
# Specify a network interface with NAT to access the public
# internet.
'networkInterfaces': [{
'network': 'global/networks/' + network,
'subnetwork': 'regions/us-central1/subnetworks/' + subnet,
'accessConfigs': [
{'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'}
]
}],
# Allow the instance to access cloud storage and logging.
'serviceAccounts': [{
'email': 'default',
'scopes': [
#'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write'
#scope # scope <~~~~~~~~~~~~~~~~~~~~
]
}],
'scheduling': {
"preemptible": True
},
# Metadata is readable from the instance and allows you to
# pass configuration from deployment scripts to instances.
'metadata': {
'items': [{
# Startup script is automatically executed by the
# instance upon startup.
'key': 'startup-script',
'value': startup_script
}]
}
}
return compute.instances().insert(
project=project,
zone=zone,
body=config).execute()
# [END create_instance]
with open('vms.csv', newline='') as csvfile:
data = csv.DictReader(csvfile)
for row in data:
vm_name = row['vm_name']
image_project = row['image_project']
image_family = row['image_family']
machinetype = row['machinetype']
startupscript = row['startupscript']
zone = row['zone']
network = row['network']
subnet = row['subnet']
project = row['project']
scope = row['scopes']
tag = row['tags']
print(create_instance(compute, vm_name, image_project, image_family, machinetype, startupscript, zone, network,
subnet, project, scope, tag))
csvfile.close()
error when use scope variable
G:\python\pythonProject\venv\Scripts\python.exe G:/python/pythonProject/read-excel-gcp/vm/create_vm.py
Traceback (most recent call last):
File "G:\python\pythonProject\read-excel-gcp\vm\create_vm.py", line 100, in <module>
print(create_instance(compute, vm_name, image_project, image_family, machinetype, startupscript, zone, network,
File "G:\python\pythonProject\read-excel-gcp\vm\create_vm.py", line 79, in create_instance
return compute.instances().insert(
File "G:\python\pythonProject\venv\lib\site-packages\googleapiclient\_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "G:\python\pythonProject\venv\lib\site-packages\googleapiclient\http.py", line 915, in execute
raise HttpError(resp, content, uri=self.uri)
googleapiclient.errors.HttpError: <HttpError 400 when requesting https://compute.googleapis.com/compute/v1/projects/mykids-280210/zones/us-central1-a/instances?alt=json returned "One or more of the service account scopes are invalid: 'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write''". Details: "One or more of the service account scopes are invalid: 'https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write''">
Process finished with exit code 1
I get a similar error when I use tag variable.
I have # the value the way I am passing in the above code.
Below is my csv file details
vm_name,image_project,image_family,machinetype,startupscript,zone,network,subnet,project,scopes,tags
python-vm1,debian-cloud,debian-9,e2-micro,G:/python/json/startup-script.sh,us-central1-a,myvpc,subnet-a,mykids-280210,"https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write'","external', 'home', 'local'"
python-vm2,debian-cloud,debian-9,e2-micro,G:/python/json/startup-script.sh,us-central1-a,myvpc,subnet-a,mykids-280210,"https://www.googleapis.com/auth/devstorage.read_write', 'https://www.googleapis.com/auth/logging.write'","external', 'home', 'local'"
I am not sure that when the value are passed directly it works, but when passing the value through variable, it fails.
I have marked the problem area with <~~~~~~~~~~~~
Please suggest if anyone understands the issue.
#d.s can you try changing your scope format to something like this:
'serviceAccounts': [
{
'email': 'default'
'scopes':[
'https://www.googleapis.com/auth/compute',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/monitoring.write',
'https://www.googleapis.com/auth/trace.append',
'https://www.googleapis.com/auth/devstorage.read_write']}]
The listed scopes are the default scopes that you will need for an instance. I think the problem you are facing is you where trying to only list two scopes which are not enough to allow you to deploy your instance.

Google Cloud logging, Python3.8 standard environment, group request related logs by trace id

I stucked with problem during Google Cloud Logging setup for Python3.8 in Google App Engine Standard environment.
I'm using FastAPI with unicorn. My code logging configuration:
import logging.config
import sys
from google.cloud import logging as google_logging
from app.settings import ENV, _settings
if _settings.ENV == ENV.LOCAL:
MAIN_LOGGER = 'console'
LOGGER_CONF_DICT = {
'class': 'logging.StreamHandler',
'formatter': 'verbose',
'stream': sys.stdout,
'level': _settings.LOG_LEVEL.upper(),
}
else:
log_client = google_logging.Client()
MAIN_LOGGER = 'stackdriver_logging'
LOGGER_CONF_DICT = {
'class': 'app.gcloud_logs.GCLHandler',
'client': log_client,
'name': 'appengine.googleapis.com%2Frequest_log'
# I've tried other names: stdout, %2FA instead of / symbol, appengine.googleapis.com/stdout
# the same result or no logs at all
}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'verbose': {
'format': '%(log_color)s%(asctime)s [%(levelname)s] [%(name)s] %(message)s (%(filename)s:%(lineno)d)',
'()': 'colorlog.ColoredFormatter',
'log_colors': {
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'bold_red',
},
}
},
'handlers': {
MAIN_LOGGER: {**LOGGER_CONF_DICT},
'blackhole': {'level': 'DEBUG', 'class': 'logging.NullHandler'},
},
'loggers': {
'fastapi': {'level': 'INFO', 'handlers': [MAIN_LOGGER]},
'uvicorn.error': {'level': 'INFO', 'handlers': [MAIN_LOGGER], 'propagate': False},
'uvicorn.access': {'level': 'INFO', 'handlers': [MAIN_LOGGER], 'propagate': False},
'uvicorn': {'level': 'INFO', 'handlers': [MAIN_LOGGER], 'propagate': False},
'google.cloud.logging.handlers.transports.background_thread': {'level': 'DEBUG', 'handlers': ['blackhole'],
'propagate': False},
'': {
'level': _settings.LOG_LEVEL.upper(),
'handlers': [MAIN_LOGGER],
'propagate': True,
},
}
}
logging.config.dictConfig(LOGGING)
And my logging handler code:
import os
from typing import Any, Dict, Optional
from google.cloud.logging.handlers import CloudLoggingHandler
from google.cloud.logging.resource import Resource
from starlette.requests import Request
from starlette_context import context
from app.settings import _settings
class GCLHandler(CloudLoggingHandler):
def emit(self, record):
message = super(GCLHandler, self).format(record)
request: Optional[Request] = None
trace: Optional[str] = None
span_id: Optional[str] = None
user_id: Optional[int] = None
resource = Resource(
type='gae_app',
labels={
'module_id': os.environ['GAE_SERVICE'],
'project_id': _settings.PROJECT_NAME,
'version_id': os.environ['GAE_VERSION'],
'zone': 'us16' # tried without zone - the same result
}
)
labels: Dict[str, Any] = {}
if context.exists(): # I'm sure that it works
request = context.get('request') # I'm sure that it works
user_id = context.get('user_id') # I'm sure that it works
if user_id is not None:
labels['user_id'] = user_id
if request:
if request.headers.get('X-Cloud-Trace-Context'):
cloud_trace = request.headers.get('X-Cloud-Trace-Context').split('/')
if len(cloud_trace) > 1:
span_id = cloud_trace[1].split(';')[0]
trace = f'projects/{_settings.PROJECT_NAME}/traces/{cloud_trace[0]}'
labels['logging.googleapis.com/trace'] = cloud_trace[0] # Found in some guides, not sure that its neccessary
labels['appengine.googleapis.com/trace_id'] = cloud_trace[0] # Found in some guides, not sure that its neccessary
self.transport.send(
record,
message,
resource=resource,
labels=labels,
trace=trace,
span_id=span_id
)
I've got some strange results in logs viewer that my log has the same trace as request log, but they're not grouped
Any ideas?
There are 2 types of logs in App Engine :
Request log: A log of the requests that are sent to your app. App Engine automatically creates entries in the request log.
App log: log entries that you write to a supported framework or file as described on this page.
The both logs are send to the Cloud Logging Agent automatically by App Engine Standard.
On first request, app logs and request logs are not correlated and that's why there are not shown in a group , this is a known issue stated in App Engine Official Documentation. However in the second request, you can see that the logs are shown in a group.
A feature request in Public Issue Tracker has already been created for this behavior where you will get all the updates regarding the fix.
log_client = google_logging.Client()
MAIN_LOGGER = 'stackdriver_logging'
LOGGER_CONF_DICT = {
'class': 'app.gcloud_logs.GCLHandler',
'client': log_client,
'name': 'app'
}
Change name to app is helped

Resources