I'm trying to create a cloud function which goes to the certain google sheet and collects necessary data, after that, it connects with bigquery database and writes down data to a bigquery table. When I run apps script, which trigers cloud function, I get next message - Error: could not handle the request
The code from the cloud function (main.py)
from __future__ import print_function
import json
import os.path
import pickle
import functions_framework
import requests
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from google.auth.transport.requests import Request
from google.cloud import bigquery
class GoogleSheetService:
# The settings of the particular google table
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']
TOKEN_PICKLE = 'settings/token.pickle'
service = None
# The settings for the bigquery service
credentials_path = 'settings/pythonbq.privateKey.json'
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_path
client = bigquery.Client()
table_id = 'cobalt-alliance-365419.BTC_Daily.table-btc'
DATA = []
def __init__(self):
creds = None
if os.path.exists(self.TOKEN_PICKLE):
with open(self.TOKEN_PICKLE, 'rb') as token:
creds = pickle.load(token)
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'settings/credentials.json', self.SCOPES)
creds = flow.run_local_server(port=0)
with open(self.TOKEN_PICKLE, 'wb') as token:
pickle.dump(creds, token)
self.service = build('sheets', 'v4', credentials=creds)
def get_data(self, spreadsheet_id, range_name):
sheet = self.service.spreadsheets()
result = sheet.values().get(spreadsheetId=spreadsheet_id, range=range_name).execute()
self.DATA = result.get('values', [])
def get_row(self, data_of_column):
r = []
for row in self.DATA:
if data_of_column == 'date':
r.append(row[1].replace('0:00:00', '').rstrip())
if data_of_column == 'symbol':
r.append(row[2])
if data_of_column == 'volume_BTC':
r.append(float(row[4]))
if data_of_column == 'volume_USD':
r.append(float(row[5]))
return r
def sample_data(self, row1=None, row2=None, row3=None, row4=None):
return {u'date': f'{row1}', u'symbol': f'{row2}', u'volume_BTC': f'{row3}', u'volume_USD': f'{row4}'}
def write_data(self):
rows_array = []
number_of_rows = len(self.DATA)
for i in range(number_of_rows):
rows_array.append(self.sample_data(self.get_row('date')[i], self.get_row('symbol')[i],
self.get_row('volume_BTC')[i], self.get_row('volume_USD')[i]))
return rows_array
def write_to_db(self):
rows_to_insert = self.write_data()
if not rows_to_insert:
return 'Data is empty'
errors = self.client.insert_rows_json(self.table_id, rows_to_insert)
if not errors:
return f'New rows have been added.'
else:
return f'Encountered errors while inserting rows: {errors}'
#functions_framework.http
def main(request):
gs = GoogleSheetService()
if requests.method == "GET":
gs.get_data('164RTnYK49DvV2Ion45JHMCFQa8S', 'A2:F100')
data_json = json.dumps(gs.DATA)
data = {'data_json': data_json}
return requests.get(data=data)
elif requests.method == "POST":
gs.get_data('164RTnYK49DvV2Ion45JHMCFQa8S', 'A2:F100')
gs.write_to_db()
Apps script
function callCloudRun() {
const token = ScriptApp.getIdentityToken();var options = {
'method' : 'post',
'headers': {'Authorization': 'Bearer ' + token},
};
options = {muteHttpExceptions: true};
var response = UrlFetchApp.fetch(CLOUD_RUN_URL, options);
Logger.log(response.getContentText());
}
Related
So I have this flask application and I'm trying to test it with pytest
app.py:
import os
from flask import Flask, jsonify
from flask_smorest import Api
from flask_jwt_extended import JWTManager
from flask_migrate import Migrate
from db import db
from resources.item import blp as ItemBlueprint
from resources.store import blp as StoreBlueprint
from resources.tag import blp as TagBlueprint
from resources.user import blp as UserBlueprint
from blocklist import BLOCKLIST
def create_app(db_url=None):
app = Flask(__name__)
app.config["PROPAGATE_EXCEPTIONS"] = True
app.config["API_TITLE"] = "Stores REST API"
app.config["API_VERSION"] = "v1"
app.config["OPENAPI_VERSION"] = "3.0.3"
app.config["OPENAPI_URL_PREFIX"] = "/"
app.config["OPENAPI_SWAGGER_UI_PATH"] = "/swagger-ui"
app.config["OPENAPI_SWAGGER_UI_URL"] = "https://cdn.jsdelivr.net/npm/swagger-ui-dist/"
app.config["SQLALCHEMY_DATABASE_URI"] = db_url or os.getenv("DATABASE_URL", "sqlite:///data.db")
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True
db.init_app(app)
api = Api(app)
migrate = Migrate(app, db)
app.config['JWT_SECRET_KEY'] = '69490938337699758397870296439802775085'
jwt = JWTManager(app)
#jwt.needs_fresh_token_loader
def token_not_fresh_callback(jwt_header, jwt_payload):
return (
jsonify(
{
"description": "The token is not fresh.",
"error": "fresh_token_required",
}
),
401,
)
#jwt.token_in_blocklist_loader
def check_if_token_in_blocklist(jwt_header, jwt_payload):
return jwt_payload['jti'] in BLOCKLIST
#jwt.revoked_token_loader
def revoked_token_callback(jwt_header, jwt_payload):
return (
jsonify(
{
"description": "The token has been revoked.",
"error": "token_revoked"
}
)
)
#jwt.additional_claims_loader
def add_claims_to_jwt(identity):
if identity == 1:
return {"is_admin": True}
return {"is_admin": False}
#jwt.expired_token_loader
def expired_token_callback(jwt_header, jwt_payload):
return (
jsonify(
{
"message": "The token has expired.",
"error": "token_expired"
}
),
401,
)
#jwt.invalid_token_loader
def invalid_token_callback(error):
return (
jsonify(
{
"message": "Signature verification failed.",
"error": "invalid_token"
}
), 401,
)
#jwt.unauthorized_loader
def missing_token_callback(error):
return (
jsonify(
{
"description": "Request does not contain an access token.",
"error": "authorization_required",
}
), 401,
)
# #app.before_first_request
# def create_tables():
# db.create_all()
api.register_blueprint(ItemBlueprint)
api.register_blueprint(StoreBlueprint)
api.register_blueprint(TagBlueprint)
api.register_blueprint(UserBlueprint)
return app
And I am trying to test some endpoints like create a store, so I'm doing this in the tests/conftest.py
from app import create_app
#pytest.fixture(scope='module')
def client():
flask_app = create_app()
flask_app.testing = True
flask_app.testing = True
with flask_app.test_client() as testing_client:
with flask_app.app_context():
yield testing_client
And there is the test for creation of a store
from flask_jwt_extended import create_access_token
def test_store_creation(client):
"""
GIVEN a Store model
WHEN a new Store is created
THEN check the name field is defined correctly
"""
access_token = create_access_token('admin')
headers = {
'Authorization': 'Bearer {}'.format(access_token)
}
response = client.post('/store', data={"name": "test_store"}, headers=headers)
print(response.get_json())
assert response.status_code == 201
The Store schema
class PlainItemSchema(Schema):
id = fields.Str(dump_only=True)
name = fields.Str(required=True)
price = fields.Float(required=True)
class PlainTagSchema(Schema):
id = fields.Int(dump_only=True)
name = fields.Str()
class PlainStoreSchema(Schema):
id = fields.Str(dump_only=True)
name = fields.Str(required=True)
class StoreSchema(PlainStoreSchema):
items = fields.List(fields.Nested(PlainItemSchema()), dump_only=True)
tags = fields.List(fields.Nested(PlainTagSchema()), dump_only=True)
My Store Blueprint
from flask.views import MethodView
from flask_smorest import abort, Blueprint
from sqlalchemy.exc import SQLAlchemyError, IntegrityError
from flask_jwt_extended import jwt_required, get_jwt
from schemas import StoreSchema
from db import db
from models import StoreModel
blp = Blueprint('stores', __name__, description='Operations on stores')
#blp.route('/store/<int:store_id>')
class Store(MethodView):
#blp.response(200, StoreSchema)
def get(self, store_id):
store = StoreModel.query.get_or_404(store_id)
return store
#jwt_required(fresh=True)
def delete(self, store_id):
"""Only admins can delete stores"""
jwt = get_jwt()
if not jwt.get('is_admin'):
abort(400, message='Admin privilege required.')
store = StoreModel.query.get_or_404(store_id)
db.session.delete(store)
db.session.commit()
return {"message": "Store deleted"}
#blp.route("/store")
class StoreList(MethodView):
#blp.response(200, StoreSchema(many=True))
def get(self):
return StoreModel.query.all()
#jwt_required()
#blp.arguments(StoreSchema)
#blp.response(201, StoreSchema)
def post(self, store_data):
store = StoreModel(**store_data)
try:
db.session.add(store)
db.session.commit()
except IntegrityError:
abort(
400,
message="A store with that name already exists.",
)
except SQLAlchemyError:
abort(500, message="An error occurred creating the store.")
return store
The error I'm getting is this
============================= test session starts ==============================
collecting ... collected 1 item
test_stores.py::test_store_creation FAILED [100%]{'code': 422, 'errors': {'json': {'name': ['Missing data for required field.']}}, 'status': 'Unprocessable Entity'}
test_stores.py:4 (test_store_creation)
422 != 201
Expected :201
Actual :422
<Click to see difference>
client = <FlaskClient <Flask 'app'>>
def test_store_creation(client):
"""
GIVEN a Store model
WHEN a new Store is created
THEN check the name field is defined correctly
"""
data = {
"name": "test"
}
access_token = create_access_token('admin')
headers = {
'Authorization': 'Bearer {}'.format(access_token)
}
response = client.post('/store', headers=headers, data=data)
print(response.get_json())
> assert response.status_code == 201
E assert 422 == 201
E + where 422 = <WrapperTestResponse 109 bytes [422 UNPROCESSABLE ENTITY]>.status_code
test_stores.py:22: AssertionError
It's like I'm not sending any data in the cliet.post() method, In fact if I remove the 'data={'name': 'test'}' I get the same result.
I tried to put data dict outside the method and convert it to json with json.dumps(data) still getting the result
flask-smorest is expecting you to post json data as default, so automatically send it as json OR adjust your header manually to reflect the data your are sending.
Fix your code as:
response = client.post("/store", json={"name": "test_store"}, headers=headers)
I am working with Spotify API to request song data through a URL. I imported urlencode to run the song's ID as a function parameter into the url. Essentially, I need the ID portion of the url request to be the ID by itself, not "id=<id string>"
I tried assigning the id string to a parameter of my 'search' function. The search function takes a user input song ID from spotify, inserts it into the proper position in the URL request, and sends it to the spotify database to retrieve that songs data analysis. The program successfully sends out the request, but the id portion I am filling in puts "id=<song ID>" instead of the song ID by itself.
import requests
import datetime
from urllib.parse import urlencode
# In[3]:
import base64
# In[4]:
client_id = 'fb5af83351d4402fa82904fc04f7fc9e'
client_secret = 'b5057eb39b024180b61b02eb45fb97a6'
# In[5]:
class SpotifyAPI(object):
access_token = None
access_token_expires = datetime.datetime.now()
access_token_did_expire = True
client_id = None
client_secret = None
token_url = "https://accounts.spotify.com/api/token"
def __init__(self, client_id, client_secret, *args, **kwargs):
super().__init__(*args, **kwargs)
self.client_id = client_id
self.client_secret = client_secret
def get_client_credentials(self):
client_id = self.client_id
client_secret = self.client_secret
if client_secret == None or client_id == None:
raise Exception("You must set client_id and client_secret")
client_creds = f"{client_id}:{client_secret}"
client_creds_b64 = base64.b64encode(client_creds.encode())
return client_creds_b64.decode()
def get_token_headers(self):
client_creds_b64 = self.get_client_credentials()
return {
"Authorization": f"Basic {client_creds_b64}"
}
def get_token_data(self):
return {
"grant_type": "client_credentials"
}
def perform_authorization(self):
token_url = self.token_url
token_data = self.get_token_data()
token_headers = self.get_token_headers()
r = requests.post(token_url, data=token_data, headers=token_headers)
print(r.json())
if r.status_code not in range(200,299):
raise Exception("Could not authenticate client")
#return False
data = r.json()
now = datetime.datetime.now()
access_token = data['access_token']
expires_in = data['expires_in'] #seconds
expires = now + datetime.timedelta(seconds=expires_in)
self.access_token = access_token
self.access_token_expires = expires
self.access_token_did_expire = expires < now
return True
def get_access_token(self):
token = self.access_token
expires = self.access_token_expires
now = datetime.datetime.now()
if expires < now:
self.perform_authorization()
return self.get_access_token()
elif token == None:
self.perform_authorization()
return self.get_access_token()
return token
def search(self, id):
access_token = self.get_access_token()
headers = {
"Authorization": f"Bearer {access_token}"
}
end_point = "https://api.spotify.com/v1/audio-analysis/"
data = urlencode({"id":id})
print(data)
lookup_url = f"{end_point}{data}"
print(lookup_url)
r = requests.get(lookup_url, headers = headers)
if r.status_code not in range(200, 299):
return r
return r.json()
it returns
{'access_token': 'BQCLoKT_b2PF7KPSbscosa1dCpE5rzd_RBkswOvwklVdlAL4AeEGCDn0iYuqac5o86BTqCIz0m95u3olLp4', 'token_type': 'Bearer', 'expires_in': 3600}
id=1UGD3lW3tDmgZfAVDh6w7r
https://api.spotify.com/v1/audio-analysis/id=1UGD3lW3tDmgZfAVDh6w7r
I am working on a script which i have to modify in order loop through the multiple resources within a functions.
Below are the items which we need to loop through to get the data from and, this is coming from Config_local
BASE_URL = "https://synergy.hpe.example.com/rest/"
RES_EXT = [ 'resource-alerts?count=500&start=1',
'resource-alerts?count=500&start=501'
'resource-alerts?count=500&start=1001,
'resource-alerts?count=500&start=1501'
]
While i am looping through above list under def main(): section and taking get_resource_alerts_response() under loop the data coming out of the loop getting over-written and thus returning only last loop data only.
Main Script:
import os
import shutil
import smtplib
from email.message import EmailMessage
import pandas as pd
pd.set_option('expand_frame_repr', True)
import requests
from Config_local import (
BASE_URL,
DST_DIR,
OUTFILE,
PASSWORD,
SRC_DIR,
TIMESTAMP_DST,
USERNAME,
SUBJECT,
FROM,
TO,
EMAIL_TEMPLATE,
SMTP_SERVER,
RES_EXT,
)
class FileMoveFailure(Exception):
pass
class SynergyRequestFailure(Exception):
pass
class SessionIdRetrievalFailure(SynergyRequestFailure):
pass
class ResourceAlertsRetrievalFailure(SynergyRequestFailure):
pass
def move_csv_files():
for csv_file in os.listdir(SRC_DIR):
if csv_file.endswith(".csv") and os.path.isfile(os.path.join(SRC_DIR, csv_file)):
try:
shutil.move(
os.path.join(f"{SRC_DIR}/{csv_file}"),
f"{DST_DIR}/{csv_file}-{TIMESTAMP_DST}.log"
)
except OSError as os_error:
raise FileMoveFailure(
f'Moving file {csv_file} has failed: {os_error}'
)
def get_session_id(session):
try:
response = session.post(
url=f"{BASE_URL}/login-sessions",
headers={
"accept": "application/json",
"content-type": "application/json",
"x-api-version": "120",
},
json={
"userName": USERNAME,
"password": PASSWORD
},
verify=False
)
except requests.exceptions.RequestException as req_exception:
# you should also get this logged somewhere, or at least
# printed depending on your use case
raise SessionIdRetrievalFailure(
f"Could not get session id: {req_exception}"
)
json_response = response.json()
if not json_response.get("sessionID"):
# always assume the worse and do sanity checks & validations
# on fetched data
raise KeyError("Could not fetch session id")
return json_response["sessionID"]
#def get_all_text(session, session_id):
# all_text = ''
# for res in RES_EXT:
# url= f"{BASE_URL}{res}"
# newresult = get_resource_alerts_response(session, session_id, url)
# all_text += newresult
# print(f"{all_text}")
# return str(all_text)
#
def get_resource_alerts_response(session, session_id, res):
try:
return session.get(
url=f"{BASE_URL}{res}",
headers={
"accept": "application/json",
"content-type": "text/csv",
"x-api-version": "2",
"auth": session_id,
},
verify=False,
stream=True
)
except requests.exceptions.RequestException as req_exception:
# you should also get this logged somewhere, or at least
# printed depending on your use case
raise ResourceAlertsRetrievalFailure(
f"Could not fetch resource alerts: {req_exception}"
)
def resource_alerts_to_df(resource_alerts_response):
with open(OUTFILE, 'wb') as f:
for chunk in resource_alerts_response.iter_content(chunk_size=1024*36):
f.write(chunk)
return pd.read_csv(OUTFILE)
def send_email(df):
server = smtplib.SMTP(SMTP_SERVER)
msg = EmailMessage()
msg['Subject'], msg['From'], msg['To'] = SUBJECT, FROM, TO
msg.set_content("Text version of your html template")
msg.add_alternative(
EMAIL_TEMPLATE.format(df.to_html(index=False)),
subtype='html'
)
server.send_message(msg)
def main():
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
print(resource_alerts_response)
df = resource_alerts_to_df(resource_alerts_response)
print(df)
send_email(df)
if __name__ == '__main__':
main()
any help or hint will be much appreciated.
This is a copy of the code which I recall we had over SO but not what you want now, However, as the whole code body is okay and the idea of for loop is also looks good, you Just need to tweek it to meet the requirement.
1- You need to create and empty DataFrame assignment Synergy_Data = pd.DataFrame()
2- then you can append the data you received from for loop ie resource_alerts_response which becomes df = resource_alerts_to_df(resource_alerts_response)
3- lastly, you can append this df to the empty Synergy_Data and then call that under your if __name__ == '__main__' to send an e-mail. Also don't forget to declare Synergy_Data as a global variable.
Synergy_Data = pd.DataFrame()
def main():
global Synergy_Data
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
df = resource_alerts_to_df(resource_alerts_response)
Synergy_Data = Synergy_Data.append(df)
if __name__ == '__main__':
main()
send_email(Synergy_Data)
Hope this will be helpful.
Only the last response is being used because this value is overwritten in resource_alerts_response on each iteration of the loop. You may consider acting on the data on each iteration or storing it for use later i.e. after the loop. I've included these options with modifications to the main() function below.
Option 1
Send an email for each resource alert response
def main():
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
print(resource_alerts_response)
# Indent lines below so that the operations below are executed in each loop iteration
df = resource_alerts_to_df(resource_alerts_response)
print(df)
send_email(df)
Option 2
Merge all resource alert responses and send one email
def main():
move_csv_files()
session = requests.Session()
session_id = get_session_id(session)
df_resource_alerts_responses = None
for res in RES_EXT:
resource_alerts_response = get_resource_alerts_response(session,
session_id, res)
print(resource_alerts_response)
df = resource_alerts_to_df(resource_alerts_response)
if df_resource_alerts_responses is None:
df_resource_alerts_responses = df
else:
df_resource_alerts_responses = df_resource_alerts_responses.append(df, ignore_index=True)
print(df_resource_alerts_responses)
if df_resource_alerts_responses is not None:
send_email(df_resource_alerts_responses)
I'm trying to get quotes from the etrade API. I'm able to list accounts, get transactions, but not get quotes. I've tried removing the accounts and transactions api calls but it makes no difference. I get an "oauth_problem=signature_invalid" response. Any ideas what I need to do differently?
from rauth import OAuth1Service
import webbrowser
import hmac
# required for google sheets
# from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
class ETradeManager():
def __init__(self):
session = None
service = None
def connect_to_etrade(self):
self.service = OAuth1Service(
name='etrade',
consumer_key='',
consumer_secret='',
request_token_url='https://apisb.etrade.com/oauth/request_token',
access_token_url='https://apisb.etrade.com/oauth/access_token',
authorize_url='https://us.etrade.com/e/t/etws/authorize?key={}&token={}',
base_url='https://etsw.etrade.com')
oauth_token, oauth_token_secret = self.service.get_request_token(params=
{'oauth_callback': 'oob',
'format': 'json'})
auth_url = self.service.authorize_url.format(self.service.consumer_key, oauth_token)
webbrowser.open(auth_url)
verifier = input('Please input the verifier: ')
print("Attempting to get session")
self.session = self.service.get_auth_session(oauth_token, oauth_token_secret, params={'oauth_verifier': verifier})
url = 'https://apisb.etrade.com/v1/accounts/list'
resp = self.session.get(url, params={'format': 'json'})
accountid = ""
print(resp.text)
trans_url_template = "https://apisb.etrade.com/v1/accounts/{}/transactions"
trans_url = trans_url_template.format(accountid)
resp = self.session.get(trans_url, params={'format': 'json'})
f = open("trans.xml", "w")
f.write(resp.text)
# key = self.service.consumer_secret + \
# '&' + \
# oauth_token_secret
# hashed = hmac.new(key.encode(), base_string.encode(), sha1)
# def get_quote(self):
quote_url_template = "https://apisb.etrade.com/v1/market/quote/{}"
quote_url = quote_url_template.format("TSLA")
resp = self.session.get(quote_url_template, params={'format': 'json'})
f = open("quote.xml", "w")
f.write(resp.text)
trade_manager = ETradeManager()
trade_manager.connect_to_etrade()
# trade_manager.get_quote()
Not sure if you figured this out but you had a typo here:
resp = self.session.get(quote_url_template, params={'format': 'json'})
Should be using quote_url, not quote_url_template
import httplib2
import os
import oauth2client
from oauth2client import client, tools
from apiclient import errors, discovery
SCOPES = 'https://mail.google.com/'
APPLICATION_NAME = 'Gmail API Python List Email'
CLIENT_SECRET_FILE = 'client_secret_ishandutta2007.json'# This file will be in local dir
CREDENTIAL_FILE_NAME = 'gmail-python-email-send_ishandutta2007.json'
def get_credentials():
home_dir = os.path.expanduser('~')
credential_dir = os.path.join(home_dir, '.credentials')
if not os.path.exists(credential_dir):
os.makedirs(credential_dir)
credential_path = os.path.join(credential_dir, CREDENTIAL_FILE_NAME)
store = oauth2client.file.Storage(credential_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
flow.user_agent = APPLICATION_NAME
credentials = tools.run_flow(flow, store)
print(('Storing credentials to ' + credential_path))
return credentials
def ListMessagesMatchingQuery(service, user_id, query=''):
try:
response = service.users().messages().list(userId=user_id, q=query).execute()
messages = []
if 'messages' in response:
messages.extend(response['messages'])
while 'nextPageToken' in response:
page_token = response['nextPageToken']
response = service.users().messages().list(userId=user_id, q=query,
pageToken=page_token).execute()
messages.extend(response['messages'])
return messages
except errors.HttpError as error:
print('An error occurred: %s' % error)
def main():
credentials = get_credentials()
http = credentials.authorize(httplib2.Http())
service = discovery.build('gmail', 'v1', http=http)
ListMessagesMatchingQuery(service, "me", query="senior recruit")
if __name__ == '__main__':
main()
An error occurred: https://www.googleapis.com/gmail/v1/users/me/messages?q=senior+recruit&alt=json
returned "Insufficient Permission">
Found the issues,in the code above it was picking up a cached credentials from the file gmail-python-email-send_ishandutta2007.json which has permission to send mails only, changing this filename to something new fixed the issue.