I've been trying out cloud functions for a little while now. Recently, I found out about functions-framework. It basically allows you to run your functions locally. This helps/should help in reducing the time it takes to test your code.
I am running into an issue where calling the functions locally - curl localhost:8080 outputs Insufficient Permission: Request had insufficient authentication scopes while testing them via the console produces the expected result.
I am trying to move data from google drive to google cloud storage
const { google } = require("googleapis");
const { google } = require("googleapis");
const SCOPES = [
"profile",
"email",
"https://www.googleapis.com/auth/drive",
"https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/drive.readonly",
"https://www.googleapis.com/auth/drive.metadata.readonly",
"https://www.googleapis.com/auth/drive.appdata",
"https://www.googleapis.com/auth/drive.metadata",
];
async function addDocument(authClient, bucket, data) {
const version = "v1";
const storage = google.storage({ version, auth: authClient });
const response = await storage.objects.insert({
bucket,
requestBody: {
name: data.name,
},
media: {
body: data.content,
mimeType: data.mimeType,
},
});
return response.data;
}
async function getDocumentContent(authClient, fileId) {
const version = "v3";
const drive = google.drive({ version, client: authClient });
const json = await drive.files.get({
alt: "media",
fileId: fileId,
auth: authClient,
});
const content = await drive.files.get({
alt: "media",
fileId: fileId,
auth: authClient,
});
const data = {
content: content.data,
mimeType: json.data.mimeType,
name: json.data.name,
};
return data;
}
async function getDocuments(authClient, folderId) {
const query = `parents= '${folderId}'`;
const fields = "files(id, name)";
const version = "v3";
const drive = google.drive({ version, client: authClient });
const response = await drive.files.list({
q: query,
fields,
auth: authClient,
});
const { files } = response.data;
return files;
}
exports.copy = async (req, res) => {
const auth = new google.auth.GoogleAuth({
scopes: SCOPES,
});
const client = await auth.getClient();
const containingFolder = "folder_id";
const bucketName = "some_bucket";
try {
const files = await getDocuments(client, containingFolder);
res.send(files);
} catch (err) {
res.send(err.message);
}
};
My question here is, how do I get local calls to have the same authentication as my remote calls?
I wrote an article on that. At the time where I wrote the article, only Java and Go (because I contributed to implement the feature in the client library) were compliant.
Have a try with NodeJS, it might be implemented now.
Related
I'm having issues with the Google drive API. I am passing in the folder ID and it's not showing any files. How do I traverse within the sub folders to check for folders and files there?
const path = require('path')
const { JWT } = require('google-auth-library')
const { google } = require('googleapis')
const axios = require('axios')
// Get auth token
const getAuth = ({ email, key }) => {
const scopes = ['https://www.googleapis.com/auth/drive']
return new JWT({
email,
key,
scopes,
})
}
async function loadDrive(options) {
const { folderId, key, service_email } = options
const auth = await getAuth({ email: service_email, key })
const drive = google.drive({
version: 'v3',
auth: auth,
})
try {
const res = await drive.files.list({
pageSize: 10,
fields: 'nextPageToken, files(id, name)',
q: `'${folderId}' in parents`,
spaces: 'drive',
})
const files = res.data.files
if (files.length === 0) {
console.log('No files found.')
} else {
console.log('Files:')
for (const file of files) {
console.log(`${file.name} (${file.id})`)
}
}
} catch (e) {
console.log(e)
}
}
[Update 01/04/22]: I shared my service account email under https://console.cloud.google.com/iam-admin/serviceaccounts with the folder in Google drive and was able access the Drive data.
I am using node to list all users from domain. I had created service account with domain wide delegation.
The domain admin gave access to the service account to required scopes.
Code:
const { JWT } = require('google-auth-library');
const {google, chat_v1} = require('googleapis');
const keys = require('./keys.json')
async function main() {
const client = new JWT(keys.client_email, keys, keys.private_key,
['https://www.googleapis.com/auth/admin.directory.user',
'https://www.googleapis.com/auth/admin.directory.user.readonly',
'https://www.googleapis.com/auth/cloud-platform'],
"admin#domain.com"
);
await client.authorize();
const service = google.admin("directory_v1");
service.users.list({
domain: "domain.com",
maxResults: "10",
orderBy: "email",
}, (err, res) => {
if (err) return console.error('The API returned an error:', err.message);
const users = res.data.users;
if (users.length) {
console.log('Users:');
users.forEach((user) => {
console.log(`${user.primaryEmail} (${user.name.fullName})`);
});
} else {
console.log('No users found.');
}
});
}
main();
but only thing I recive is:
The API returned an error: Login Required.
also enabled admin sdk api for this service account
any idea why this is happening?
I'm not sure if JWT is the correct way, but this is how I do it.
const google = require("googleapis").google;
const SRVC_ACCOUNT_CREDS = require('./keys.json');
const getClient = async (scopes: string[], user: string)=>{
const auth = new google.auth.GoogleAuth({
credentials: SRVC_ACCOUNT_CREDS,
scopes: scopes
});
const client = await auth.getClient();
client.subject = user;
return client;
};
const listUsers = async (query = "", limit = 500, pageToken = null, user, fields, getAll = false)=>{
const scopes = ["https://www.googleapis.com/auth/admin.directory.user"];
const client = await getClient(scopes, user);
const service = google.admin({version: "directory_v1", auth: client});
const result = {
users: [],
nextPageToken: ""
};
if(!fields) {
fields = "users(name.fullName,primaryEmail,organizations(department,primary,title),thumbnailPhotoUrl),nextPageToken";
}
do{
const request = await service.users.list({
customer: "my_customer",
fields: fields,
orderBy: "givenName",
maxResults: limit,
pageToken: pageToken,
query: query,
viewType: "admin_view"
});
pageToken = getAll ? request.data.nextPageToken : null;
const users = request.data.users;
if(users && users.length){
result.users.push(...users);
result.nextPageToken = request.data.nextPageToken;
}
} while(pageToken);
return result;
};
I am using the node Google Auth Library to get an authorized client. I know this step is successful from the line console.log('Project ID', await auth.getProjectId()) which prints the correct result. I am getting an error when I try to use this authorized client to authenticate the Google Drive api:
Error: invalid_scope: Invalid OAuth scope or ID token audience provided.
Code:
import { google } from 'googleapis'
const { GoogleAuth } = require('google-auth-library')
import { config } from 'dotenv'
import { resolve } from 'path'
/**
* GCLOUD_PROJECT and GOOGLE_APPLICATION_CREDENTIALS are set
* in the top-level .env file, GoogleAuth requires these to be set
*/
config({ path: resolve(__dirname, '../.env')})
const main = async () => {
const csv = ''
const auth = await getAuthClient()
await uploadToGoogleDrive({ csv, auth })
}
const getAuthClient = async () => {
const scopes = ['https://www.googleapis.com/auth.drive']
const auth = new GoogleAuth({ scopes })
console.log('Project ID', await auth.getProjectId())
return await auth.getClient()
}
const uploadToGoogleDrive = async ({ csv, auth }) => {
const drive = google.drive({
version: 'v3',
auth,
})
await drive.files.create({
requestBody: {
name: 'test.csv',
mimeType: 'text/plain',
parents: ['** Parent Folder Name **']
},
media: {
mimeType: 'text/csv',
body: csv
}
})
}
main().catch(console.error)
I'm using a custom service account (using --service-account parameter in the deploy command). That service account has domain-wide delegation enabled and it's installed in the G Apps Admin panel.
I tried this code:
app.get('/test', async (req, res) => {
const auth = new google.auth.GoogleAuth()
const gmailClient = google.gmail({ version: 'v1' })
const { data } = await gmailClient.users.labels.list({ auth, userId: 'user#domain.com' })
return res.json(data).end()
})
It works if I run it on my machine (having the GOOGLE_APPLICATION_CREDENTIALS env var setted to the path of the same service account that is assigned to the Cloud Run service) but when it's running in Cloud Run, I get this response:
{
"code" : 400,
"errors" : [ {
"domain" : "global",
"message" : "Bad Request",
"reason" : "failedPrecondition"
} ],
"message" : "Bad Request"
}
I saw this solution for this same issue, but it's for Python and I don't know how to replicate that behaviour with the Node library.
After some days of research, I finally got a working solution (porting the Python implementation):
async function getGoogleCredentials(subject: string, scopes: string[]): Promise<JWT | OAuth2Client> {
const auth = new google.auth.GoogleAuth({
scopes: ['https://www.googleapis.com/auth/cloud-platform'],
})
const authClient = await auth.getClient()
if (authClient instanceof JWT) {
return (await new google.auth.GoogleAuth({ scopes, clientOptions: { subject } }).getClient()) as JWT
} else if (authClient instanceof Compute) {
const serviceAccountEmail = (await auth.getCredentials()).client_email
const unpaddedB64encode = (input: string) =>
Buffer.from(input)
.toString('base64')
.replace(/=*$/, '')
const now = Math.floor(new Date().getTime() / 1000)
const expiry = now + 3600
const payload = JSON.stringify({
aud: 'https://accounts.google.com/o/oauth2/token',
exp: expiry,
iat: now,
iss: serviceAccountEmail,
scope: scopes.join(' '),
sub: subject,
})
const header = JSON.stringify({
alg: 'RS256',
typ: 'JWT',
})
const iamPayload = `${unpaddedB64encode(header)}.${unpaddedB64encode(payload)}`
const iam = google.iam('v1')
const { data } = await iam.projects.serviceAccounts.signBlob({
auth: authClient,
name: `projects/-/serviceAccounts/${serviceAccountEmail}`,
requestBody: {
bytesToSign: unpaddedB64encode(iamPayload),
},
})
const assertion = `${iamPayload}.${data.signature!.replace(/=*$/, '')}`
const headers = { 'content-type': 'application/x-www-form-urlencoded' }
const body = querystring.encode({ assertion, grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer' })
const response = await fetch('https://accounts.google.com/o/oauth2/token', { method: 'POST', headers, body }).then(r => r.json())
const newCredentials = new OAuth2Client()
newCredentials.setCredentials({ access_token: response.access_token })
return newCredentials
} else {
throw new Error('Unexpected authentication type')
}
}
What you can do here is define ENV variables in your yaml file as described in this documentation to set the GOOGLE_APPLICATION_CREDENTIALS to the path of the JSON key.
Then use a code such as the one mentioned here.
const authCloudExplicit = async ({projectId, keyFilename}) => {
// [START auth_cloud_explicit]
// Imports the Google Cloud client library.
const {Storage} = require('#google-cloud/storage');
// Instantiates a client. Explicitly use service account credentials by
// specifying the private key file. All clients in google-cloud-node have this
// helper, see https://github.com/GoogleCloudPlatform/google-cloud-node/blob/master/docs/authentication.md
// const projectId = 'project-id'
// const keyFilename = '/path/to/keyfile.json'
const storage = new Storage({projectId, keyFilename});
// Makes an authenticated API request.
try {
const [buckets] = await storage.getBuckets();
console.log('Buckets:');
buckets.forEach(bucket => {
console.log(bucket.name);
});
} catch (err) {
console.error('ERROR:', err);
}
// [END auth_cloud_explicit]
};
Or follow an approach similar to the one mentioned here.
'use strict';
const {auth, Compute} = require('google-auth-library');
async function main() {
const client = new Compute({
serviceAccountEmail: 'some-service-account#example.com',
});
const projectId = await auth.getProjectId();
const url = `https://dns.googleapis.com/dns/v1/projects/${projectId}`;
const res = await client.request({url});
console.log(res.data);
}
main().catch(console.error);
I need to authorize from Google analytics to get the response data.
var google = require('googleapis'),
q = require('q'),
SERVICE_ACCOUNT_EMAIL = '838823084353-cjjoiv9di67fuh7geqgggociibataf9v#developer.gserviceaccount.com',
SERVICE_ACCOUNT_KEY_FILE = __dirname + '/google-services-private-key.pem';
var def = q.defer();
var gAnalytics = google.analytics('v3');
var authClient = new google.auth.JWT( SERVICE_ACCOUNT_EMAIL, SERVICE_ACCOUNT_KEY_FILE, null, ['https://www.googleapis.com/auth/analytics.readonly']);
console.log(authClient)
authClient.authorize(function (err, tokens) {
if (err) {
console.log("err is: " + err, tokens);
return;
}
But it fails to authorize
getting error
JWT { transporter: DefaultTransporter {}, clientId_: undefined,
clientSecret_: undefined, redirectUri_: undefined, opts: {},
credentials: { refresh_token: 'jwt-placeholder', expiry_date: 1 },
email:
'838823084353-cjjoiv9di67fuh7geqgggociibataf9v#developer.gserviceaccount.com',
keyFile:
'/home/aaa/Desktop/ampretailer/server/google-services-private-key.pem',
key: null, scopes: [
'https://www.googleapis.com/auth/analytics.readonly' ], subject:
undefined, gToken: [Function: GoogleToken] } err is: Error:
invalid_grant: Invalid JWT Signature. { access_token: null,
token_type: 'Bearer', expiry_date:null }
I recommend you try using Google Analytics v4 instead of v3 there are a number of dimensions and metrics which you will not have access to using V3.
'use strict';
const { google } = require('googleapis');
const sampleClient = require('../sampleclient');
const analyticsreporting = google.analyticsreporting({
version: 'v4',
auth: sampleClient.oAuth2Client
});
async function runSample () {
const res = await analyticsreporting.reports.batchGet({
resource: {
reportRequests: [{
viewId: '65704806',
dateRanges: [
{
startDate: '2018-03-17',
endDate: '2018-03-24'
}, {
startDate: '14daysAgo',
endDate: '7daysAgo'
}
],
metrics: [
{
expression: 'ga:users'
}
]
}]
}
});
console.log(res.data);
return res.data;
}
// if invoked directly (not tests), authenticate and run the samples
if (module === require.main) {
const scopes = ['https://www.googleapis.com/auth/analytics'];
sampleClient.authenticate(scopes)
.then(c => runSample())
.catch(e => console.error);
}
// export functions for testing purposes
module.exports = {
runSample,
client: sampleClient.oAuth2Client
};
Code ripped from analyticsReporting/batchGet.js
Service account - To use the service account based samples, create a new service account in the cloud developer console, and save the file as jwt.keys.json in the samples directory.
'use strict';
const {google} = require('googleapis');
const path = require('path');
/**
* The JWT authorization is ideal for performing server-to-server
* communication without asking for user consent.
*
* Suggested reading for Admin SDK users using service accounts:
* https://developers.google.com/admin-sdk/directory/v1/guides/delegation
*
* See the defaultauth.js sample for an alternate way of fetching compute credentials.
*/
async function runSample () {
// Create a new JWT client using the key file downloaded from the Google Developer Console
const client = await google.auth.getClient({
keyFile: path.join(__dirname, 'jwt.keys.json'),
scopes: 'https://www.googleapis.com/auth/analytics.readonly'
});
// Obtain a new drive client, making sure you pass along the auth client
const analyticsreporting = google.analyticsreporting({
version: 'v4',
auth: client
});
// Make an authorized request to list Drive files.
const res = = await analyticsreporting.reports.batchGet({
resource: {
reportRequests: [{
viewId: '65704806',
dateRanges: [
{
startDate: '2018-03-17',
endDate: '2018-03-24'
}, {
startDate: '14daysAgo',
endDate: '7daysAgo'
}
],
metrics: [
{
expression: 'ga:users'
}
]
}]
}
});
console.log(res.data);
return res.data;
}
if (module === require.main) {
runSample().catch(console.error);
}
// Exports for unit testing purposes
module.exports = { runSample };
code ripped from samples/jwt.js