I am using the node Google Auth Library to get an authorized client. I know this step is successful from the line console.log('Project ID', await auth.getProjectId()) which prints the correct result. I am getting an error when I try to use this authorized client to authenticate the Google Drive api:
Error: invalid_scope: Invalid OAuth scope or ID token audience provided.
Code:
import { google } from 'googleapis'
const { GoogleAuth } = require('google-auth-library')
import { config } from 'dotenv'
import { resolve } from 'path'
/**
* GCLOUD_PROJECT and GOOGLE_APPLICATION_CREDENTIALS are set
* in the top-level .env file, GoogleAuth requires these to be set
*/
config({ path: resolve(__dirname, '../.env')})
const main = async () => {
const csv = ''
const auth = await getAuthClient()
await uploadToGoogleDrive({ csv, auth })
}
const getAuthClient = async () => {
const scopes = ['https://www.googleapis.com/auth.drive']
const auth = new GoogleAuth({ scopes })
console.log('Project ID', await auth.getProjectId())
return await auth.getClient()
}
const uploadToGoogleDrive = async ({ csv, auth }) => {
const drive = google.drive({
version: 'v3',
auth,
})
await drive.files.create({
requestBody: {
name: 'test.csv',
mimeType: 'text/plain',
parents: ['** Parent Folder Name **']
},
media: {
mimeType: 'text/csv',
body: csv
}
})
}
main().catch(console.error)
Related
I'm having issues with the Google drive API. I am passing in the folder ID and it's not showing any files. How do I traverse within the sub folders to check for folders and files there?
const path = require('path')
const { JWT } = require('google-auth-library')
const { google } = require('googleapis')
const axios = require('axios')
// Get auth token
const getAuth = ({ email, key }) => {
const scopes = ['https://www.googleapis.com/auth/drive']
return new JWT({
email,
key,
scopes,
})
}
async function loadDrive(options) {
const { folderId, key, service_email } = options
const auth = await getAuth({ email: service_email, key })
const drive = google.drive({
version: 'v3',
auth: auth,
})
try {
const res = await drive.files.list({
pageSize: 10,
fields: 'nextPageToken, files(id, name)',
q: `'${folderId}' in parents`,
spaces: 'drive',
})
const files = res.data.files
if (files.length === 0) {
console.log('No files found.')
} else {
console.log('Files:')
for (const file of files) {
console.log(`${file.name} (${file.id})`)
}
}
} catch (e) {
console.log(e)
}
}
[Update 01/04/22]: I shared my service account email under https://console.cloud.google.com/iam-admin/serviceaccounts with the folder in Google drive and was able access the Drive data.
I've been trying out cloud functions for a little while now. Recently, I found out about functions-framework. It basically allows you to run your functions locally. This helps/should help in reducing the time it takes to test your code.
I am running into an issue where calling the functions locally - curl localhost:8080 outputs Insufficient Permission: Request had insufficient authentication scopes while testing them via the console produces the expected result.
I am trying to move data from google drive to google cloud storage
const { google } = require("googleapis");
const { google } = require("googleapis");
const SCOPES = [
"profile",
"email",
"https://www.googleapis.com/auth/drive",
"https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/drive.readonly",
"https://www.googleapis.com/auth/drive.metadata.readonly",
"https://www.googleapis.com/auth/drive.appdata",
"https://www.googleapis.com/auth/drive.metadata",
];
async function addDocument(authClient, bucket, data) {
const version = "v1";
const storage = google.storage({ version, auth: authClient });
const response = await storage.objects.insert({
bucket,
requestBody: {
name: data.name,
},
media: {
body: data.content,
mimeType: data.mimeType,
},
});
return response.data;
}
async function getDocumentContent(authClient, fileId) {
const version = "v3";
const drive = google.drive({ version, client: authClient });
const json = await drive.files.get({
alt: "media",
fileId: fileId,
auth: authClient,
});
const content = await drive.files.get({
alt: "media",
fileId: fileId,
auth: authClient,
});
const data = {
content: content.data,
mimeType: json.data.mimeType,
name: json.data.name,
};
return data;
}
async function getDocuments(authClient, folderId) {
const query = `parents= '${folderId}'`;
const fields = "files(id, name)";
const version = "v3";
const drive = google.drive({ version, client: authClient });
const response = await drive.files.list({
q: query,
fields,
auth: authClient,
});
const { files } = response.data;
return files;
}
exports.copy = async (req, res) => {
const auth = new google.auth.GoogleAuth({
scopes: SCOPES,
});
const client = await auth.getClient();
const containingFolder = "folder_id";
const bucketName = "some_bucket";
try {
const files = await getDocuments(client, containingFolder);
res.send(files);
} catch (err) {
res.send(err.message);
}
};
My question here is, how do I get local calls to have the same authentication as my remote calls?
I wrote an article on that. At the time where I wrote the article, only Java and Go (because I contributed to implement the feature in the client library) were compliant.
Have a try with NodeJS, it might be implemented now.
I'm using a custom service account (using --service-account parameter in the deploy command). That service account has domain-wide delegation enabled and it's installed in the G Apps Admin panel.
I tried this code:
app.get('/test', async (req, res) => {
const auth = new google.auth.GoogleAuth()
const gmailClient = google.gmail({ version: 'v1' })
const { data } = await gmailClient.users.labels.list({ auth, userId: 'user#domain.com' })
return res.json(data).end()
})
It works if I run it on my machine (having the GOOGLE_APPLICATION_CREDENTIALS env var setted to the path of the same service account that is assigned to the Cloud Run service) but when it's running in Cloud Run, I get this response:
{
"code" : 400,
"errors" : [ {
"domain" : "global",
"message" : "Bad Request",
"reason" : "failedPrecondition"
} ],
"message" : "Bad Request"
}
I saw this solution for this same issue, but it's for Python and I don't know how to replicate that behaviour with the Node library.
After some days of research, I finally got a working solution (porting the Python implementation):
async function getGoogleCredentials(subject: string, scopes: string[]): Promise<JWT | OAuth2Client> {
const auth = new google.auth.GoogleAuth({
scopes: ['https://www.googleapis.com/auth/cloud-platform'],
})
const authClient = await auth.getClient()
if (authClient instanceof JWT) {
return (await new google.auth.GoogleAuth({ scopes, clientOptions: { subject } }).getClient()) as JWT
} else if (authClient instanceof Compute) {
const serviceAccountEmail = (await auth.getCredentials()).client_email
const unpaddedB64encode = (input: string) =>
Buffer.from(input)
.toString('base64')
.replace(/=*$/, '')
const now = Math.floor(new Date().getTime() / 1000)
const expiry = now + 3600
const payload = JSON.stringify({
aud: 'https://accounts.google.com/o/oauth2/token',
exp: expiry,
iat: now,
iss: serviceAccountEmail,
scope: scopes.join(' '),
sub: subject,
})
const header = JSON.stringify({
alg: 'RS256',
typ: 'JWT',
})
const iamPayload = `${unpaddedB64encode(header)}.${unpaddedB64encode(payload)}`
const iam = google.iam('v1')
const { data } = await iam.projects.serviceAccounts.signBlob({
auth: authClient,
name: `projects/-/serviceAccounts/${serviceAccountEmail}`,
requestBody: {
bytesToSign: unpaddedB64encode(iamPayload),
},
})
const assertion = `${iamPayload}.${data.signature!.replace(/=*$/, '')}`
const headers = { 'content-type': 'application/x-www-form-urlencoded' }
const body = querystring.encode({ assertion, grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer' })
const response = await fetch('https://accounts.google.com/o/oauth2/token', { method: 'POST', headers, body }).then(r => r.json())
const newCredentials = new OAuth2Client()
newCredentials.setCredentials({ access_token: response.access_token })
return newCredentials
} else {
throw new Error('Unexpected authentication type')
}
}
What you can do here is define ENV variables in your yaml file as described in this documentation to set the GOOGLE_APPLICATION_CREDENTIALS to the path of the JSON key.
Then use a code such as the one mentioned here.
const authCloudExplicit = async ({projectId, keyFilename}) => {
// [START auth_cloud_explicit]
// Imports the Google Cloud client library.
const {Storage} = require('#google-cloud/storage');
// Instantiates a client. Explicitly use service account credentials by
// specifying the private key file. All clients in google-cloud-node have this
// helper, see https://github.com/GoogleCloudPlatform/google-cloud-node/blob/master/docs/authentication.md
// const projectId = 'project-id'
// const keyFilename = '/path/to/keyfile.json'
const storage = new Storage({projectId, keyFilename});
// Makes an authenticated API request.
try {
const [buckets] = await storage.getBuckets();
console.log('Buckets:');
buckets.forEach(bucket => {
console.log(bucket.name);
});
} catch (err) {
console.error('ERROR:', err);
}
// [END auth_cloud_explicit]
};
Or follow an approach similar to the one mentioned here.
'use strict';
const {auth, Compute} = require('google-auth-library');
async function main() {
const client = new Compute({
serviceAccountEmail: 'some-service-account#example.com',
});
const projectId = await auth.getProjectId();
const url = `https://dns.googleapis.com/dns/v1/projects/${projectId}`;
const res = await client.request({url});
console.log(res.data);
}
main().catch(console.error);
I am trying to execute the Cloud Bigtable to Cloud Storage SequenceFile template via REST API in a NodeJS backend server.
I am using axios 0.17.1 to send the request and I'm getting 401 status.
I tried to follow the google documentation however I couldn't figure out how to authorize an HTTP request to run a dataflow template.
I want to be authenticated as a service account and I successfully generated and dowloaded the json file containing the private key.
Can anyone help me by showing an example of sending HTTP POST request to https://dataflow.googleapis.com/v1b3/projects/[YOUR_PROJECT_ID]/templates:launch?gcsPath=gs://dataflow-templates/latest/
You need to generate a jwt from your service account credentials. The jwt can be exchanged for an access token which can then be used to make the request to execute the Dataflow job. Complete example:
import axios from "axios";
import jwt from "jsonwebtoken";
import mem from "mem";
import fs from "fs";
const loadServiceAccount = mem(function(){
// This is a string containing service account credentials
const serviceAccountJson = process.env.GOOGLE_APPLICATION_CREDENTIALS;
if (!serviceAccountJson) {
throw new Error("Missing GCP Credentials");
}
})
const loadCredentials = mem(function() {
loadServiceAccount();
const credentials = JSON.parse(fs.readFileSync("key.json").toString());
return {
projectId: credentials.project_id,
privateKeyId: credentials.private_key_id,
privateKey: credentials.private_key,
clientEmail: credentials.client_email,
};
});
interface ProjectCredentials {
projectId: string;
privateKeyId: string;
privateKey: string;
clientEmail: string;
}
function generateJWT(params: ProjectCredentials) {
const scope = "https://www.googleapis.com/auth/cloud-platform";
const authUrl = "https://www.googleapis.com/oauth2/v4/token";
const issued = new Date().getTime() / 1000;
const expires = issued + 60;
const payload = {
iss: params.clientEmail,
sub: params.clientEmail,
aud: authUrl,
iat: issued,
exp: expires,
scope: scope,
};
const options = {
keyid: params.privateKeyId,
algorithm: "RS256",
};
return jwt.sign(payload, params.privateKey, options);
}
async function getAccessToken(credentials: ProjectCredentials): Promise<string> {
const jwt = generateJWT(credentials);
const authUrl = "https://www.googleapis.com/oauth2/v4/token";
const params = {
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
assertion: jwt,
};
try {
const response = await axios.post(authUrl, params);
return response.data.access_token;
} catch (error) {
console.error("Failed to get access token", error);
throw error;
}
}
function buildTemplateParams(projectId: string, table: string) {
return {
jobName: `[job-name]`,
parameters: {
bigtableProjectId: projectId,
bigtableInstanceId: "[table-instance]",
bigtableTableId: table,
outputDirectory: `[gs://your-instance]`,
filenamePrefix: `${table}-`,
},
environment: {
zone: "us-west1-a" // omit or define your own,
tempLocation: `[gs://your-instance/temp]`,
},
};
}
async function backupTable(table: string) {
console.info(`Executing backup template for table=${table}`);
const credentials = loadCredentials();
const { projectId } = credentials;
const accessToken = await getAccessToken(credentials);
const baseUrl = "https://dataflow.googleapis.com/v1b3/projects";
const templatePath = "gs://dataflow-templates/latest/Cloud_Bigtable_to_GCS_Avro";
const url = `${baseUrl}/${projectId}/templates:launch?gcsPath=${templatePath}`;
const template = buildTemplateParams(projectId, table);
try {
const response = await axios.post(url, template, {
headers: { Authorization: `Bearer ${accessToken}` },
});
console.log("GCP Response", response.data);
} catch (error) {
console.error(`Failed to execute template for ${table}`, error.message);
}
}
async function run() {
await backupTable("my-table");
}
try {
run();
} catch (err) {
process.exit(1);
}
I need to authorize from Google analytics to get the response data.
var google = require('googleapis'),
q = require('q'),
SERVICE_ACCOUNT_EMAIL = '838823084353-cjjoiv9di67fuh7geqgggociibataf9v#developer.gserviceaccount.com',
SERVICE_ACCOUNT_KEY_FILE = __dirname + '/google-services-private-key.pem';
var def = q.defer();
var gAnalytics = google.analytics('v3');
var authClient = new google.auth.JWT( SERVICE_ACCOUNT_EMAIL, SERVICE_ACCOUNT_KEY_FILE, null, ['https://www.googleapis.com/auth/analytics.readonly']);
console.log(authClient)
authClient.authorize(function (err, tokens) {
if (err) {
console.log("err is: " + err, tokens);
return;
}
But it fails to authorize
getting error
JWT { transporter: DefaultTransporter {}, clientId_: undefined,
clientSecret_: undefined, redirectUri_: undefined, opts: {},
credentials: { refresh_token: 'jwt-placeholder', expiry_date: 1 },
email:
'838823084353-cjjoiv9di67fuh7geqgggociibataf9v#developer.gserviceaccount.com',
keyFile:
'/home/aaa/Desktop/ampretailer/server/google-services-private-key.pem',
key: null, scopes: [
'https://www.googleapis.com/auth/analytics.readonly' ], subject:
undefined, gToken: [Function: GoogleToken] } err is: Error:
invalid_grant: Invalid JWT Signature. { access_token: null,
token_type: 'Bearer', expiry_date:null }
I recommend you try using Google Analytics v4 instead of v3 there are a number of dimensions and metrics which you will not have access to using V3.
'use strict';
const { google } = require('googleapis');
const sampleClient = require('../sampleclient');
const analyticsreporting = google.analyticsreporting({
version: 'v4',
auth: sampleClient.oAuth2Client
});
async function runSample () {
const res = await analyticsreporting.reports.batchGet({
resource: {
reportRequests: [{
viewId: '65704806',
dateRanges: [
{
startDate: '2018-03-17',
endDate: '2018-03-24'
}, {
startDate: '14daysAgo',
endDate: '7daysAgo'
}
],
metrics: [
{
expression: 'ga:users'
}
]
}]
}
});
console.log(res.data);
return res.data;
}
// if invoked directly (not tests), authenticate and run the samples
if (module === require.main) {
const scopes = ['https://www.googleapis.com/auth/analytics'];
sampleClient.authenticate(scopes)
.then(c => runSample())
.catch(e => console.error);
}
// export functions for testing purposes
module.exports = {
runSample,
client: sampleClient.oAuth2Client
};
Code ripped from analyticsReporting/batchGet.js
Service account - To use the service account based samples, create a new service account in the cloud developer console, and save the file as jwt.keys.json in the samples directory.
'use strict';
const {google} = require('googleapis');
const path = require('path');
/**
* The JWT authorization is ideal for performing server-to-server
* communication without asking for user consent.
*
* Suggested reading for Admin SDK users using service accounts:
* https://developers.google.com/admin-sdk/directory/v1/guides/delegation
*
* See the defaultauth.js sample for an alternate way of fetching compute credentials.
*/
async function runSample () {
// Create a new JWT client using the key file downloaded from the Google Developer Console
const client = await google.auth.getClient({
keyFile: path.join(__dirname, 'jwt.keys.json'),
scopes: 'https://www.googleapis.com/auth/analytics.readonly'
});
// Obtain a new drive client, making sure you pass along the auth client
const analyticsreporting = google.analyticsreporting({
version: 'v4',
auth: client
});
// Make an authorized request to list Drive files.
const res = = await analyticsreporting.reports.batchGet({
resource: {
reportRequests: [{
viewId: '65704806',
dateRanges: [
{
startDate: '2018-03-17',
endDate: '2018-03-24'
}, {
startDate: '14daysAgo',
endDate: '7daysAgo'
}
],
metrics: [
{
expression: 'ga:users'
}
]
}]
}
});
console.log(res.data);
return res.data;
}
if (module === require.main) {
runSample().catch(console.error);
}
// Exports for unit testing purposes
module.exports = { runSample };
code ripped from samples/jwt.js