How to update a variable NodeJs - node.js

I am trying to update "documentCopyId" variable, but print undefined
function listFiles(auth) {
const drive = google.drive({ version: 'v3', auth });
const apiKey = 'xxxxx';
const paragraph = "Hello World";
let documentCopyId;
var copyTitle = "Copy Title";
let request = {
name: copyTitle,
};
drive.files.copy({
fileId: 'xxxx',
resource: request,
}, (err, driveResponse) => {
console.log('document from drive copy ' + driveResponse.data.id) //Print id
documentCopyId = driveResponse.data.id;
});
console.log('document from listFiles '+ documentCopyId); // print undefined
}
complete Log:
document from listFiles undefined
document from drive copy 1IjXkk5QgTNVT85xxxxxx

It is because the code
console.log('document from listFiles '+ documentCopyId); // print undefined
doesn't wait until this code completing
drive.files.copy({
fileId: 'xxxx',
resource: request,
}, (err, driveResponse) => {
console.log('document from drive copy ' + driveResponse.data.id) //Print id
documentCopyId = driveResponse.data.id;
});
which means that
console.log('document from listFiles '+ documentCopyId)
executing before
documentCopyId = driveResponse.data.id;
And in that case, documentCopyId is undefined.
As a solution, you can to promisify driver.files.copy part, and resolve the needed value. Or do need manipulations in a callback of drive.files.copy.
For example, you can do something like this
const listFiles = async (auth) => {
let documentCopyId;
const driveResponse = await copyPromise('Copy Title');
documentCopyId = driveResponse.data.id;
console.log('document from listFiles ' + documentCopyId);
};
const copyPromise = (name) => {
return new Promise((resolve, reject) => {
try {
const drive = google.drive({ version: 'v3', auth });
const apiKey = 'xxxxx';
const paragraph = 'Hello World';
let request = {
name
};
drive.files.copy(
{
fileId: 'xxxx',
resource: request,
},
(err, driveResponse) => {
if (err) throw new Error(err);
console.log('document from drive copy ' + driveResponse.data.id);
resolve(driveResponse);
}
);
} catch (error) {
console.log('Error in copyPromise:', error);
reject(error);
}
});
};

Related

Lambda returns "Malformed Lambda proxy response"

weirdest thing ever, was trying out one of my endpoints in lambda, everything was working a few days back and nothing was changed.
Basically the functions runs fine up until the point where it needs to return a status code, for some reason, then it just returns a 502 and in the API Gateway it states "Malformed Lambda proxy response"
app.post("/api/v1/documents/create", async (req, res) => {
res.setHeader('Content-Type', 'application/json');
const filename = req.body.filename
const partner = req.body.partner
const payload = req.body
const uid = req.body.uid
console.log(payload)
try {
// Initial setup, create credentials instance.
const credentials = PDFServicesSdk.Credentials
.serviceAccountCredentialsBuilder()
.fromFile("./pdfservices-api-credentials.json")
.build();
// Setup input data for the document merge process.
const jsonString = payload,
jsonDataForMerge = jsonString;
// Create an ExecutionContext using credentials.
const executionContext = PDFServicesSdk.ExecutionContext.create(credentials);
// Create a new DocumentMerge options instance.
const documentMerge = PDFServicesSdk.DocumentMerge,
documentMergeOptions = documentMerge.options,
options = new documentMergeOptions.DocumentMergeOptions(jsonDataForMerge, documentMergeOptions.OutputFormat.PDF);
// Create a new operation instance using the options instance.
const documentMergeOperation = documentMerge.Operation.createNew(options);
// Set operation input document template from a source file.
const input = PDFServicesSdk.FileRef.createFromLocalFile('./darbo_sutartis.docx');
documentMergeOperation.setInput(input);
// Execute the operation and Save the result to the specified location.
documentMergeOperation.execute(executionContext)
.then(result => {
console.log("saving File to TMP?")
result.saveAsFile('/tmp/' + uid + '_' + partner + '.pdf')
const checkTime = 1000;
const timerId = setInterval(() => {
const isExists = fs.existsSync('/tmp/' + uid + '_' + partner + '.pdf', 'utf8')
if (isExists) {
console.log("\nFile written -> creating AWS Bucket")
const params1 = {
Bucket: "darbo-manija-docs",
Key: "employment_documents/" + uid + "/" + partner + "/",
};
s3.putObject(params1, (err, data) => {
if (err) {
console.log(err)
} else {
console.log(data)
}
});
console.log("\nAWS Bucket directory created...")
// do something here
console.log("\nUplaoding file to AWS\n")
fs.readFile('/tmp/' + uid + '_' + partner + '.pdf', function (err, data) {
if (err) throw err;
const pdf = data.toString('base64'); //PDF WORKS
const pdfNew = Buffer.from(pdf, 'base64')
const params = {
Bucket: 'darbo-manija-docs/employment_documents/' + uid + "/" + partner,
Key: uid + '_' + partner + '.pdf', // File name you want to save as in S3
Body: pdfNew, // <---------
ContentType: 'application/pdf'
};
// Uploading files to the bucket
s3.upload(params, function (err, data) {
if (err) {
res.status(400).send(JSON.stringify({
message: "ERR",
code: 0
}));
}
console.log(`\nFile uploaded successfully. ${data.Location}`);
console.log("\nCreating entry in Firebase")
var fb_ref = admin.database().ref('/documents');
fb_ref.push({
documentBucket: params.Bucket,
documentKey: params.Key,
candidate: partner,
employer: uid
})
.then(function (fb_ref) {
admin.database().ref('/documents').child(fb_ref.key).update({
documentID: fb_ref.key
})
});
console.log("\nFirebase entry created");
console.log("\nRemoving temp file...")
fs.unlinkSync('/tmp/' + uid + '_' + partner + '.pdf')
res.status(200).send(JSON.stringify({
result: pdf,
code: 100
}));
});
});
clearInterval(timerId)
}
}, checkTime)
})
.catch(err => {
if (err instanceof PDFServicesSdk.Error.ServiceApiError ||
err instanceof PDFServicesSdk.Error.ServiceUsageError) {
console.log('Exception encountered while executing operation', err);
res.status(400).send(JSON.stringify({
result: "Bad request",
code: 400
}));
} else {
console.log('Exception encountered while executing operation', err);
res.status(400).send(JSON.stringify({
result: "Bad request",
code: 401
}));
}
});
} catch (err) {
console.log('Exception encountered while executing operation', err);
}
});
No idea what is happening, read many posts regarding the same error, but none of them seem to have the same setup. Any suggestions? Thanks

How to properly make a promise.all function with a .map?

I attempted to ask a simplified version of this here but I realized that it probably doesn't contain enough information. So unfortunately I'm just going to post the whole thing and hope that it doesn't offend anyone.
Basically I have 4 functions, with their stated purposes below:
1.InitializeDrive() takes a user email and uses Google's JWT user impersonation method to return the appropriate authorization for that user.
2.listfiles() calls on InitializeDrive() for authorization and retrieves a list of the documents associated with the associated user auth.
3.singleUserData() expects a list of the files such as that from listfiles() and refines them.
4.all_user_data() is intended to be the Promise.all async function that combines all of the aforementioned functions, and maps to an array of users getListUsers(), creating a master array containing all of the refined file data for each user.
Basically my poorly formed and ignorant question is 'how can make the all_user_data() in such a way that it will return the aforementioned master array?'. I am relatively new to async programming, and have a persistent problem getting confused with nested functions.
// loads credentials from .env file
require('dotenv').config();
const util = require('util');
const { google } = require('googleapis');
const { logger } = require('handlebars');
const { getListUsers } = require('./get_users');
const target_users = getListUsers();
function initializeDrive(version, user) {
return new Promise((resolve, reject) => {
const client_email = process.env.GOOGLE_CLIENT_EMAIL;
console.log(client_email);
// add some necessary escaping so to avoid errors when parsing the private key.
const private_key = process.env.GOOGLE_PRIVATE_KEY.replace(/\\n/g, '\n');
// impersonate an account with rights to create team drives
const emailToImpersonate = user;
const jwtClient = new google.auth.JWT(
client_email,
null,
private_key,
['https://www.googleapis.com/auth/drive'],
emailToImpersonate,
);
return google.drive({
version: version,
auth: jwtClient,
});
});
}
const listfiles = async (pagetokenObj, user) => {
let pageToken = '';
let version = 'v3';
if (pagetokenObj !== undefined) {
pageToken = pagetokenObj.pageToken;
}
const drive = await initializeDrive(version, user);
//const drive = initializeDrive(version,user);
return new Promise((resolve, reject) => {
drive.files.list(
{
pageSize: 100,
fields:
'nextPageToken, files(id, name, owners(emailAddress),
sharingUser(emailAddress), permissions)',
...(pageToken ? { pageToken } : {}),
},
function (err, { data: { nextPageToken = '', files = [] } = {} }) {
if (err) {
return reject(err);
}
if (!nextPageToken) {
return resolve(files);
}
// if page token is present we'll recursively call ourselves until
// we have a complete file list.
return listfiles({ pageToken: nextPageToken }).then((otherfiles) => {
resolve(files.concat(otherfiles));
});
},
);
});
};
//Function returns formatted
const singleUserData = async (files) => {
return new Promise((resolve, reject) => {
const single_user_json = await listfiles();
const res = single_user_json
.filter((doc) => Boolean(doc.permissions))
.map((doc) => {
return {
id: doc.id,
sharedWith: doc.permissions
.filter((permission) => permission.type === 'user')
.map((permission) => {
return {
emailAddress: permission.emailAddress,
role: permission.role || null,
};
}), // convert the perm object into a string (email address)
};
});
// this is how you get nicer console.logs instead of just [Object] and [Array] BS
// https://stackoverflow.com/a/10729284/9200245
console.log(util.inspect(res, { showHidden: false, depth: null, colors: true }));
if (err) {
return reject(err);
}
if (!nextPageToken) {
return resolve(files);
};
})
};
const all_user_data = async() => {
const users = await getListUsers();
const pagetokenObj = "{ pageToken = '' } = {}";
Promise.all(
users.map(async (pagetokenObj,user) => {
const files = await listfiles(pagetokenObj, user);
console.log(files);
const singleUserData = await singleUserData(files)
console.log(singleUserData);
}),
);
console.log(users)
}
//returnJSON();
//getListUsers();
//singleUserData();
all_user_data();
I finally figured it out. Basically needed to await a lot more and properly declare async.
// loads credentials from .env file
require('dotenv').config();
const util = require('util');
const { google } = require('googleapis');
const { logger } = require('handlebars');
const { getListUsers } = require('./get_users');
const target_users = getListUsers();
async function initializeDrive(user) {
// return new Promise((resolve, reject) => {
const client_email = process.env.GOOGLE_CLIENT_EMAIL;
console.log(user + ': ' + client_email);
// add some necessary escaping so to avoid errors when parsing the private key.
const private_key = process.env.GOOGLE_PRIVATE_KEY.replace(/\\n/g, '\n');
// impersonate an account with rights to create team drives
const emailToImpersonate = await user;
const jwtClient = new google.auth.JWT(
client_email,
null,
private_key,
['https://www.googleapis.com/auth/drive'],
emailToImpersonate,
);
return google.drive({
version: "v3",
auth: jwtClient,
});
//});
}
//
async function listfiles(pagetokenObj, user) {
let pageToken = '';
if (pagetokenObj !== undefined) {
pageToken = pagetokenObj.pageToken;
}
const drive = await initializeDrive(user);
//console.log(drive)
return new Promise((resolve, reject) => {
drive.files.list(
{
pageSize: 100,
fields:
'nextPageToken, files(parents, id, name, properties, owners(emailAddress), sharingUser(emailAddress), permissions)',
...(pageToken ? { pageToken } : {}),
},
function (err, { data: { nextPageToken = '', files = [] } = {} }) {
if (err) {
return reject(err);
}
if (!nextPageToken) {
return resolve(files);
}
// if page token is present we'll recursively call ourselves until
// we have a complete file list.
return listfiles({ pageToken: nextPageToken }).then((otherfiles) => {
resolve(files.concat(otherfiles));
});
},
);
});
};
async function singleUserData(user) {
const pagetokenObj = "{ pageToken = '' } = {}"
const single_user_json = await listfiles(pagetokenObj, user);
// return new Promise ((resolve, reject) => {
console.log(user+ ":" + JSON.stringify(single_user_json));
const res = await single_user_json
.filter((doc) => Boolean(doc.permissions))
.map((doc) => {
return {
id: doc.id,
sharedWith: doc.permissions
.filter((permission) => permission.type === 'user')
.map((permission) => {
return {
emailAddress: permission.emailAddress,
role: permission.role || null,
};
}) // convert the perm object into a string (email address)
};
});
return JSON.stringify(res);
//})
}
async function getAllUserData() {
try {
const users = await getListUsers();
// const userString = JSON.parse(users);
console.log("test22222222222222: " + users)
const usersData = await Promise.all(
users.map((user) => {
return singleUserData(user);
})
);
console.log("[" + usersData + "]");
return usersData;
} catch (error) {
console.log(error);
}
}
getAllUserData();
You're almost there.
Remove listfiles call in all_user_data
Pass user to singleUserData
Remove pagetokenObj in all_user_data
Remove async and await inside users.map and return promise directly
Await Promise.all, assign it to a variable and return that variable
Remove unnecessary Promise wrapping in singleUserData
Change function signature of singleUserData to take in a user
Pass user to listfiles in singleUserData
Return res instead of files in singleUserData
Here are the changes I made to all_user_data and singleUserData:
// Function returns formatted
async function singleUserData(user) {
const single_user_json = await listfiles(user);
const res = single_user_json
.filter((doc) => Boolean(doc.permissions))
.map((doc) => {
return {
id: doc.id,
sharedWith: doc.permissions
.filter((permission) => permission.type === "user")
.map((permission) => {
return {
emailAddress: permission.emailAddress,
role: permission.role || null,
};
}), // convert the perm object into a string (email address)
};
});
return res;
};
async function getAllUserData() {
const users = await getListUsers();
const usersData = await Promise.all(
users.map((user) => {
return singleUserData(user);
})
);
return usersData;
};
const usersData = await getAllUserData();

Kuzzle / Minio example usage

Dose the Kuzzle or Minio development teams have a working example of using the Kuzzle S3 plugin for Minio? I have the following but my file isnt being uploaded and the pre-signed url is referring to https://your-s3-bucket.s3.eu-west-3.amazonaws.com/
const fs = require("fs");
const fsPromises = require('fs').promises;
// Create a JS File object instance from a local path using Node.js
const fileObject = require("get-file-object-from-local-path");
// Promise based HTTP client for the browser and node.js
const axios = require('axios');
// Loads the Kuzzle SDK modules
const {
Kuzzle,
WebSocket
} = require('kuzzle-sdk');
var start = new Date();
const webSocketOptionsObject = {
"autoReconnect": true,
"ssl": true,
"port": 443
};
const kuzzle = new Kuzzle(new WebSocket('myurl.com', webSocketOptionsObject));
const credentials = { username: 'xyz123', password: 'fithenmgjtkj' };
const path = __dirname + "\\" + "yellow_taxi_data.csv"; // the "\\" is for Windows path
var fileData = {};
// check file exists
fs.access(path, fs.F_OK, (err) => {
if (err) {
console.error(err)
return
}
fileData = new fileObject.LocalFileData(path);
// Adds a listener to detect connection problems
kuzzle.on('networkError', error => {
console.error('Network Error:', error);
});
});
const connectToKuzzle = async () => {
// Connects to the Kuzzle server
await kuzzle.connect();
return await kuzzle.auth.login('local', credentials);
// console.log('jwt auth token: ', jwt);
}
const disConnectFromKuzzle = async () => {
console.log('Disconnected from Kuzzle');
kuzzle.disconnect();
var time = new Date() - start;
// sec = Math.floor((time/1000) % 60);
console.log('Execution time in milliseconds: ', time);
}
const presignedURL = async () => {
// Get a Presigned URL
const result = await kuzzle.query({
controller: 's3/upload',
action: 'getUrl',
uploadDir: 'proxybucket', // directory name inside the Bucket specified in the s3 plugin bucket name
filename: fileData.name
});
console.log("result: ", result);
return result;
}
const loadFileStream = async () => {
console.log('getting file: ', path);
targetFile = null;
await fs.promises.readFile(path)
.then(function (result) {
console.log("file loaded------", result.length);
targetFile = result;
})
.catch(function (error) {
console.log(error);
return;
});
return targetFile;
}
const kuzzleValidate = async (kuzzleResource) => {
// console.log("kuzzleResource: ", kuzzleResource.result.fileKey);
// validate
// Validate and persist a previsously uploaded file.
// https://docs.kuzzle.io/official-plugins/s3/2/controllers/upload/validate/
const Presult = await kuzzle.query({
// Kuzzle API params
"controller": "s3/upload",
"action": "validate",
// File key in S3 bucket
"fileKey": kuzzleResource.result.fileKey
});
console.log('validate: ', Presult.result.fileUrl);
}
const uploadFile = async (fileBuffer, kuzzleResource, jwt) => {
// options at https://github.com/axios/axios
const axiosOptions = {
headers: {
'Content-Type': fileData.type
},
maxBodyLength: 200000000 // 200,000,000 bytes 200 Mb
};
// PUT the fileBuffer to the Kuzzle S3 endpoint
// https://github.com/axios/axios
axios.defaults.headers.common['Authorization'] = jwt;
const response = await axios.put(kuzzleResource.result.uploadUrl, fileBuffer, axiosOptions)
.then((response) => {
console.log('file uploaded......');
})
.catch(function (error) {
console.log("File upload error: ", error);
return;
});
return "Upload successful";
}
if (fileData) {
connectToKuzzle().then((jwt) => {
console.log(jwt);
// upload(jwt);
presignedURL().then((kuzzleResource) => {
loadFileStream().then((fileBuffer) => {
uploadFile(fileBuffer, kuzzleResource, jwt).then((doneMessage) => {
console.log("doneMessage: ", doneMessage);
}).then(() => {
kuzzleValidate(kuzzleResource).then(() => {
disConnectFromKuzzle();
});
});
});
});
});
}
I'm looking to upload to a Minio bucket and obtain a pre-signedURL so I can store it in a document later.
You can change the endpoint configuration to set a different s3-compatible endpoint who can be a Minio one.
This configuration can be changer under the plugins.s3.endpoint key. You should also disable the usage of default s3 path.
Example:
app.config.set('plugins.s3.endpoint', 'https://minio.local');
app.config.set('plugins.s3.s3ClientOptions.s3ForcePathStyle', false);

Create folder returns undefined when trying to find file id

I am using Google drive API to create a folder and I want to return the id of this folder after creating it so I can make a folder tree. I don't understand why I am unable to do this, even using promises I consistently get the result of undefined for the id of the folder. This code returns undefined for the id of the folder. I don't want to use the list method to search the drive after creating the folder because it's slow. Any ideas?
const fs = require('fs');
const readline = require('readline');
const {google, run_v1} = require('googleapis');
const SCOPES = ['https://www.googleapis.com/auth/drive'];
const TOKEN_PATH = 'token.json';
const CLIENT_NAME = 'Bowie, David';
const ROOT_FOLDER_LOCATION = '11ljl1lk1lk';
function getCreds(){
return new Promise((resolve, reject) => {
fs.readFile('credentials.json', (err, content) => {
if (err) {
reject('error loading client secret');
} else{
resolve(JSON.parse(content));
}
})
})
}
function authorize(creds){
const {client_secret, client_id, redirect_uris} = creds.installed;
const oAuth2Client = new google.auth.OAuth2(
client_id, client_secret, redirect_uris[0]);
return new Promise((resolve, reject) => {
fs.readFile(TOKEN_PATH, (err, token) => {
if (err) {
reject('this failed authorization');
} else {
oAuth2Client.setCredentials(JSON.parse(token));
resolve(oAuth2Client);
}
})
})
}
function makeRoot(auth) {
const drive = google.drive({version: 'v3', auth});
var fileMetadata = {
'name': CLIENT_NAME,
'mimeType': 'application/vnd.google-apps.folder',
'parents': [ROOT_FOLDER_LOCATION]
};
return new Promise((resolve, reject) => {
drive.files.create({
resource: fileMetadata,
fields: 'id'
}, function(err, file) {
if (err) {
reject(err);
} else {
resolve('Folder ID:', file.id);
}
})
})
}
async function doWork() {
try {
const response = await getCreds();
const oAuth2Client = await authorize(response);
const rootFolder = await makeRoot(oAuth2Client);
console.log(rootFolder);
} catch (err) {
console.log(err)
}
}
doWork();
The only output I get is "undefined"...
Modification points:
When you want to retrieve the folder ID from the created new folder using googleapis for Node.js, in the case of your script, you can retrieve it using file.data.id. Ref1, Ref2
But, in your script, when resolve('Folder ID:', file.id); is run, rootFolder of const rootFolder = await makeRoot(oAuth2Client); is Folder ID:.
When you want to retrieve the value of Folder ID:###folderId###, please modify as follows.
From:
resolve('Folder ID:', file.id);
To:
resolve("Folder ID:" + file.data.id);
If you want to separate the values of Folder ID: and ###folderId###, you can also use the following modification. In this case, you can retrieve the folder ID like const [text, rootFolder] = await makeRoot(oAuth2Client);.
resolve(["Folder ID:", file.data.id]);
Reference:
google-api-nodejs-client

Firebase functions promises not firing correctly

I'm having a hard time understanding promises in Firebase functions. I have a function that listens for new files in a storage bucket and then emails the user as well as sending them a Discord message. I'm getting inconsistent results and I'm pretty sure its to do with promises and callbacks being setup incorrectly.
exports.sendSetup = functions.storage.bucket('*the-bucket-id*').object().onFinalize((object) => {
// Get a URL for the new config file
console.log('New conf file: ' + object.name);
const { Storage } = require('#google-cloud/storage');
const storage = new Storage({
projectId: '*the-project-id*',
keyFilename: 'googleServiceAccountKey.json'
});
var bucket = storage.bucket('*the-bucket-name*');
const file = bucket.file(object.name);
console.log('Generating download url for file ' + object.name);
return file.getSignedUrl({
promptSaveAs: '*the-file-name*',
action: 'read',
expires: '03-09-2491'
}).then(signedUrls => {
var split = object.name.split("/");
var env = split[0];
var customer_id = split[1];
getCustomerDetails(customer_id, signedUrls[0], env);
});
});
function getCustomerDetails(customer_id, fileUrl, env) {
console.log('Retrieving customer details for customer id ' + customer_id + ' from Stripe');
var stripe = stripeLive;
if (env == 'test') {
stripe = stripeTest;
}
stripe.customers.retrieve(
customer_id,
function (err, customer) {
if (err == null) {
sendMail(fileUrl, customer.email, customer_id, customer.metadata.name);
console.log('discordId= ' + customer.metadata.discordId);
if (customer.metadata.discordId != 'undefined') {
sendDiscord(fileUrl, customer.metadata.discordId, customer.metadata.discordName);
}
console.log('Finished');
} else {
console.log(err);
}
}
);
}
function sendDiscord(fileUrl, discordId, discordName) {
console.log('Attempting to send a discord message to Discord id ' + discordId);
const Discord = require('discord.js');
const client = new Discord.Client();
client.login('*discord-api-key*');
client.once('ready', () => {
console.log('Discord client ready');
client.fetchUser(discordId)
.then((User) => {
console.log('Got Discord user object. Attempting to send message');
return User.send({
embed: {
color: 3447003,
fields: [
{
name: 'Hey ' + discordName + '!',
value: 'Below are the instructions to get you up and running'
},
{
name: '**Step 1**',
value: 'some instructions'
}
]
}
});
})
.catch((err) => {
console.log(err);
})
});
}
function sendMail(fileUrl, customer_email, customer_id, customer_name) {
console.log('customer_name in sendMail function = ' + customer_name);
var firstName = customer_name.substring(0, customer_name.indexOf(' '));
console.log(firstName);
const sgMail = require('#sendgrid/mail');
sgMail.setApiKey(*sendGridApiKey*);
sgMail.setSubstitutionWrappers('{{', '}}'); // Configure the substitution tag wrappers globally
const msg = {
to: customer_email,
subject: 'Welcome!',
from: {
email: 'noreply#example.com.au',
name: 'me'
},
text: 'Let\'s get you setup...',
html: '<p></p>',
templateId: '*template-id*',
substitutions: {
first_name: firstName,
file_url: fileUrl
},
};
console.log('Sending email to ' + customer_email + ' customer id:' + customer_id);
sgMail.send(msg);
}
I've read a heap of articles about promises and callbacks but can't seem to wrap my head around it. The "sendSetup" function actually returns OK but appears to stop right at the start of the getCustomerDetails function. Appreciate any assistance! I'm a bit lost!

Resources