Node.js node-gcloud synchronous call - node.js

I'm using node-gcloud https://github.com/GoogleCloudPlatform/gcloud-node to interact with Google Cloud Storage.
I'm developing a node.js server (my first node.js project) to provide a small set of APIs to clients. Basically when an user uploads a file the API call return the signed url to show that file.
The getSignedUrl function is asynchronous https://googlecloudplatform.github.io/gcloud-node/#/docs/v0.8.1/storage?method=getSignedUrl and I can't find a way to return that result from another function.
I've started playing with Bluebird promises but I can't get to the point of it. Here is my code:
var _signedUrl = function(bucket,url,options) {
new Promise(function (resolve, reject) {
var signed_url
bucket.getSignedUrl(options, function(err, url) {
signed_url = err || url;
console.log("This is defined: " + signed_url)
return signed_url
})
})
}
var _getSignedUrl = function(url) {
new Promise(function(resolve) {
var options = config.gs
, expires = Math.round(Date.now() / 1000) + (60 * 60 * 24 * 14)
, bucket = project.storage.bucket({bucketName: config.gs.bucket, credentials: config.gs })
, signed_url = null
options.action = 'read'
options.expires = expires// 2 weeks.
options.resource= url
signed_url = resolve(_signedUrl(bucket,url,options))
console.log("This is undefined: " + signed_url)
return JSON.stringify( {url: signed_url, expires: expires} );
});
}
I think that I'm missing the basics of how it is supposed to work, so any hint will be appreciated.
Edit:
I have reworked my solution as for the first comment:
getSignedUrl: function() {
var options = config.gs
, expires = Math.round(Date.now() / 1000) + (60 * 60 * 24 * 14)
, bucket = project.storage.bucket({bucketName: config.gs.bucket, credentials: config.gs })
, signed_url = null
options.action = 'read'
options.expires = expires// 2 weeks.
options.resource= this.url
Promise.promisifyAll(bucket);
return bucket.getSignedUrlAsync(options).catch(function(err) {
return url; // ignore errors and use the url instead
}).then(function(signed_url) {
return JSON.stringify( {url: signed_url, expires: expires} );
});
}
It's not clear to me how the double return is supposed to work, but if I keep the
return bucket
what I get is this output:
{ url:
{ _bitField: 0,
_fulfillmentHandler0: undefined,
_rejectionHandler0: undefined,
_promise0: undefined,
_receiver0: undefined,
_settledValue: undefined,
_boundTo: undefined }
}
, and if remove it and keep the
return JSON.stringify( {url: signed_url, expires: expires} );
I get undefined as before. What am I missing?

Some points:
Inside a new Promise(function(res, rej){ … }) resolver callback, you actually need to call resolve() or reject() (asynchronously), not return anything.
resolve doesn't return anything. You seemed to use it like a "wait" operation that returns the result of the promise, but such is impossible. A promise is still asynchronous.
Actually, you should never need to call new Promise at all. Use Promisification instead.
Your code should rather look like
var gcloud = require('gcloud');
Promise.promisifyAll(gcloud); // if that doesn't work, call it once on a
// specific bucket instead
function getSignedUrl(url) {
var options = config.gs,
expires = Math.round(Date.now() / 1000) + (60 * 60 * 24 * 14),
bucket = project.storage.bucket({bucketName: config.gs.bucket, credentials: config.gs });
options.action = 'read';
options.expires = expires; // 2 weeks.
options.resource = url;
return bucket.getSignedUrlAsync(options).catch(function(err) {
return url; // ignore errors and use the url instead
}).then(function(signed_url) {
console.log("This is now defined: " + signed_url);
return JSON.stringify( {url: signed_url, expires: expires} );
});
}

Related

Simple fetch from API in Nodejs but UnhandledPromiseRejectionWarning: nestedFunction is not a Function

I'm a non-professional using nodejs server (backend) and javascript/html (frontend) to fetch data from two API's: one API gives a response and I use an ID from the first API to fetch data from the other API. The API returns XML so I use XML2Json and JSON.parse to get the Javascript Object.
everything works perfect until I get to the "return nestedFunction(new_details")-function in the second API-call
so this is where the results are sent back to the client
I do it for the first API and it works fine (backend + frontend)
I tried Async/await but the problem isn't solved
I get the error: "UnhandledPromiseRejectionWarning: TypeError: nestedFunction is not a function"
What could be the problem?
app.get('/AATGetTermMatch', function(req,res) {
let termMatch = req.query.term;
let termLogop = req.query.logop;
let termNotes = req.query.notes;
AATGetTermMatch(termMatch, termLogop, termNotes, function (conceptResults) {
res.send(conceptResults);
});
});
function AATGetTermMatch(termMatch, termLogop, termNotes, nestedFunction) {
let URL = baseURL + "AATGetTermMatch?term=" + termMatch + "&logop=" + termLogop + "&notes=" + termNotes;
fetch(URL)
.then(function (response){
return response.text();
}).then(function (response) {
APICallResults = response;
parseJson();
let objectAPI = JSON.parse(APICallResults);
let full_Concepts = objectAPI.Vocabulary.Subject;
let i;
for (i = 0; i < full_Concepts.length; i++) {
global.ID = full_Concepts[i].Subject_ID;
searchTermDetails(global.ID);
} return nestedFunction(full_Concepts);
});
}
app.get('/subjectID', function(req, res) {
let selectedID = req.query.subjectID;
searchTermDetails(selectedID, function (termDetails) {
res.json(termDetails);
});
});
2nd API : http://vocabsservices.getty.edu/AATService.asmx/AATGetSubject?subjectID=300004838
function searchTermDetails(selectedID, nestedFunction) {
selectedID = global.ID;
let URL_Details = baseURL + "AATGetSubject?" + "subjectID=" + selectedID;
fetch(URL_Details)
.then(function (response) {
return response.text();
}).then(function (response) {
APICallResults_new = response;
parseJsonAgain();
let detailAPI = JSON.parse(APICallResults_new);
let new_details = detailAPI.Vocabulary.Subject;
let Subject_ID = new_details[0].Subject_ID;
let descriptive_Notes_English = new_details[0].Descriptive_Notes[0].Descriptive_Note[0].Note_Text;
} **return nestedFunction(new_details);**
}).catch(function (error) {
console.log("error");
});
}
function parseJson() {
xml2js.parseString(APICallResults, {object: true, trim:true, sanitize: true, arrayNotation: true, mergeAttrs: true}, (err, result) => {
if (err) {
throw err;
}
const resultJson = JSON.stringify(result, null, 4);
//JSON.parse(resultJson);
APICallResults = resultJson;
});
}
function parseJsonAgain() {
xml2js.parseString(APICallResults_new, {object: true, trim:true, sanitize: true, arrayNotation: true, mergeAttrs: true}, (err, result) => {
if (err) {
throw err;
}
const resultJsonAgain = JSON.stringify(result, null, 4);
APICallResults_new = resultJsonAgain;
//console.log(APICallResults_new);
});
}
I've read many threads about this error but the proposed solutions don't seem to work.
In here:
for (i = 0; i < full_Concepts.length; i++) {
global.ID = full_Concepts[i].Subject_ID;
searchTermDetails(global.ID);
}
where you call searchTermDetails(), you are not passing the nestedFunction second argument. Thus, when searchTermDetails() tries to use that argument, it causes the error you get.
You can either add the callback to this call or, if the callback could be optional, then you can modify searchTermDetails to check to see if the second argument is a function and, if not, then don't try to call it.

How to write clean asynchronous requests that can be called again

I'm attempting to clean up my code with Promises and Async await. My problem is that I need these requests to be recallable with the same handling afterwards.
I've tried Promises, but if I nest everything in functions, it gets really messy fast. How do I make this code so that it only continues in the go() async function when a value is returned?
const request = require('request-promise');
require('console-stamp')(console, 'HH:MM:ss.l');
const colors = require('colors');
const kws = 'sweatsasaaser'.toLowerCase();
const size = 'Small';
go();
async function go(){
const f = await getproduct()
console.log('Finished ' + f)
if (f == undefined) getproduct()
}
async function getproduct(){
console.log('Requesting')
let result = await request('https://www.supremenewyork.com/mobile_stock.json');
let data = JSON.parse(result);
let prodid;
for (var i = 0; i < data.products_and_categories['Tops/Sweaters'].length; i++){
if (data.products_and_categories['Tops/Sweaters'][i].name.toLowerCase().includes(kws)){
console.info('Found product: '.green + data.products_and_categories['Tops/Sweaters'][i].name.green);
return prodid = data.products_and_categories['Tops/Sweaters'][i].id;
};
};
if (prodid == undefined){
console.log(`Product id: ${prodid}`.blue);
return prodid;
}
else {
setTimeout(function(){
//getproduct()
}, 4000);
}
}
Write a separate function:
/**
* Re-executes an async function n times or until it resolves
* #param {function} fn Function to call
* #param {number} [times=3] Times to retry before rejecting
* #param {number} [delay=1000] Delay between retries
* #param {number} [i=0] Counter for how many times it's already retried
*/
async function retry(fn, times = 3, delay = 1000, i = 0) {
try {
return await fn()
} catch (error) {
if (i < times) {
await new Promise(r => setTimeout(r, delay))
return retry(fn, times, delay, i + 1)
}
else throw error;
}
}
Have your main function getproduct simply throw the error
else {
// setTimeout(function(){
// //getproduct()
// }, 4000)
throw new Error('Cannot get productid')
}
And use it with the new retry function:
async function go(){
const f = await retry(getproduct, 3)
In case you wanna pass arguments, simply wrap it
const f = await retry(() => getproduct(...args), 3)

Lambda function taking >3 seconds to run + 5-10 secs warmup each time

I have a simple node.js function with 2 REST API calls and a socket connection output hosted in an AWS lambda. It takes 5-10 secs warmup time and >3+ secs execution time.
When the code is run locally it executes both requests, socket connection and completes in about ~1300ms. Why is AWS more then double the execution time? I have set-timeout to 120seconds and memory at 128mb (default).
I appreciate the code is not very tidy; I am working on cleaning it but needed something going for the time being.
The project simply gets info from ServiceM8 via API when called by a webhook subscription, then formats the info into ZPL strings and forwards them to a tcp server for printing via thermal printer.
My questions are:
Is it my code bottle necking?
Can it be optimized to run faster?
Do i simply need to employ a warming plugin for my function to allow hot starting?
My function:
'use strict';
//Require libraries
var request = require("request");
var net = require('net');
exports.handler = (event, context, callback) => {
if (event.eventName != 'webhook_subscription') {
callback(null, {});
}
//Global Variables
var strAssetUUID;
var strAssetURL;
var strFormUUID;
var strTestDate;
var strRetestDate;
var appliancePass = true;
var strAccessToken;
var strResponseUUID;
//Printer Access
const tcpUrl = 'example.com';
const tcpPort = 12345;
var client = new net.Socket();
//UUID of Appliance Test Form.
const strTestFormUUID = 'UUID_of_form';
//Begin function
/**
* Inspect the `eventArgs.entry` argument to get details of the change that caused the webhook
* to fire.
*/
strResponseUUID = event.eventArgs.entry[0].uuid;
strAccessToken = event.auth.accessToken;
console.log('Response UUID: ' + strResponseUUID);
console.log('Access Token: ' + strAccessToken);
//URL Options for FormResponse UUID query
const urlFormResponse = {
url: 'https://api.servicem8.com/api_1.0/formresponse.json?%24filter=uuid%20eq%20' + strResponseUUID,
headers: {
// Use the temporary Access Token that was issued for this event
'Authorization': 'Bearer ' + strAccessToken
}
};
//Query form Response UUID to get information required.
request.get(urlFormResponse, function(err, res, body) {
//Check response code from API query
if (res.statusCode != 200) {
// Unable to query form response records
callback(null, {err: "Unable to query form response records, received HTTP " + res.statusCode + "\n\n" + body});
return;
}
//If we do recieve a 200 status code, begin
var arrRecords = JSON.parse(body);
//Store the UUID of the form used for the form response.
strFormUUID = arrRecords[0].form_uuid;
console.log('Form UUID: ' + strFormUUID);
//Store the UUID of the asset the form response relates to.
strAssetUUID = arrRecords[0].asset_uuid;
console.log('Asset UUID: ' + strAssetUUID);
if (strFormUUID == strTestFormUUID){
//Get the edited date and parse it into a JSON date object.
var strEditDate = new Date(arrRecords[0].edit_date);
//Reassemble JSON date to dd-mm-yyyy.
strTestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
//Extract the response for retest period.
var strRetestAnswer = JSON.parse(arrRecords[0].field_data);
strRetestAnswer = strRetestAnswer[0].Response;
//Appropriate function based on retest response.
switch(strRetestAnswer) {
case '3 Months':
//Add x months to current test date object
strEditDate.setMonth(strEditDate.getMonth() + 3);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '6 Months':
strEditDate.setMonth(strEditDate.getMonth() + 6);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '12 Months':
strEditDate.setMonth(strEditDate.getMonth() + 12);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '2 Years':
strEditDate.setMonth(strEditDate.getMonth() + 24);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '5 Years':
strEditDate.setMonth(strEditDate.getMonth() + 60);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
default:
strRetestDate = "FAIL";
appliancePass = false;
}
console.log('Appliance Pass: ' + appliancePass);
console.log('Test Date: ' + strTestDate);
console.log('Retest Period: ' + strRetestAnswer);
console.log('Retest Date: ' + strRetestDate);
//URL Options for Asset UUID query
const urlAssetResponse = {
url: 'https://api.servicem8.com/api_1.0/asset/' + strAssetUUID + '.json',
headers: {
// Use the temporary Access Token that was issued for this event
'Authorization': 'Bearer ' + strAccessToken
}
};
//Query the api for the asset URL of the provided asset UUID.
request.get(urlAssetResponse, function(err, res, body) {
//Check response code from API query
if (res.statusCode != 200) {
// Unable to query asset records
callback(null, {err: "Unable to query asset records, received HTTP " + res.statusCode + "\n\n" + body});
return;
}
//If we do recieve a 200 status code, begin
var strAssetResponse = JSON.parse(body);
//Store the asset URL
strAssetURL = 'https://sm8.io/' + strAssetResponse.asset_code;
console.log('Asset URL: ' + strAssetURL);
//generate tag and send to printer
var strZPLPass = ('^XA....^XZ\n');
var strZPLFail = ('^XA....^XZ\n');
//Now that we have our ZPL generated from our dates and URLs
//Send the correct ZPL to the printer.
client.connect(tcpPort, tcpUrl, function() {
console.log('Connected');
//Send Appropriate ZPL
if (appliancePass) {
client.write(strZPLPass);
}else {
client.write(strZPLFail);
}
console.log('Tag Successfully Printed!');
//As the tcp server receiving the string does not return any communication
//there is no way to know when the data has been succesfully received in full.
//So we simply timeout the connection after 750ms which is generally long enough
//to ensure complete transmission.
setTimeout(function () {
console.log('Timeout, connection closing...');
client.destroy();
}, 750);
});
});
}
});
};
First of all, I would suggest you stop using the request module and switch to native. Everything can be done without a tons of lines these days. request is a module with 48 total dependencies; if you do the math, that's thousands of lines for a simple GET request.
You should always minimize the complexity of your dependencies. I use a Lambda to check the health of my sites, grabbing the whole request and checking the HTML on completely different servers. VPS is located in Frankfurt, AWS in Ireland. My ms/request is ranging between 100~150 ms.
Here's a simple promise request I'm using:
function request(obj, timeout) {
return new Promise(function(res, rej) {
if (typeof obj !== "object") {
rej("Argument must be a valid http request options object")
}
obj.timeout = timeout;
obj.rejectUnauthorized = false;
let request = http.get(obj, (response) => {
if (response.statusCode !== 200) {
rej("Connection error");
}
var body = '';
response.on('data', (chunk) => {
body += chunk;
});
response.on('end', () => {
res(body);
});
response.on('error', (error) => {
rej(error);
});
});
request.setTimeout(timeout);
request.on('error', (error) => {
rej(error);
})
request.on('timeout', () => {
request.abort();
rej("Timeout!")
})
});
}
Example
const reqOpts = {
hostname: 'www.example.com',
port: 443,
path: '/hello',
method: 'GET',
headers: {
handshake: "eXTNxFMxQL4pRrj6JfzQycn3obHL",
remoteIpAddress: event.sourceIp || "lambda"
}
}
try {
httpTestCall = await request(reqOpts, 250);
}
catch (e) {
console.error(e);
}
Now based on that change switch your handler to async using exports.handler = async(event, context, callback) => {} and use console to measure the execution time of every request using console.time() and console.timeEnd() for your request or anything. From there you could see what's stepping down your code using Cloudwatch logs. Here's another example based on your code:
let reqOpts = {
hostname: 'api.servicem8.com',
port: 443,
path: '/api_1.0/formresponse.json?%24filter=uuid%20eq%20' + strResponseUUID,
method: 'GET',
headers: {
// Use the temporary Access Token that was issued for this event
'Authorization': 'Bearer ' + strAccessToken
}
}
console.time("=========MEASURE_servicem8=========")
let error = null;
await request(reqOpts, 5555).catch((e)=>{
error = e;
})
console.timerEnd("=========MEASURE_servicem8=========")
if (error){
callback(null, {err: "Unable to query form response records, received HTTP" + error}); /* or anything similar */
}
References
https://docs.aws.amazon.com/lambda/latest/dg/best-practices.html
https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
aws lambdas are not fast by nature (as of writing this answer). The startup time is not guaranteed, and it is known to be high.
If you need performance - you will not get it this way.

Why is PUT request body undefined?

I'm making the following request to my koajs server:
$.ajax({
type : 'PUT', // this.request.body undefined server side
// type : 'POST', // this.request.body all good server side
url : url,
data : body,
dataType : 'json'
})
But on the server side this.request.body is always undefined.
If I change the request type to POST, it works fine.
Any ideas?
EDIT
I'm using koa-route.
EDIT 2
Just realised I'm using koa-body-parser, which is probably more relevant.
Try using the koa-body parser:
const bodyParser = require('koa-bodyparser')
app.use(bodyParser())
I think koa-router will parse typical request stuff, url params, forms etc. If you want to parse the body of a request that contains a JSON object you need to apply a middleware (as alex alluded to).
Also please check to see if you are putting valid JSON.
Take a look at this Koa-bodyparser:
/**
* #param [Object] opts
* - {String} jsonLimit default '1mb'
* - {String} formLimit default '56kb'
* - {string} encoding default 'utf-8'
*/
return function *bodyParser(next) {
if (this.request.body !== undefined) {
return yield* next;
}
if (this.is('json')) {
this.request.body = yield parse.json(this, jsonOpts);
} else if (this.is('urlencoded')) {
this.request.body = yield parse.form(this, formOpts);
} else {
this.request.body = null;
}
yield* next;
};
there looks to be a 1mb limit on the amount of JSON. then to co-body/lib/json.js
module.exports = function(req, opts){
req = req.req || req;
opts = opts || {};
// defaults
var len = req.headers['content-length'];
if (len) opts.length = ~~len;
opts.encoding = opts.encoding || 'utf8';
opts.limit = opts.limit || '1mb';
return function(done){
raw(req, opts, function(err, str){
if (err) return done(err);
try {
done(null, JSON.parse(str));
} catch (err) {
err.status = 400;
err.body = str;
done(err);
}
});
}
};

sending response in message passing on history search

I have been trying to send search history in the background page and send response back to front page. I am using message parsing .
I am unable to send response back after searching history.
--------background.js-------------
var SECONDS_PER_WEEK = 1000 * 60 * 60 * 24 * 7;
chrome.extension.onMessage.addListener(
function(request, sender, sendResponse) {
if(request === "getHistory"){
var value = getHistory();
console.log("Response" , value);
sendResponse(value);
}
}
);
function getHistory(){
var current_time = new Date().getTime();
var timeToFetch = current_time - SECONDS_PER_WEEK;
return chrome.history.search({
'text' : '',
'startTime' : timeToFetch
},
function(resp){
return resp;
});
}
The problem lies here itself as on logging I get 'undefined' as response.
Can anyone direct me to the solution
Most of the Chrome.* API functions are asynchronous so try doing it something like this:
background.js
var SECONDS_PER_WEEK = 1000 * 60 * 60 * 24 * 7;
chrome.extension.onMessage.addListener(function(request, sender, sendResponse) {
if(request === "getHistory"){
getHistory(sendResponse);
return true;
}
});
function getHistory(sendResponse){
var current_time = new Date().getTime();
var timeToFetch = current_time - SECONDS_PER_WEEK;
chrome.history.search({
'text' : '',
'startTime' : timeToFetch
},
function(resp){
console.log("Response: " + resp);
sendResponse(resp);
});
}

Resources