AWS nodejs lambda persist output of asynchronous call in JavaScript object - node.js

It's a silly question perhaps, but I spent quite sometime trying to resolve but not able to get the data.
In the below function checkMailCount, I have an object called test with a function in it to assign the value to mails. The value assigned from res.on('end') to this test.mails via mailCount is not retained/persisting.
'use strict';
const https = require('https');
exports.handler = (event, context, callback) => {
let session = event.session;
let request = event.request;
if (request.type === "something") {
if (request.name === "blah blah") {
let emailCount = checkMailCount(session, callback);
context.succeed(callback(null, emailCount));
}
}
function checkMailCount(session, callback) {
let url = 'https://www.googleapis.com/gmail/v1/users/me/messages?
access_token = $ {
session.user.accesstoken
} & q = "is:unread"
';
let test = {
mails: "",
mailCount: function(val) {
this.mails = val;
}
};
let data = "";
https.get(url, function(res) {
res.on('data', function(chunk) {
data += chunk;
//console.log(data);
});
res.on('end', function() {
let result = JSON.parse(data);
//result2+= result.replace('/\//g','');
test.mailCount(result["resultSizeEstimate"]);
// result["resultSizeEstimate"] inaccessible outside
});
});
return test.mails; // This is returning undefined
}
//context.fail();
}
Below is the response JSON from gmail API
{
"messages": [
{
"id": "165f1627a53f70c6",
"threadId": "165f000031cee15b"
},
{
"id": "165f00d9e2e07737",
"threadId": "165f00d9e2e07237"
}
],
"nextPageToken": "10349401829330188334",
"resultSizeEstimate": 2
}
I need to return the value of result["resultSizeEstimate"] from the function checkMailCount. The value of result, result["resultSizeEstimate"] is available inside res.on('end', f(){It is available here}) but not outside.
I tried this in VS code not as lambda, I was able to do it by other means. Its sort of hard to unit test lambda functions. Is there a way, I can send the email count to context.succeed(callback(null, emailCount)) ?

It seems like there's a fundamental issue with callbacks in this code. It is not possible for checkMailCount to behave like you want it to in its current form. checkMailCount is asynchronous. JavaScript has a few different ways to deal with asynchronous code. The oldest is what you're doing here, callbacks. Next is promises and finally async/await. This code is probably easier to write if you switch to async/await. That said, if you want to keep using the callback pattern you're gonna have to ya know, use them.
Instead of passing lambda's callback to checkMailCount you should pass your own.
checkMailCount(session, function(err, result) {
// result === result.resultSizeEstimate
// Call your lambda callback from within this function
});
res.on('end', function() {
let result = JSON.parse(data);
test.mailCount(result["resultSizeEstimate"]);
callback(null, result.resultSizeEstimate)
});

Related

Cannot access value from function node.js

I am trying to access what a function returns in node.js
I have the following function:
function getImg(callback) {
https.get('https://api.nasa.gov/planetary/apod?api_key=api-key', response => {
let data = "";
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
let img = JSON.parse(data).hdurl;
callback(null, img);
})
}).end();
}
let image = getImg(function(err, image) {
console.log(image);
})
res.render('index', {
indexCSS: true,
image
})
It can log it to the console correctly, but if I want to access the value of the variable like I do in the last line of my code or if I console.log(image) I get undefined.
What have I done wrong. How can I access what the function produces?
It is a callback style function which wouldn't return any thing. Better to convert it to promise sttyle and use async/await to get the value in a variable
function getImg() {
return new Promise((resolve, reject) => {
https.get('https://api.nasa.gov/planetary/apod?api_key=api-key', response => {
let data = "";
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
let img = JSON.parse(data).hdurl;
resolve(img);
})
}).end();
});
}
(async() => {
let image = await getImg();
res.render('index', {
indexCSS: true,
image
});
})();
You can't really store the return value of your function like that. Unfortunately JS is non-blocking, so the code will continue to execute past it, before it has a chance to return that value from the https request. I am not sure exactly when you call this function, but you could call res.render in the callbacks response after calling getImg() without assigning its value to something. You can use promises, otherwise it's better to handle the response you need when it is returned from the callback. That would just be a simple call like:
getImg(function(err, image) {
res.render('index', {
indexCSS: true,
image
});
})
Within whatever route is calling this function. You just cannot assign any kind of return value from a callback to a variable (really not recommended at least) in the normal way.

Request JSON within a loop

I'm having trouble to surf correctly between objects that are been returned by an API request.
Basically, I have an Array (tickerArr with 25 elements that I get from another request) which I use for a forEach loop for another request.
Here's what the code looks like:
var request = require('request');
var async = require('async');
var fs = require('fs')
var getTickerList = require('./getTickerList').getTickerList;
var unixTimeStamp = new Date().toISOString();
async function getTickerInfo() {
var tickerArr = await getTickerList;
var arrLength = tickerArr.length;
var init = 0;
console.log(`${arrLength} to process to the batchfile...`);
var getTickerInfo = new Promise((resolve, reject) => {
async.forEach(tickerArr, (item, callback) => {
request({
url:`https://api.kraken.com/0/public/Ticker?pair=${item}`,
json: true
}, (error, response, body) => {
var tickerInfo = JSON.stringify(body.result)
fs.appendFile(`./modules/batches/${unixTimeStamp}batch.json`, tickerInfo, (err) => {
if (err) throw err;
init ++;
var progress = Math.round(init / arrLength * 100);
console.log(`[${progress}%] - ${item} successfully added!`);
});
resolve();
})
});
});
}
getTickerInfo()
Unfortunately, and even if the request works correctly, the objects being returned have a specific path:
Error: []
Result:
Pair(x):
a: [1, 2, 3]
b: [1, 2, 3] ect...
You can find an exact example of the information being returned from the request here:
{
"error": [],
"result": {
"XXBTZEUR": {
"a": ["2212.31000", "3", "3.000"],
"b": ["2212.21100", "1", "1.000"],
"c": ["2212.31000", "0.15800000"],
"v": ["4999.06419498", "9993.55448658"],
"p": ["2206.04624", "2181.36028"],
"t": [15065, 29524],
"l": ["2167.00000", "2122.92000"],
"h": ["2239.00000", "2239.00000"],
"o": "2184.99000"
}
}
}
The only problem is: I would like to 'transform' the object I get from the request, to another object (just arranging stuff around, most of all: put the 'Pair(x)' attribute as a value for a name key) but since I don't know the pair in advance (the values in my tickerArray are enough to make the request but do not apparently correspond to reference the object), I can't access the information contained within result to manipulate it.
Anybody have any idea?
Thanks in advance!
Your question has multiple levels and maybe it's best to break up the task into its constituent parts first.
Calling the Kraken REST API and unwrapping the response
Creating small, composable functions
Transforming an object into a new layout
Writing the results to a file
0. Preface
Since you want to work with await, the first thing to know is that this is syntactical sugar for "wait for promise resolution". async is the keyword that enables await semantics inside a function.
So given this promise-returning function:
function foo() {
return Promise.resolve('some value');
}
this:
async function worker() {
var fooResult = await foo();
console.log(fooResult);
}
worker();
and this:
foo().then(function worker(fooResult) {
console.log(fooResult);
});
are effectively the same thing. (As always, it's a little more complicated than that, but that's the gist of it.)
Since we are working with promises here, the most sensible thing to do is to use libraries that also use promises. request has the request-promise counter-part, so let's use that.
var request = require('request-promise');
1. Calling the Kraken REST API and unwrapping the response
Making an HTTP request and breaking up the response (err and result parts) works the same for all API endpoints, so we can write a function that handles this task.
function getFromApi(endpoint, params) {
return request({
url: "https://api.kraken.com/0/public/" + endpoint,
qs: params,
json: true
}).then(function (data) {
if (data.error && data.error.length) {
throw new Error(`API error xyz for ${endpoint} with params ...`);
}
return data.result;
});
}
Users of this function will not have to deal with err at all and have access to result directly.
2. Creating small, composable functions
Our getFromApi() function returns a promise, therefore we can re-use it in wrappers for various API endpoints:
function getTickerList() {
return getFromApi("TickerList"); // ...or something like that
}
function getSingleTicker(pair) {
return getFromApi("Ticker", {pair: pair}).then(transformTicker);
}
function getTickerInfo(tickerList) {
return Promise.all(tickerList.map(getSingleTicker));
}
Note that, like in getFromApi(), we can use Promise#then to modify the overall output of an operation.
getTickerInfo() accepts an array of ticker names. It uses Array#map to run all API requests in parallel, and Promise#all to allow awaiting the overall result. The fairly complex operation of fetching and transforming multiple results in parallel composes into a pretty straight-forward one-liner.
3. Transforming an object into a new layout
transformTicker() is meant to accept an object in form {"XXBTZEUR": {"a": [...], ... }} and return a transformed variant of it.
function transformTicker(ticker) {
var result = {};
// ...massage ticker into desired form
Object.keys(ticker).forEach( pair => {
Object.keys(ticker[pair]).forEach( k => {
result[k] = { /* ... */ };
});
});
return result;
}
4. Writing the results to a file
Appending to a JSON file does not work. JSON can only be read and written as a whole. Let's fetch a list of ticker names, the associated tickers and write the results to a file.
async function worker() {
var tickerList = await getTickerList();
var tickerInfo = await getTickerInfo(tickerList);
var filename = `./modules/batches/${new Date().toISOString()}batch.json`;
fs.writeFile(filename, JSON.stringify(tickerInfo), (err) => {
if (err) throw err;
console.log("Done");
});
}
worker();
You can switch use a promisified version (see Bluebird#promisifyAll) of the fs module, since the plain version breaks the nice async/await semantics again by requiring a continuation callback.
With a promisified fs module, the worker could look like this:
async function worker() {
var tickerList = await getTickerList();
var tickerInfo = await getTickerInfo(tickerList);
var filename = `./modules/batches/${new Date().toISOString()}batch.json`;
await fs.writeFileAsync(filename, JSON.stringify(tickerInfo));
console.log("Done");
}

Array.push is not working with promisified function but callback is not retaining the message

Note - Message variable is not retaining data after calling promisified functions. Callback is giving null array.
Code -
'use strict';
const Promise = require('bluebird');
let _connectResolve, _connectReject, onConnected = new Promise((resolve, reject) => {
_connectResolve = resolve;
_connectReject = reject;
}),
redis = require("redis"),
redisClient = redis.createClient({
host: 'localhost',
port: 6379
});
Promise.promisifyAll(redis.RedisClient.prototype);
redisClient.on('connect', _connectResolve);
const results = Promise.all([
'it/0I0g2I3D312s192u0U3k/10es.zip',
'items/25210B0c0Q1L3u0X462g/10ges.zip',
'items/2x0n440V1A1n3x1y0f1K/Fs.zip',
'items/2l023931u0w1S2a3j/es.zip',
'items/2O2x212i3t0B2h/es.zip',
]);
var message = [];
var a = Promise.promisify(function(callback) {
results.map(function(result) {
redisClient.getAsync(result).then(function(reply) {
if (reply == null) {
message.push({
"key": result,
"bucket_name": 'dsdds'
});
}
//console.log(message);
});
callback(null, message);
});
});
onConnected.then(() => {
Promise.resolve(a()).then(function(message) {
console.log(message);
});
});
Output - message is undefined
There are quite a few things wrong with how you've coded this. Asynchronous operations run on their own schedule and finish some indeterminate time in the future. As such, you can't do something like use a .map() loop with asynchronous operations in it and then expect the results to be ready right after the .map() loop. Instead, you have to use tools to keep track of when all the async operations in the .map() loop have completed and look at the result only when that tool tells you all the operations are done.
In addition, there are some very weird uses of Promise.promisify() which makes it look like you think promisifying a plain function will somehow magically manage the async operations inside it. It will not. You can only use Promise.promisify() on an async function that has a specific calling convention.
Fortunately, since you have the Bluebird promise library, you can use its tools to help you to do something like this:
function a() {
let message = [];
return Promise.map(results, function(result) {
return redisClient.getAsync(result).then(function(reply) {
if (reply == null) {
message.push({
"key": result,
"bucket_name": 'dsdds'
});
}
});
}).then(function() {
// make the message array be the resolved value of the returned promise
return message;
});
});
onConnected.then(() => {
a().then(function(message) {
console.log(message);
});
});

node.js callback function at the after loop has ended

I have an array of URLs and I want to loop through them and fetch thr content. After I have looped through them and fetched thr content I want a callback function to be called.
I know I can do this via async library but I want to do this without using any library.
Sample of what kind of code I want is below
['yahoo.com', 'gmail.com'].each(function(item){
//code to fetch URL content
},someCallbackFunctionToBeExecutedAtTheEndOfLoop);
This is typically the type of thing you do using promises (But you would need a library), with a code like:
var ops = [];
urls.forEach(function(url) {
ops.push(fetchUrl(url));
});
P.all(ops).then(callback);
function fetchUrl(url) {
var defer = P.defer();
//do stuff
// call defer.resolve(result);
return defer.promise;
}
If you don't want to use promises, you can use a counter of operations, like:
var ops = urls.length;
urls.forEach(function(url) {
// do stuff
ops--;
if (ops === 0) {
callback();
}
});
If you chose the promises, I advice to use p-promise module, which is far more optimized than Q.
If you want to do it without any sort of library like async, then you have to write your own counter to keep track of when all the async responses have been completed:
var request = require('request');
function loadAll(list, fn) {
var cnt = list.length;
var responses = [];
list.forEach(function(url, index) {
request(url, function(error, response, body) {
if (error) {
fn(error);
} else {
responses[index] = response;
--cnt;
if (cnt === 0) {
fn(0, responses);
}
}
});
})
}
loadAll(['http://www.yahoo.com', 'http://www.gmail.com'], function(err, results) {
if (!err) {
// process results array here
}
});
If you're going to be doing many async operations in node.js, then getting a promise library like Bluebird will save you a lot of time. For example, I think you could do the above in something like this (untested):
var Promise = require("bluebird");
var requestP = Promise.promisfy(require("request"));
Promise.map(['http://www.yahoo.com', 'http://www.gmail.com'], requestP).then(function(results) {
// process the array of results here
});

Writing async http returns in Nodejs

I realize this quesiton probably gets asked a million times but I'll try anyway. So I am used to Python and Ruby and 'blocking' imperative code blocks and objects and methods and trying to learn this newfangled Node thing for a project I joined. Try as I might, I can't quite wrap my head around async callbacks and passing return values.
Here's my code:
var getAnHttp = function getAnHttp (args, callback) {
var req = http.request(args, callback);
req.on('error', function(errmsg) {
console.log('problem with request\t' + errmsg.message);
});
req.end();
};
var getWebPageBody = function getWebPageBody (res) {
var pageRes = "";
res.setEncoding('utf8');
res.on('data', function(requestBody) {
pageRes = requestBody;
console.log('[ DEBUG ] BODY:\t' + pageRes);
});
res.on('end', function() {
return pageRes;
});
};
exports.captureLink = function captureLink(targetLink) {
getAnHttp(targetLink, getWebPageBody);
};
And actually calling it:
var crawl = require('../grab_site');
var aGreatSite = { hostname : 'hotwebcrawlingaction.xxx',
port : 80,
path : '/asyncpron',
method : 'GET'
};
var something = crawl.captureLink(aGreatSite);
getWebPageBody will print my output, but I can't get my returns to bubble up through the funcitons. I know I'm calling something in a sync fashion that and some in async but I can't quite sort this out. While I could certainly just put this in one big function I'm trying to do this right, not hack it out.
Anyway sorry for a noob question--I feel like I'm trying to be functional and OOP+imperative at the same time--i see plenty of other examples but I'm trying not to cargo cult or hail mary this one.
Async functions will never return something that has to fetched with async. getWebPageBody in your case returns undefined, and then later your callbacks happen.
In synchronous programming you return a value. But in async, you provide it as an argument to a callback function.
So instead, make your function accept a callback argument, and simply call it when you are done.
var getWebPageBody = function(res, callback) {
var pageRes = "";
res.setEncoding('utf8');
res.on('data', function(requestBody) {
pageRes = requestBody;
console.log('[ DEBUG ] BODY:\t' + pageRes);
});
res.on('end', function() {
callback(pageRes); // invoke callback
});
};
// and call it
getWebPageBody(res, function(pageRes) {
// pageRes is now the thing you expect, inside this function.
});
If you invoke a function which triggered async actions, return will never get any values out of that. It's all about callbacks. You fire your own callback, from the callback of something else.

Resources