I realize this quesiton probably gets asked a million times but I'll try anyway. So I am used to Python and Ruby and 'blocking' imperative code blocks and objects and methods and trying to learn this newfangled Node thing for a project I joined. Try as I might, I can't quite wrap my head around async callbacks and passing return values.
Here's my code:
var getAnHttp = function getAnHttp (args, callback) {
var req = http.request(args, callback);
req.on('error', function(errmsg) {
console.log('problem with request\t' + errmsg.message);
});
req.end();
};
var getWebPageBody = function getWebPageBody (res) {
var pageRes = "";
res.setEncoding('utf8');
res.on('data', function(requestBody) {
pageRes = requestBody;
console.log('[ DEBUG ] BODY:\t' + pageRes);
});
res.on('end', function() {
return pageRes;
});
};
exports.captureLink = function captureLink(targetLink) {
getAnHttp(targetLink, getWebPageBody);
};
And actually calling it:
var crawl = require('../grab_site');
var aGreatSite = { hostname : 'hotwebcrawlingaction.xxx',
port : 80,
path : '/asyncpron',
method : 'GET'
};
var something = crawl.captureLink(aGreatSite);
getWebPageBody will print my output, but I can't get my returns to bubble up through the funcitons. I know I'm calling something in a sync fashion that and some in async but I can't quite sort this out. While I could certainly just put this in one big function I'm trying to do this right, not hack it out.
Anyway sorry for a noob question--I feel like I'm trying to be functional and OOP+imperative at the same time--i see plenty of other examples but I'm trying not to cargo cult or hail mary this one.
Async functions will never return something that has to fetched with async. getWebPageBody in your case returns undefined, and then later your callbacks happen.
In synchronous programming you return a value. But in async, you provide it as an argument to a callback function.
So instead, make your function accept a callback argument, and simply call it when you are done.
var getWebPageBody = function(res, callback) {
var pageRes = "";
res.setEncoding('utf8');
res.on('data', function(requestBody) {
pageRes = requestBody;
console.log('[ DEBUG ] BODY:\t' + pageRes);
});
res.on('end', function() {
callback(pageRes); // invoke callback
});
};
// and call it
getWebPageBody(res, function(pageRes) {
// pageRes is now the thing you expect, inside this function.
});
If you invoke a function which triggered async actions, return will never get any values out of that. It's all about callbacks. You fire your own callback, from the callback of something else.
Related
Ive spent a bit of time trying to understand this. I hope the answer is obvious and just show my lack of experience
My goal is to send API request to steam for various IDs of game mods and find the time_updated for each one, to put these all into an array and then find out which one most the most recently updated
I have got the code below, but its not quite doing what I want, I think I am just getting muddled over timings
My plan was to have a few different values in arrMODID = [], and to loop through each one, get the time_updated, push that to an array and for const result = await myfunction(); to be able to access the data in the modinfoArray
however that is returning an array with just [{test},{}] in it and is being fired before the function has put any data into the array
can anyone give me a shove in the right direction please
thank you
import request from 'request';
const myfunction = async function(x, y) {
var arrMODID = ["2016338122"];
var modinfoArray = []
var timeUpdated
for (const element of arrMODID) {
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: 'http://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1',
body: 'itemcount=1&publishedfileids[0]=2016338122',
},
function(error, response, body){
var response = JSON.parse(body);
var myResponse = response.response.publishedfiledetails
myResponse.forEach(function(arrayItem) {
//console.log(arrayItem.time_updated)
timeUpdated = arrayItem.time_updated
//console.log(timeUpdated)
modinfoArray.push({"2016338122":arrayItem.time_updated})
console.log(modinfoArray) // only this log returns the added items
})
});
}
return ["test", modinfoArray];
};
// Start function
const start = async function(a, b) {
const result = await myfunction();
console.log(result); // this returns the empty array
}
// Call start
start();
You need to use an http request library that supports promises so you can await that inside your function. You cannot successfully mix promises and asynchronous operations like request.post() that uses plain callbacks because you can manage the control flow in a promise-like way with plain callbacks.
I'd suggest using the got() library. Also, the request() library has been deprecated and is not recommended for new code. If you absolutely wanted to stay with the request() library, you could use the request-promise module instead, but keep in mind that the request() library is in maintenance mode only (no new feature development) whereas this list of alternatives are all being actively developed.
Here's a runnable implementation using the got() library:
import got from 'got';
const myfunction = async function() {
const arrMODID = ["2016338122"];
const modinfoArray = [];
for (const element of arrMODID) {
const response = await got.post({
headers: { 'content-type': 'application/x-www-form-urlencoded' },
url: 'http://api.steampowered.com/ISteamRemoteStorage/GetPublishedFileDetails/v1',
body: 'itemcount=1&publishedfileids[0]=2016338122',
}).json();
const myResponse = response.response.publishedfiledetails;
for (const arrayItem of myResponse) {
modinfoArray.push({ "2016338122": arrayItem.time_updated });
}
}
return ["test", modinfoArray];
};
// Start function
const start = async function() {
const result = await myfunction();
console.log(result);
return result;
}
// Call start
start().then(result => {
console.log("done");
}).catch(err => {
console.log(err);
});
It's a silly question perhaps, but I spent quite sometime trying to resolve but not able to get the data.
In the below function checkMailCount, I have an object called test with a function in it to assign the value to mails. The value assigned from res.on('end') to this test.mails via mailCount is not retained/persisting.
'use strict';
const https = require('https');
exports.handler = (event, context, callback) => {
let session = event.session;
let request = event.request;
if (request.type === "something") {
if (request.name === "blah blah") {
let emailCount = checkMailCount(session, callback);
context.succeed(callback(null, emailCount));
}
}
function checkMailCount(session, callback) {
let url = 'https://www.googleapis.com/gmail/v1/users/me/messages?
access_token = $ {
session.user.accesstoken
} & q = "is:unread"
';
let test = {
mails: "",
mailCount: function(val) {
this.mails = val;
}
};
let data = "";
https.get(url, function(res) {
res.on('data', function(chunk) {
data += chunk;
//console.log(data);
});
res.on('end', function() {
let result = JSON.parse(data);
//result2+= result.replace('/\//g','');
test.mailCount(result["resultSizeEstimate"]);
// result["resultSizeEstimate"] inaccessible outside
});
});
return test.mails; // This is returning undefined
}
//context.fail();
}
Below is the response JSON from gmail API
{
"messages": [
{
"id": "165f1627a53f70c6",
"threadId": "165f000031cee15b"
},
{
"id": "165f00d9e2e07737",
"threadId": "165f00d9e2e07237"
}
],
"nextPageToken": "10349401829330188334",
"resultSizeEstimate": 2
}
I need to return the value of result["resultSizeEstimate"] from the function checkMailCount. The value of result, result["resultSizeEstimate"] is available inside res.on('end', f(){It is available here}) but not outside.
I tried this in VS code not as lambda, I was able to do it by other means. Its sort of hard to unit test lambda functions. Is there a way, I can send the email count to context.succeed(callback(null, emailCount)) ?
It seems like there's a fundamental issue with callbacks in this code. It is not possible for checkMailCount to behave like you want it to in its current form. checkMailCount is asynchronous. JavaScript has a few different ways to deal with asynchronous code. The oldest is what you're doing here, callbacks. Next is promises and finally async/await. This code is probably easier to write if you switch to async/await. That said, if you want to keep using the callback pattern you're gonna have to ya know, use them.
Instead of passing lambda's callback to checkMailCount you should pass your own.
checkMailCount(session, function(err, result) {
// result === result.resultSizeEstimate
// Call your lambda callback from within this function
});
res.on('end', function() {
let result = JSON.parse(data);
test.mailCount(result["resultSizeEstimate"]);
callback(null, result.resultSizeEstimate)
});
Note - Message variable is not retaining data after calling promisified functions. Callback is giving null array.
Code -
'use strict';
const Promise = require('bluebird');
let _connectResolve, _connectReject, onConnected = new Promise((resolve, reject) => {
_connectResolve = resolve;
_connectReject = reject;
}),
redis = require("redis"),
redisClient = redis.createClient({
host: 'localhost',
port: 6379
});
Promise.promisifyAll(redis.RedisClient.prototype);
redisClient.on('connect', _connectResolve);
const results = Promise.all([
'it/0I0g2I3D312s192u0U3k/10es.zip',
'items/25210B0c0Q1L3u0X462g/10ges.zip',
'items/2x0n440V1A1n3x1y0f1K/Fs.zip',
'items/2l023931u0w1S2a3j/es.zip',
'items/2O2x212i3t0B2h/es.zip',
]);
var message = [];
var a = Promise.promisify(function(callback) {
results.map(function(result) {
redisClient.getAsync(result).then(function(reply) {
if (reply == null) {
message.push({
"key": result,
"bucket_name": 'dsdds'
});
}
//console.log(message);
});
callback(null, message);
});
});
onConnected.then(() => {
Promise.resolve(a()).then(function(message) {
console.log(message);
});
});
Output - message is undefined
There are quite a few things wrong with how you've coded this. Asynchronous operations run on their own schedule and finish some indeterminate time in the future. As such, you can't do something like use a .map() loop with asynchronous operations in it and then expect the results to be ready right after the .map() loop. Instead, you have to use tools to keep track of when all the async operations in the .map() loop have completed and look at the result only when that tool tells you all the operations are done.
In addition, there are some very weird uses of Promise.promisify() which makes it look like you think promisifying a plain function will somehow magically manage the async operations inside it. It will not. You can only use Promise.promisify() on an async function that has a specific calling convention.
Fortunately, since you have the Bluebird promise library, you can use its tools to help you to do something like this:
function a() {
let message = [];
return Promise.map(results, function(result) {
return redisClient.getAsync(result).then(function(reply) {
if (reply == null) {
message.push({
"key": result,
"bucket_name": 'dsdds'
});
}
});
}).then(function() {
// make the message array be the resolved value of the returned promise
return message;
});
});
onConnected.then(() => {
a().then(function(message) {
console.log(message);
});
});
I went through many examples of nodejs with callbacks, but didn't understand how exactly they work. I know that they're executed after the function, of which they're a part of is done, but I didn't understand callbacks as a function. Let me give an example:
function processData (callback) {
fetchData(function (err, data) {
if (err) {
console.log("An error has occured. Abort everything!");
callback(err);
}
data += 1;
callback(data);
});
}
Here, what how do callback(err) and callback(data) execute, as in, is it an alias for some other function to which we're passing the parameters - err/ data? If so, which function does it call when we write callback(parameter)?
Here's another example :
var express = require("express");
var bodyParser = require("body-parser");
var multer = require('multer');
var app = express();
app.use(bodyParser.json());
var storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, './uploads');
},
filename: function (req, file, callback) {
callback(null, file.fieldname + '-' + Date.now());
}
});
var upload = multer({ storage : storage }).array('userPhoto',2);
app.get('/',function(req,res){
res.sendFile(__dirname + "/index.html");
});
app.post('/api/photo',function(req,res){
upload(req,res,function(err) {
//console.log(req.body);
//console.log(req.files);
if(err) {
return res.end("Error uploading file.");
}
res.end("File is uploaded");
});
});
app.listen(3000,function(){
console.log("Working on port 3000");
});
Again I sort of understood the other type of callback - suppose, here fs.readdir
fs.readdir(path, function (err, files) {
if (err) console.error(err);
for (var i = 0; i<files.length; i++) console.log(files[i];
}
console.log("done");
I know how this executes, most probably it'll print done first, and then prints the list of files, as soon as readdir is executed with files having the list of files.
But I didn't exactly understand the first and second code snippet. Can someone please explain that in simple terms, specific to the multer code snippet?
Callbacks and asynchronous behaviour are two different but related things. They're relate in the sense that asynchronous code can use callback to execute code in the intended sequence. But callbacks by themselves are just the result of the language treating functions as objects.
In your examples, you can see two sides of the callback story. One is the implementation and the other is the usage. The first code is an example of how to implement a function that accepts a callback. The last is an example of how to use a function that accepts a callback.
For simplicity let's ignore asynchronous behaviour for now and implement synchronous callbacks.
The following is an example of a function that loops through an array to construct a string:
function joiner (arr, callback) {
var return_value = "";
for (var i=0; i<arr.length; i++) {
return_value += callback(arr[i]); // use callback on each item
}
return return_value;
}
Once we've implemented the function above we can use it:
var result = joiner(['this','is','cool'],function(x) {
return "--> " + x + "!";
});
So, how do we know what arguments the callback accept? It accepts what the caller passes to it.
How do we know what arguments to pass to a callback function? We pass what the function is defined to accept?
If that sounds circular, it is. So how we really know what arguments to pass to a callback is:
If it's an API we created, we can do anything we want
If it's an API someone else created, read the documentation (or if your IDE can do it, check out the intellisense)
Callbacks gone meta
The above explanation is of course only the simplest example of callbacks: functions that accept a callback. The express example (and indeed your first example) shows how you can get even more complicated. You can write functions that accept a callback that accept a callback.
A simplified example of functions that accept a callback that accept a callback is something like this:
function processData (data) {
return "<" + data + ">";
}
function render (callback) {
return callback(processData);
}
So the render function above accepts a callback that will be passed a callback that returns a string. So if we now do:
var result = render(function(callback){
return "--" + callback('hello');
});
we will get:
--<hello>
As you can see, the callback we passed to render does not know how to process the data but render does (it calls processData). And render does not know what the data is but the callback we passed to it does ('hello'). Being able to pass a callback to a callback allows two pieces of code to cooperate without exposing each other's implementation details.
Async code uses this technique to order the sequence of code being executed. Because async code cannot return a value you can only pass it a callback so that once it gets its value your code (callback) can continue.
If you think about it, the Express API could have easily been implemented so that request handler callbacks must return the HTML page instead of calling res.send(). Indeed, this is how frameworks work in many languages. But that means that the request handler cannot execute asynchronous code to fetch data. Because Express doesn't want to limit you to using only synchronous code it passes an object with callback methods instead (res). This "design pattern" is called a monad. The one you see in Express is a continuation monad while the one you see in fs.readdir() is an I/O monad.
First, you got confused because you're not following the Node JS standards for passing parameters to asynchronous function. Every async callback must take 2 parameters. First should always be error and 2nd should always be data. Refer to how the callbacks for readdir and fetchData are defined.
So your code should ideally look like
function processData (callback) {
fetchData(function (err, data) {
if (err) {
console.log("An error has occured. Abort everything!");
callback(err, null); // here data will be null
}
data += 1;
callback(null, data); // here err will be null.
});
}
And you will define your callback function for processData at the time you call it like
processData(function(err, data){
err ? console.error(err) : console.log(data);
});
This way, in your callback, you know for sure whether the function has succeeded or failed.
I have written the following test case in Mocha, where the my code uses Q module.
var expect = require("chai").expect;
var utils = require("../Utils.js");
var utils1 = require("../a.js");
var sinon = require('sinon');
var request = require('requestretry');
var querySys = require('../b.js');
var Q = require("q");
describe("Sample", function () {
var results;
describe("#get()", function () {
before(function (done) {
done();
});
it("equal", function () {
var deferred = Q.defer();
var responseData = {};
responseData.code = 200;
responseData.data = [{a:1,b:2}];
deferred.resolve(responseData);
//querySys1 method uses Q promises. That is how I stubbed the response.
sinon.stub(querySys, 'querySys1').returns(deferred.promise);
//get function internally calls querySys1. Hence I have stubbed that.
results = utils1.get(specification);
results.then(function (data) {
//Here I do see data coming as {in:1, out:1}. But still the test case is passing when compare it with {}.
console.log(data);
//Ideally, it should have failed. But it is passing.
expect(data).to.be.equal({});
});
});
after(function (done) {
done();
})
});
});
So, if you see, I am trying to do assertion check in results.then part. I have printed the response, which I am receiving. That is coming as expected. But I am intentionally trying to match with wrong value, but test case is still passing.
Since your test does not include a callback, execution runs through the main block and declares the test as passing without waiting for the result of the then function. You need to let mocha know to wait for the callback:
it("equal", function (done) {
...
results.then(function (data) {
console.log(data);
expect(data).to.be.equal({});
return done();
});
});
When I changed results.then to results.done, test started failing as expected. Can someone say whether this is the right approach.