async waterfall callback was already called error in nodejs - node.js

I am newbee to nodejs. I have been scratching my head for long time on why below code throwing error.I found some similar in stack overflow, but couldn't get help in finding the bug.
var albums_coll,photos_coll;
async.waterfall([
function(cb){
MongoClient.connect(url,
(err,dbase)=>{
if(err) {
console.log('bad');
process.exit(-1);
}
console.log("I have a connection!");
db=dbase;
cb(null);
}
);
},
function(cb){
db.collection("albums",cb);
},
function(album_obj,cb){
albums_coll = album_obj;
db.collection("photos",cb);
},
function(photos_obj,cb){
photos_coll = photos_obj;
cb(null);
},
function(cb){
albums_coll.InsertOne(a1,cb);
},
function(inserted_doc,cb){
console.log("I have Inserted a document!!");
console.log(inserted_doc);
cb(null);
}
],
function(err,results){
console.log("Done!!!");
console.log(err);
console.log(results);
db.close();
});
Please suggest!!
the following is the error its showing.
I have a connection!
C:\Users\thathine\NodeLive\Chapter08\mongotest\node_modules\mongodb\lib\mongo_client.js:433
throw err
^
Error: Callback was already called.
at C:\Users\thathine\NodeLive\Chapter08\mongotest\node_modules\async\dist\async.js:903:32
at Db.collection (C:\Users\thathine\NodeLive\Chapter08\mongotest\node_modules\mongodb\lib\db.js:466:27)
.
.
.

The callback() function you are passing into the db.collection() function is being called twice. The first time this happens is inside the MongoDB library while invoking db.collection(). It is happening in the following block - I added a comment pointing to line 466 as shown in the stack:
if(options == null || !options.strict) {
try {
var collection = new Collection(this, this.s.topology, this.s.databaseName, name, this.s.pkFactory, options);
if(callback) callback(null, collection);
return collection;
} catch(err) {
// if(err instanceof MongoError && callback) return callback(err);
if(callback) return callback(err); // <-- Line 466 - first time callback is called
throw err;
}
}
The second time it gets called, is by the async library when it needs to determine if it has to execute the next task in the array or invoke the final callback; by then, the callback is already null. The error you're seeing is being raised in the onlyOnce() function that is part of the async library:
function onlyOnce(fn) {
return function() {
if (fn === null) throw new Error("Callback was already called.");
var callFn = fn;
fn = null;
callFn.apply(this, arguments);
};
}
So the following code will raise the error:
async.waterfall([
function(callback) {
MongoClient.connect('mongodb://localhost:27017/test', (error, db) => {
if (error) {
console.error(error);
process.exit(-1);
}
callback(null, db);
});
},
function(db, callback) {
console.log('Querying albums collection...');
// This will cause the exception
db.collection('albums', callback);
},
function(db, albums, callback) {
console.log('Got albums...');
console.log('Querying photos collection...');
db.collection('photos', (error, photos) => {
if (error) {
return callback(error);
}
callback(null, db, albums, photos);
});
},
function(db, albums, photos, callback) {
console.log('Got photos...');
callback(null, 'DONE');
}
], function (error, results) {
console.error(error);
console.log(results);
process.exit(0);
});
While this version of the code will execute as expected:
async.waterfall([
function(callback) {
MongoClient.connect('mongodb://localhost:27017/test', (error, db) => {
if (error) {
console.error(error);
process.exit(-1);
}
callback(null, db);
});
},
function(db, callback) {
console.log('Querying albums collection...');
db.collection('albums', (error, albums) => {
if (error) {
return callback(error);
}
callback(null, db, albums);
});
},
function(db, albums, callback) {
console.log('Got albums...');
console.log('Querying photos collection...');
db.collection('photos', (error, photos) => {
if (error) {
return callback(error);
}
callback(null, db, albums, photos);
});
},
function(db, albums, photos, callback) {
console.log('Got photos...');
callback(null, 'DONE');
}
], function (error, results) {
console.error(error);
console.log(results);
process.exit(0);
});
/* Output:
Querying albums collection...
Got albums...
Querying photos collection...
Got photos...
null
DONE
*/
Hope this helps!

Related

Callback function is not working as expected in watefall on async.foreachlimit

The callback function is not working as expected in a waterfall on async.for each limit.
When I remove that rest api call and if I call callback(null,arg2) it is working as expected, I think the problem with callback function or some api call in the below code.
var async = require("async");
var users = [1, 2, 3, 45]; // Initialize user array or get it from DB
var e = [];
var unirest = require("unirest");
var data = function() {
return new Promise((resolve, reject) => {
async.forEachLimit(
users,
1,
function(user, userCallback) {
async.waterfall(
[
function(callback) {
console.log(user);
callback(null, user);
},
function(arg1, callback) {
console.log(arg1);
callback(null, arg1);
},
function(arg2, callback) {
unirest
.get("http://dummy.restapiexample.com/api/v1/employee/1")
.end(function(response) {
if (response.error) callback("null", "data");
else callback(null, arg2);
});
}
],
function(err, result) {
if (err) {
console.log("err");
reject(e);
} else {
console.log("done", result);
e.push(result);
userCallback();
resolve(e);
}
}
);
},
function(err, result) {
console.log("User For Loop Completed", err, result);
}
);
});
};
I'm getting output as
expected outputs as

i have come across a strange error in nodejs filesystem

i am working with the filesystem protocol of nodejs. i did writing the file, reading the file and now trying to rename it. it actually renames the file but throws me this error and my localhost stops running.
this is the error:
_http_outgoing.js:690
throw new ERR_INVALID_ARG_TYPE('chunk', ['string', 'Buffer'], chunk);
TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be one of type string or Buffer. Received type object
file1: var fs = require('fs');
// write
function write(fileName, content) {
return new Promise(function (resolve, reject) {
fs.writeFile(`./test/${fileName}`, content, function (err, done) {
if (err) {
reject(err);
return;
}
resolve(done);
// console.log('file created >>', done);
});
})
}
function readFile(fileName, cb) {
fs.readFile(`./test/${fileName}`, 'utf-8', function (err, done) {
if (err) {
cb(err);
return;
}
cb(null, done);
})
}
function rename(oldname, newname, cb){
fs.rename(`./bands/${oldname}`, `./bands/${newname}`, function(err,done){
if(err){
cb(err)
return
}
cb(null,done)
})
}
rename('pinkfloyd.txt', 'tool.txt', function(err, done){
if(err){
console.log('error in renaming')
}
else{
console.log('renamed>>')
}
})
readFile('kisor.txt', function(err,done){
if(err){
console.log("error in file reading >>",err);
}else{
console.log('success >>',done);
}
})
write('kisor.txt', 'i am javascript')
.then(function (data) {
console.log('write success ', data);
})
.catch(function (err) {
console.log('error in wirte >>', err);
})
// objec short hand
module.exports = {
write, readFile, rename
}
i have imported the exported stuff from file 1 here in this file:
//var { write,readFile, } = require('./file');// object destruct
var fileOp = require('./file1');
//console.log('file operation >>', fileOp);
fileOp.write('brodway.txt', 'i am infosys of nepal')
.then(function (data) {
console.log('done >>>', data);
})
.catch(function(err){
console.log('err',err);
})
fileOp.readFile('kisor.txt',function(err,done){
if(err){
console.log('err',err);
}else{
console.log('success >>',done);
}
and lastly, here is the server:
var http = require('http');
var fileOp = require('./file1');
var server = http.createServer(function (request, response) {
// this function will be executed whenever a client is connected
// request or 1st argument is http request object
// response or 2nd argument is http response object
var url = request.url;
if (url == '/write') {
fileOp.write('abcd.js', 'hi')
.then(function (data) {
response.end('data', data);
})
.catch(function (err) {
response.end(err);
})
} else if (url == '/read') {
fileOp.readFile('abcd.js', function (err, done) {
if (err) {
response.end(err);
} else {
response.end('done' + done);
}
})
} else if(url == '/rename'){
fileOp.rename('pinkfloyd.txt', 'tool.txt', function(err, done){
if(err){
response.end(err)
}
else{
response.end('done', done)
}
})
} else {
response.end('form default page');
}
console.log('client connected to server');
console.log('request url >>', request.url);
// request response cycle must be completed
// response.end('hi from node server'); response cannot sent more than once
});
server.listen(8080, function (err, done) {
if (err) {
console.log('server listening failed');
}
else {
console.log('server listening at port 8080');
console.log('press CTRL + C to exit from server');
}
});
In this section of code:
fileOp.rename('pinkfloyd.txt', 'tool.txt', function(err, done){
if(err){
response.end(err)
}
else{
response.end('done', done)
}
You are calling response.end('done', done). But, the fs.rename() callback does not have a done argument - it only has the err argument because there's no data to communicate back - the rename either succeeded or it didn't . So, done will be undefined. So you're calling:
response.end('done', undefined);
The function signature for response.end() is this:
response.end([data[, encoding]][, callback])
So, you're trying to send undefined for the encoding. That is not correct.
Your rename function should be change from this:
function rename(oldname, newname, cb){
fs.rename(`./bands/${oldname}`, `./bands/${newname}`, function(err,done){
if(err){
cb(err)
return
}
cb(null,done)
})
}
to this:
function rename(oldname, newname, cb){
fs.rename(`./bands/${oldname}`, `./bands/${newname}`, function(err){
if(err){
cb(err);
return
}
cb(null); // no 2nd argument here
})
}
or even simpler:
function rename(oldname, newname, cb){
fs.rename(`./bands/${oldname}`, `./bands/${newname}`, cb);
}
FYI, it appears you're using .end() improperly a bunch of places in your code where you do this:
response.end('done', someData);
That's not how it works. You aren't emitting an event. You're commanding the end of the http response and you need to follow this function signature:
response.end([data[, encoding]][, callback])
So, you'd probably just do response.end(data), if data was a string.

Hard time uderstaning async.parallel in node

I have this function in the controller
router.post('/', function(req, res, next) {
if (req.user.isPremium == false) {
// Free user - Single report
let website = req.body.website0;
let builtWithCall = `https://api.builtwith.com/free1/api.json?KEY=APIKEY&LOOKUP=${website}`;
let pagespeedCall = `https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://${website}&strategy=mobile&key=APIKEY`;
// curl 'https://www.googleapis.com/pagespeedonline/v4/runPagespeed?url=https://georgiancollege.ca&strategy=mobile&key=APIKEY'
var calls = [];
calls.push(function(callback) {
// First call
https.get(builtWithCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log('BuiltWith received', data);
});
});
});
calls.push(function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
});
} else {
// Premium user - comparison report
let websites = [];
}
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
});
I am trying to run several async API calls at once. The problem is, when I try to run them like this
async.parallel(calls, function(err, results) {
if(err){
console.log(err);
}
console.log('async callback ', results);
res.render('/generated-report', {
title: 'Report',
data: {},
});
});
the console doesn't log anything.
When I do the console log here though
function(callback) {
// second call
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
// console.log(data);
});
});
}
it logs the response. The pageSpeed one gets in a weird loop and repeats itself multiple times, but at least it works.
Now what am I doing wrong with the async.parallel part? Also what is this callback in function(callback) {?
EDIT:
This is the new version of the anonymous function:
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end', function(data) {
callback(null, data);
});
resource.on('error', function(err) {
callback(err);
});
});
}
You need to call the passed in callback. Looking at your one parallel function you are not calling callback(). I'll assume your resource object has an end & error
function(callback) {
// second call
var results;
https.get(pagespeedCall, function(resource) {
resource.setEncoding('utf8');
resource.on('data', function(data) {
results += data;
// console.log(data);
});
resource.on('end' function() {
callback(null, results);
});
resource.on('error' function(err) {
callback(err);
});
});
}
How async.parallel works is all the functions called must in turn call the passed in callback function; in your case that is callback.
Once each function in the parallel calls callback then and only then will the final function be called, which is the function you defined as function(err, results) {...}.
There is one caveat, if in the callback call you pass non-null for the first argument then that final function will be called immediately where you should handle that error if it happens.

async.parallel inside async.series

I have this async.parallel functionality inside an aysnc.eachSeries call.
I hardcoded an error so I could pass it, to see if it was behaving the way I thought. For some reason, when I pass an error, it doesn't get thrown in the final callback named "doneWithSeries".
async.eachSeries(jsonDataArr, function iterator(item, callback) {
async.parallel([
function (cb) {
if (item.hasOwnProperty('event.type')) {
var event_type = item['event.type'];
delete item['event.type'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + event_type + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
},
function (cb) {
if (item.hasOwnProperty('status_desc')) {
var status_desc = item['status_desc'];
delete item['status_desc'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + status_desc + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
}
],
function doneWithParallel(err) {
callback(new Error('throw this baby')); //shouldn't the first incident of error pass the error straight to the doneWithSeries callback below?
})
},
function doneWithSeries(err) {
if (err) {
throw err;
}
else {
console.log('success');
}
});
here is a distilled version of the code without anything unnecessary:
var async = require('async');
async.eachSeries(['1', '2'], function (item, callback) {
async.parallel([
function (cb) {
setTimeout(function () {
cb(null, 'one');
}, 200);
},
function (cb) {
setTimeout(function () {
cb(null, 'two');
}, 100);
}
],
function doneWithParallel(err, results) {
console.log('results', results);
callback(new Error('duh'));
})
},
function doneWithSeries(err) {
if (err)
throw err;
});
indeed that works. can't figure out why my code above doesn't, accept perhaps that the array could be empty even though when I run my code the success message gets logged...weird.
I think that's expected behavior if your list is empty. async will always call the final callback with no error even if there is no input list.

async forEach loops making api calls

I have a function that makes an api call and a second function that loops through the data of the first function and makes an api call each iteration. I'm trying to use the async library to make this happen but the 2nd function is still running asynchronously instead of waiting to finish. So I end up running function 1 runs, function 2 starts, but final callback runs before function 2 finishes.
async.series([
function (callback) {
//api call
getShelves.execute(function (err, shelves) {
if (err) { return callback(err); }
async.forEach(shelves.items, function (shelf, callback) {
var shelfObj = {id: shelf.id, title: shelf.title, books: []};
bookShelves.push(shelfObj);
callback();
});
//sort numerically to make placing books easier
bookShelves.sort(function (a, b) {return a.id - b.id; });
callback();
});
},
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
callback();
});
}
callback();
});
});
callback();
}
], function (err) {
if (err) { console.log('error'); }
res.render('collection', { shelves: bookShelves });
});
});
EDIT: Working now thanks guys
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
getBooks.execute(function (err, books) {
if (err) { return callback(err); }
if (books.items) {
async.forEach(books.items, function (book, callback) {
var bookObj = {title: book.volumeInfo.title};
bookShelves[shelf.id].books.push(bookObj);
console.log(book.volumeInfo.title);
//callback to continue book loop
callback();
}, function () {
//callback to continue shelf loop
callback();
});
}else{
callback();
}
});
}, function () {
//callback to end function and move to next. However this is never reached
callback();
});
}
The second function in your series calls its callback immidiately, not waiting until async.forEach iteration finishes. Instead, try this to call it afterwards:
function (callback) {
async.forEach(bookShelves, function (shelf, callback) {
//api call
//... skipped ...
}, function() {
callback();
});
}
function loadShelf(shelf, callback) {
//_.pick is handy for this FYI
var shelfObj = {id: shelf.id, title: shelf.title};
//presumably your getBooks call takes a shelf id to relate the
//getBooks is an asynchronous DB or API call presumably
getBooks(shelf.id, function (error, books) {
if (error) {
callback(error);
return;
}
//This is an in-memory array. No async needed.
shelfObj.books = books.map(function (book) {
return {title: book.volumeInfo.title};
});
callback(null, shelfObj);
});
}
getShelves.execute(function (error, dbShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
async.each(dbShelves, loadShelf, function (error, fullShelves) {
if (error) {
res.render('error', error); //pseudo-code error handling
return;
}
//sort numerically to make placing books easier
var sortedShelves = fullShelves.sort(function (a, b) {return a.id - b.id; });
res.render('collection', { shelves: sortedShelves });
});

Resources