I have the following piece of code, but due to it's async behavior (I suppose), the callback is called twice. I'm using node-unzip to unzip a file I download from the internet.
function DownloadAndExtract(file, callback) {
log.debug("Starting download of "+file);
fse.ensureDirSync(tempPath);
var extractor = unzip.Extract({path: tempPath});
extractor.on("close", function() {
log.debug("Done downloading "+file);
return callback(null, file);
});
extractor.on("error", function (err) {
log.error("Extracting download "+file+": "+JSON.stringify(err, null, "\t"));
return callback(err, null); // THIS IS LINE 274
});
var url = "";
if(file == "WONEN") {
url = "https://example.com/file1.zip";
}else if(file == "BOG") {
url = "https://example.com/file2.zip";
}
if(url != "") {
request
.get(url)
.on("error", function (err) {
return callback(err, null);
})
.pipe(extractor);
}else{
return callback(new Error("Invalid file indicator: '"+file+"'"), null);
}
}
I expected return to actually quit all running async functions but that is obviously nonsense. Now, the error I keep getting is the following:
/home/nodeusr/huizenier.nl/node_modules/async/lib/async.js:30
if (called) throw new Error("Callback was already called.");
^
Error: Callback was already called.
at /home/nodeusr/huizenier.nl/node_modules/async/lib/async.js:30:31
at /home/nodeusr/huizenier.nl/node_modules/async/lib/async.js:251:21
at /home/nodeusr/huizenier.nl/node_modules/async/lib/async.js:575:34
at Extract.<anonymous> (/home/nodeusr/huizenier.nl/realworks.js:274:10)
at Extract.emit (events.js:129:20)
at Parse.<anonymous> (/home/nodeusr/huizenier.nl/node_modules/unzip/lib/extract.js:24:10)
at Parse.emit (events.js:107:17)
at /home/nodeusr/huizenier.nl/node_modules/unzip/lib/parse.js:60:12
at processImmediate [as _immediateCallback] (timers.js:358:17)
The output of the log.error() call is the following:
21-02-2015 03:00:05 - [ERROR] Extracting download WONEN: {} so I'm quite confused. There isn't really an error, then why is the event emitted?
How would I prevent the callback from being called twice here? Contacting the creator of the package or create a work around?
Code calling DownloadAndExtract
async.parallel([
function (callback) {
DownloadAndExtract("WONEN", callback);
},
function (callback) {
DownloadAndExtract("BOG", callback);
}
],
function (err, done) {
if(err) return log.error("Update entries: "+JSON.stringify(err, null, "\t"));
// Do different logic if no error
});
Edit
One of my attempts is declaring a var callbackAlreadyCalled = false within the function, and at any given point where I call the callback, I do
if(!callbackAlreadyCalled) {
callbackAlreadyCalled = true;
return callback(callback params);
}
Is this a good approach or could I handle it in a better way?
Edit 2
Already found out that the empty error is caused by errors not working properly when using JSON.stringify(), however, problem doesn't change.
It looks like you have waterfall error calls.
When you call the request, it calls the extractor then if there is an error it calls the on_error of the extractor and then the on_error of the request.
Try to unify your returns from errors or call it only once. Make a test removing one of your "on('error'" validation.
Related
I'm new to Node.js, Javascript and callbacks. I have a pretty simple program I'm trying to write and I can't get the callbacks to work.
Here's the relevant code:
var keysfetched = false;
var urlsfetched = false;
function setKeysfetched(){
keysfetched = true;
}
function setUrlsfetched(){
urlsfetched = true;
}
//get list of all media in bucket
getKeys(setKeysfetched);
//get a list of all media urls in DB
getUrls(setUrlsfetched);
//check for media in the bucket which is not used
checkKeys();
function getKeys(callback) {
S3.listObjectsV2(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else{
var contents = data.Contents;
contents.forEach(function (content) {
allKeys.push(content.Key);
});
if (data.IsTruncated) {
params.ContinuationToken = data.NextContinuationToken;
//console.log("get further list...");
getKeys();
}
else{
console.log("end of loop...");
}
}
});
callback()
}
When I run this I get an error : Error [TypeError]: callback is not a function
If I comment out all of the code inside getKeys(), I don't get the error.
This runs just fine:
function getKeys(callback) {
//Hard work here
callback()
}
What am I doing wrong?
You are passing callback and calling it at the bottom of getKeys but inside you are not passing any callback
if (data.IsTruncated) {
params.ContinuationToken = data.NextContinuationToken;
//console.log("get further list...");
getKeys();
}
So it's trying to call undefined which is not a function.
I have the following code:
function get_status(){
try {
/* GET - status */
async.map(['http://url1.com/', 'http://url2.com/', 'http://url3.com/'], function(value, callback) {
/* GET - request status */
request.post({
url: value,
form: { 'mycustomdata': ""+mycustomdata+"" },
method: 'POST'}, function(err, res, body) {
/* CHECK - response */
if (!err && typeof body !== 'undefined' && res.statusCode == 200) {
console.log('get status success...')
callback();
} else {
callback('failed to get status');
}
})
}, function(err, results) {
if (err) {
console.log(err); return false;
} else {
console.log('finished...') }
})
}
I need when all three urls in async.map processing is finished(some url respond some not) to call where it outputs 'finished...' and from this output to callthis function again...so that function runs every time all url is processed.
So in short..i need when async.map process all urls to output to console 'finished...' and run same function again.
But i m getting only from console...
get status success...
failed to get status..
get status sucess..
i never get called
'finished'...
so that i can cal function itsef...could you please light me how it needs to be written?
You're only ever passing in the err param to the final callback. I believe, from my last use of async.js, you need to pass in null as the first param to continue to the next function, here being the final function. Otherwise is skips to the final function as an error.
callback(null, results);
If you want it to call itself again, that's just simple recursion. Wrap the async.map in a function, and when console.log('finished...') runs, call the function again.
Ok i made full example so that you can see what is problem and you can try to change code:
var async = require('async'),
request = require('request');
var myUrls = [ 'http://56.123.65.86:8080/api/server_status/',
'http://88.96.42.122:8080/api/server_status/',
'http://12.23.32.25:8080/api/server_status/',
'http://251.214.44.58:8080/api/server_status/',
'http://122.23.32.54:8080/api/server_status/' ]
async.map(myUrls, function(url, callback) {
console.log('getting url...'+url)
request(url, function(error, response, html) {
// Some processing is happening here before the callback is invoked
if (typeof response !== undefined) {
console.log('response ok...'+url)
return callback(null, html)
} else {
console.log('response failed...'+url)
return callback(error, html);
}
});
}, function(err, results) {
if (results) {
console.log('all finished...')
}
});
I im getting this:
getting url...http://56.123.65.86:8080/api/server_status/
getting url...http://88.96.42.122:8080/api/server_status/
getting url...http://12.23.32.25:8080/api/server_status/
getting url...http://251.214.44.58:8080/api/server_status/
getting url...http://122.23.32.54:8080/api/server_status/
And need to get this:
getting url...http://56.123.65.86:8080/api/server_status/
getting url...http://88.96.42.122:8080/api/server_status/
getting url...http://12.23.32.25:8080/api/server_status/
getting url...http://251.214.44.58:8080/api/server_status/
getting url...http://122.23.32.54:8080/api/server_status/
all finished...
I cannot get all finished...so it stays with getting url and that stays forever...so i cannot get idea why is all finished... not triggeret after all items are processed...
Ok, lets say I have two Models. Contract and CommLog. Both work find independently but I need many CommLog to relate to each Contract.
In the ContractSchema trying async
ContractSchema.methods.getCommLog = function getCommLog(){
var log = false;
async.parallel([
function(){
CommLog.find({commType:'contract',parent:this._id},function(err,comms){
log = comms;
});
}],
function(){return log;});
};
Where I am trying to use it
router.get('/:code', function(req, res, next) {
Contract.findOne({accessCode:req.params.code},function(err,contract){
if(err)
res.send(err);
var data;
if(contract != null){
var comms = contract.getCommLog();
data = {error:false,data:contract,commlog:comms}
}else{
data = {error:true,message:"No Contract"}
}
res.json(data);
});
});
Where it shows var comms = contract.getCommLog(); It is never returning anything because the getCommLog() is not executing async...
I think its my misunderstanding of mongoose querying, so if you understand what I am trying to accomplish, please let me know what I am doing wrong. I have tried without async which would always return false.
The find call can return all matching results with one query, so I don't think you need async here. The reason it is not populating correctly when you call res.json(data) is because you are not waiting for the method call to finish before you fire off your server response. You would be better off nesting an additional CommLogs.find call within the Contract.find call, and only sending your response once that finishes.
//pseudo code:
Contract.find({}, function(err, contract) {
if(err || !contract) {
//return error response
}
else {
CommLogs.find({contract: contract._id}, function(err, commlogs) {
if(err || !commlogs) {
//return error response 2
}
else {
res.json({errors: false, contract: contract, commlogs: commlogs});
}
});
}
}
I have a small data gathering web app running with NodeJS and Couchbase. The requirement is, that when a 3rd party pushes some data to us and we are able to process it, we return the 200 header, but if there are any problems with storing that data, we return 500. This means that they can re-try with the failed data batch.
I'm having an issue where the 200 is always returned (because the DB calls are completed asynchronously). Here's an example:
...
var app = express();
function create(req, res) {
var error = false;
// Parse all the entries in request
for (var i = 0; i < req.body.length; i++) {
var event = req.body[i];
if (!event.email) {
// log error to file
error = true;
res.send("Event object does not have an email address!", 500);
}
// Greate the id index value
var event_id = 'blah';
// See if record already exists
db.get(event_id, function (err, result) {
var doc = result.value;
if (doc === undefined) {
// Add a new record
db.add(event_id, event, function (err, result) {
if (err) {
error = true;
res.send('There were processing errors', 500);
}
});
}
});
}
if (error)
res.send("Try again", 500);
else
res.send("OK", 200);
}
app.post('/create', create);
Is there a way of making the app wait for those DB calls to complete, i.e. for this funciton to be synchronous? Or am I using a wrong tech for this? :(
I decided to go with NodeJS+Couchbase because we are likely to have a very high amount of calls, where the data (small JSON objects) must be written, read and deleted. EDIT: Ah the data structure is likely to change for various events, so being able to store non-uniformly shaped documents its of a great advantage!
This is a typical use case for the async library, which is a utility-belt library with lots of patterns to work with asynchronous functions.
Since you need to call an asynchronous function for each record, you can use async.each, which executes an asynchronous function for all elements of an array. A last callback is called when all asynchronous tasks are finished.
var app = express();
function handleEvent = function (event, callback) {
if (! event.email) {
callback(new Error('Event object does not have an email address!'));
}
var event_id = 'blah';
db.get(event_id, function (err, result) {
var doc = result.value;
if (doc === undefined) {
// Add a new record
db.add(event_id, event, function (err, result) {
if (err) {
callback(new Error('There were processing errors'));
}
else {
callback(null);
}
});
}
});
}
function create(req, res) {
// https://github.com/caolan/async#each
async.each(req.body, handleEvent, function (err) {
if (err)
res.send(err.message, 500);
else
res.send('OK', 200);
});
}
I am little bit confused with my code it's not worked synchronusly as it's should be. I use everyauth to do authentication.
registerUser(function(newUserAttrs) {
var login = newUserAttrs[this.loginKey()];
user.CreateNewUser(newUserAttrs.login, newUserAttrs.password, newUserAttrs.email, function(res, err) {
if(!err) {
return usersByLogin[login] = newUserAttrs;
}
else {
throw err;
}
});
})
in another file I have write this code
exports.CreateNewUser = function(login, pass, mail, callback) {
var sql = "insert into `user`(login,mail,pass) values(?,?,?)";
client.query(sql, [login, mail, pass], function(err, results, fields) {
if(!err) {
callback(results);
console.log('test')
}
else {
callback(results, err);
}
});
};
This code are working fine. I have tested him. the only problem is they are working synchronosly (as normal). Can someone explain me what thing I have done in wrong way that make it async. I want to get it done in sync way.
The current code give me error (it's make a entry in database and produce error on browser)
Error: Step registerUser of `password` is promising: userOrErrors ; however, the step returns nothing. Fix the step by returning the expected values OR by returning a Promise that promises said values.
at Object.exec (E:\proj\Node\node_modules\everyauth\lib\step.js:68:11)
at E:\proj\Node\node_modules\everyauth\lib\stepSequence.js:26:38
at [object Object].callback (E:\proj\Node\node_modules\everyauth\lib\promise.js:13:12)
at RouteTriggeredSequence._bind (E:\proj\Node\node_modules\everyauth\lib\stepSequence.js:25:20)
at RouteTriggeredSequence.start (E:\proj\Node\node_modules\everyauth\lib\stepSequence.js:52:33)
at RouteTriggeredSequence.routeHandler (E:\proj\Node\node_modules\everyauth\lib\routeTriggeredSequence.js:13:13)
at Object.<anonymous> (native)
at nextMiddleware (E:\proj\Node\node_modules\connect\lib\middleware\router.js:175:25)
at param (E:\proj\Node\node_modules\connect\lib\middleware\router.js:183:16)
at pass (E:\proj\Node\node_modules\connect\lib\middleware\router.js:191:10)
Thanks
The two pieces of code you present are asynchronous and not synchronous!
With everyauth, to be able to handle asynchronous user creation you should use a Promise. So your code will be something like :
registerUser(function(newUserAttrs) {
var promise = this.Promise();
var login = newUserAttrs[this.loginKey()];
user.CreateNewUser(newUserAttrs.login, newUserAttrs.password, newUserAttrs.email, function(res, err) {
if(!err) {
return promise.fulfill(newUserAttrs);
}
else {
promise.fulfill(user);
}
});
})
Without promise you couldn't be sure that your new user has been added in your database. But if it doesn't matter you could have something like that:
registerUser(function(newUserAttrs) {
var login = newUserAttrs[this.loginKey()];
user.CreateNewUser(newUserAttrs.login, newUserAttrs.password, newUserAttrs.email, function(res, err) {
if (err) console.log(err);
});
return newUserAttrs;
})
Because you are doing a database query, this code has to be asynchronous. The anonymous function you pass to client.query will not be called until the database query is complete, so your callback gets called asynchronously.
You will need to treat this all as asynchronous, so for instance you'll have to trigger some other callback instead of returning the user object/throwing.