I have some synchronous code that looks like this:
function bulk_upload(files, callback) {
for (file in files) {
sync_upload(file); // blocks till file uploads
}
print('Done uploading files');
callback();
}
I now have to use an asynchronous API async_upload(file, callback) instead of sync_upload(file) to do the same. I have various options but not sure what is the best:
1) Use a sleep after the for-loop - that's a hack as I have to hope my timing is correct
2) Recursively chain my array:
function bulk_upload(files, callback) {
if (files.length == 0) {
print('Done uploading files');
callback();
} else {
async_upload(files.removeLast(), function() { bulk_upload(files, callback); });
}
}
This is not only hacky but sub-optimal as I could have uploaded my files in parallel using the new async_upload API but I ended up uploading sequentially.
3) Use a global counter:
function bulk_upload(files, callback) {
uploads = 0
for(file in files) {
async_upload(file, function() { uploads++; });
}
while(uploads < files.length) ; // Empty spin - this is stupid
print('Done uploading files');
callback();
}
4) Slightly better counter (but still awful):
function bulk_upload(files, callback) {
uploads = 0
for(file in files) {
async_upload(file, function() {
if (++uploads == files.length) { // this becomes uglier as I want to await on more stuff
print('Done uploading files');
callback();
};
});
}
}
You can use the async module's forEach method to do this:
function bulk_upload(files, callback) {
async.forEach(files, async_upload(file, callback), function (err) {
if (err) {
console.error('Failed: %s', err);
} else {
console.log('Done uploading files');
}
callback(err);
});
}
Further to my comment, it looks like this code will suffice using futures (untested).
function aync_upload_promise(file) {
// create a promise.
var promise = Futures.promise();
async_upload( file, function(err, data) {
if (err) {
// break it
promise.smash(err);
} else {
// fulfill it
promise.fulfill(data);
}
});
return promise;
}
var promises = [];
for(var i=0; i<files.length; ++i )
{
promises.push( aync_upload_promise( files[i] ) );
}
Futures
.join( promises )
.when( function() {
print('Done uploading files');
callback();
} )
.fail( function(err) { print('Failed :(', err); } )
Related
I am running a cron job with node with mongodb as the database. I am trying to close db connection and exit the process once the curr_1 each loop has executed completely.
However the exit() is called while function_2 is being executed. I understand this is due to the callback and is async in nature.
How do I make sure exit is called only once the curr_1.each is complete?
Any solution without promises?
function function_1(obj){
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
function_2(doc)
});
exit(obj)
}
function function_2(obj) {
coll_2.findOne({}, function(err, document) {
dosomeprocess(obj)
})
}
function exit(obj) {
// Close connection
console.log('closing connection')
obj.db.close();
process.exit();
}
It's a job for Node async....
For example:
async.each(
curr_1, // the collection to iterate over
function(doc, callback) { // the function, which is passed each
// document of the collection, and a
// callback to call when doc handling
// is complete (or an error occurs)
function_2(doc);
},
function(err) { // callback called when all iteratee functions
// have finished, or an error occurs
if (err) {
// handle errors...
}
exit(obj); // called when all documents have been processed
}
);
Without using any library:
function function_1(obj, callback) {
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
callback(err, doc);
});
}
function function_2(err, obj) {
coll_2.findOne({}, function(err, document) {
dosomeprocess(obj)
exit(err, obj);
})
}
function exit(err, obj) {
// Close connection
console.log('closing connection')
obj.db.close();
process.exit();
}
function_1(obj, function_2);
Using async module
var async = require('async');
async.waterfall([
function function_1(callback) {
var curr_1 = coll_1.find({})
curr_1.each(function(err, doc) {
if (err) {
callback(err, null)
} else {
allback(null, doc)
}
});
},
function function_2(obj, callback) {
coll_2.findOne({}, function(err, document) {
if (err) {
callback(err, null);
} else {
dosomeprocess(obj)
callback(null, obj);
}
})
}
], function done() {
obj.db.close();
process.exit();
});
Simply give a condition in your loop using counter.
function function_1(obj){
var curr_1 = coll_1.find({})
var curr_1Length = curr_1.length;
var counter = 0;
curr_1.each(function(err, doc) {
++counter;
//Check condition everytime for the last occurance of loop
if(counter == curr_1Length - 1){
exit(obj)
}
function_2(doc)
});
}
Hope it helps :)
I'm wondering what the best way to handle errors in long functions with promises are?
My function:
module.exports = function(data) {
var deferred = Q.defer();
var config = {};
fs.readdir(path, function(err, files) {
if (err) {
deferred.reject(new Error(err));
}
var infoPath = false;
files.forEach(function(filename) {
if (filename.indexOf('.info') !== -1) {
infoPath = path + '/' + filename;
config['moduleName'] = filename.replace('.info', '');
}
});
if (!infoPath) {
deferred.reject(new Error('Did not find .info-file'));
}
if (files.indexOf(config['moduleName'] + '.module') > -1) {
config.type = 'Modules';
}
else {
deferred.reject(new Error('Unknown project type'));
}
// Parse the info file.
fs.readFile(infoPath, function (err, content) {
if (err) {
deferred.reject(new Error(err));
}
data.config = config;
data.infoContent = content.toString();
deferred.resolve(data);
});
});
return deferred.promise;
};
As far as I understand it this is the way to use Q.defer. But if a error is thrown, I don't want/need it to try the rest of function. Am I missing something or are there a better way to do this?
Rejecting a promise doesn't miraculously stop the function from executing the rest of its code. So after rejecting, you should return from the function:
if (err) {
deferred.reject(new Error(err));
return;
}
Or shorter:
if (err) {
return deferred.reject(new Error(err));
}
I'm getting a 'callback already used' error and I don't know why. I am using async and want to chain two functions because the second function depends on the first function to complete.
I'm new-ish to Node.js and still wrapping my head around async/callbacks. Thanks so much for helping out.
getCdn takes in cnames, and if the cname is part of a CDN it pushes the results into a global variable called cdnAttrs.
function getCdn(cnameDict, callback) {
// cdnAttributes contains associative array with each web attribute: {name_in_db : code_snippet_to_find_in_cname}
for (var key in cdnAttributes) {
if (cdnAttributes.hasOwnProperty(key)) {
var snippet = -1;
// some technologies contain multiple code snippets, in that case they are stored as array. Single code snippets are stored as string
if (!Array.isArray(cdnAttributes[key])) {
snippet = cnameDict['cname'].indexOf(cdnAttributes[key])
}
else {
// check each code snippet within the array, if any match the source code, update 'snippet'
for (var n = 0; n < cdnAttributes[key].length; n++) {
var val = cnameDict['cname'].indexOf(cdnAttributes[key][n])
if (val > -1) {
snippet = val
}
}
}
// if attribute found in tag, create cdnAttrs[cdn] = [{from: hostname, proof: cname}, {from: hostname2, proof: cname2}, ...]
if (snippet > -1) {
try {
cdnAttrs[key].push(cnameDict);
}
catch (e) {
cdnAttrs[key] = [];
cdnAttrs[key].push(cnameDict);
}
callback();
} else {
callback();
}
} else {
callback();
}
}
}
My async function looks like this:
async.series([
// THIS FUNCTION WORKS FINE...
function(callback) {
async.each(toCheck, function(hostname, callback) {
getCname(hostname, callback);
},callback);
},
// THIS FUNCTION RETURNS RETURNS Error("Callback was already called.")
function(callback) {
async.each(toCheckCnames, function(cnameDict, callback) {
getCdn(cnameDict, callback);
},callback);
}
], function(err){
if(err) {
console.log('ERROR');
}else{
console.log('toCheckCnames is done: '+JSON.stringify(toCheckCnames));
console.log('cdnAttrs is done: '+JSON.stringify(cdnAttrs));
}
})
the getCnames function works:
function getCname(hostname, callback){
dns.resolve(hostname, 'CNAME', function(error, cname) {
if (cname) {
toCheckCnames.push({from: hostname, cname: cname[0]});
callback();
}
// if not CNAMEd, check SOA on www.domain.com and domain.com
else {
if (hostname.slice(0,4) == 'www.') {
hostname = hostname.slice(4);
}
nativedns.resolve(hostname, 'SOA', function(error, records) {
if(!error && records) {
toCheckCnames.push({from: hostname, cname: records[0]['primary']});
callback();
}
else if (!error) {
hostname = 'www.'+ hostname
nativedns.resolve(hostname, 'SOA', function(error, records) {
if (!error) {
toCheckCnames.push({from: hostname, cname: records[0]['primary']});
callback();
}
else callback()
});
}
else callback()
});
}
});
}
Your getCdn function is a loop that will call the callback after each iteration. If calling the callback is intended to stop the loop execution, you can do return callback(). Otherwise you need to reorganize your code to only call the callback once when the function is done.
UPDATE:
You can also simplify your async.each calls:
// Was this
async.each(toCheck, function(hostname, callback) {
getCname(hostname, callback);
},callback);
// Could be this
async.each(toCheck, getCname, callback);
I would like to do something like this
function scan(apath){
var files = fs.readdirSync(apath);
for(var i=0; i<files.length;i++){
var stats = fs.statSync(path.join(apath,files[i]))
if(stats.isDirectory()){
results.push(path.join(apath, files[i]))
scan(path.join(apath,files[i]))
}
if(stats.isFile()){
results.push(path.join(apath,files[i]))
}
}
}
but asynchronously.
Trying this with asynchronous functions led me to a nightmare with something like this.
function scan(apath){
fs.readdir(apath, function(err, files)){
var counter = files.length;
files.forEach(function(file){
var newpath = path.join(apath, file)
fs.stat(newpath, function(err, stat){
if(err) return callback(err)
if(stat.isFile())
results.push(newpath)
if(stat.isDirectory()){
results.push(newpath)
scan(newpath)
}
if(--counter <=0) return
})
})
}
}
All hell breaks loose in node's stack because things don't happen in logical succession as they do in synchronous methods.
you can try async module, and use like this:
function scan(apath, callback) {
fs.readdir(apath, function(err, files) {
var counter = 0;
async.whilst(
function() {
return counter < files.length;
},
function(cb) {
var file = files[counter++];
var newpath = path.join(apath, file);
fs.stat(newpath, function(err, stat) {
if (err) return cb(err);
if (stat.isFile()) {
results.push(newpath);
cb(); // asynchronously call the loop
}
if (stat.isDirectory()) {
results.push(newpath);
scan(newpath, cb); // recursion loop
}
});
},
function(err) {
callback(err); // loop over, come out
}
);
});
}
look for more about async.whilst
I basically I have two functions :
One to read a file (post) : loadPost(name)
One to read all files : loadPosts()
Obviously loadPosts() calls loadPost(name).
Both return the final html.
Ideally I should write it asynchronously.
The problem is : I don't know how to make that asynchronously, as I need to wait until the file is completely read before I can continue.
Here is my synchronous solution:
function loadPost(name){
var post = fs.readFileSync("_posts/"+name,'utf8');
// Convert Markdown to html
var marked = mark(post);
return marked;
}
function loadPosts(){
var files = fs.readdirSync("_posts/");
var html;
for(var i = 0; i < files.length ; i++){
html += loadPost(files[i]);
}
return html;
}
Something like this, no third party libs necessary. Fairly simple really.
/*jshint node:true */
function loadPosts(callback) {
'use strict';
var allHtml = ''; //Declare an html results variable within the closure, so we can collect it all within the functions defined within this function
fs.readdir("_posts/", function (err, files) {
function loadPost(name) {
fs.read("_posts/" + name, 'utf8', function (err, data) {
allHtml += mark(data);//Append the data from the file to our html results variable
if (files.length) {
loadPost(files.pop()); //If we have more files, launch another async read.
} else {
callback(allHtml); //If we're out of files to read, send us back to program flow, using the html we gathered as a function parameter
}
});
}
loadPost(files.pop());
});
}
function doSomethingWithHtml(html) {
'use strict';
console.log(html);
}
loadPosts(doSomethingWithHtml);
You'll want to use the async.js package to make your life easier.
function loadPost(name, callback) {
fs.readFile("_posts/+"name,{encoding:'utf8'},function(err, data) {
if(err) return callback(err);
// convert markdown to html
var marked = mark(post);
callback(false, marked);
});
}
function loadPosts(cb) {
fs.readdir("_posts/", function(err, dir) {
if(err) return cb(err);
var html;
async.eachSeries(dir, function(one, callback) {
loadPost(one, function(err, post) {
if(err) return cb(err);
html += post;
callback(false);
});
},
// final callback
function(err) {
if(err) { console.log(err); cb(true); return; }
cb(false, html);
});
});
}