I'm trying to read some files from dir with async.waterfall, it seems to me that I'm doing stuff right, but I get the specified error and the readData function is never called. What's wrong?
var fs = require("fs");
var async = require("async");
var folder = "./files/";
try {
async.waterfall([
function readDir(cb) {
fs.readdir(folder, function(err, files) {
cb(err, files);
});
},
function loopFiles(files, cb) {
files.forEach(function(fn) {
console.log("loop " + fn);
cb(null, fn);
});
},
function check(fn, cb) {
console.log("check "+fn);
fs.stat(folder + fn, function(err, stats) {
console.log(stats.isFile());
cb(err, stats, fn);
});
},
function readData(stats, fn, cb) {
console.log("read "+fn);
if (stats.isFile()) {
fs.readFile(folder + fn, "utf-8", function(err, data) {
cb(err, data);
});
}
}
], function(err, result) {
if (err) {
throw err;
}
console.log(result);
});
} catch (err) {
console.log(err);
}
The problem is that you're calling cb(null, fn) in loopFiles() multiple times if files.length > 1. You will probably need to perform a separate async.waterfall() or use some other async.* method for each file.
One other problem is in readData() where you aren't calling cb() in the case that stats.isFile() evaluates to false.
Related
I am new to nodejs and trying to cat multiple css files on-the-fly while coding. The package chokidar allow me to call a function when a file is modified, however I have a problem with the execution.
var goconcat =
fs.readdir(paths, function (err, files) {
if (err) {console.log(err);}
fs.unlink(paths + 'concat.css', function (err) {
if (err) throw err;
var list = files.map(function (files) {
return path.join(paths, files);
});
concat(list, paths + 'concat.css', function(err) {
if (err) throw err
});
});
});
I want to first delete the previous file, then read the directory and then write a new "concat.css". However I have an error;
Error: ENOENT: no such file or directory, open 'public/css/concat.css'
at error (native)
It appears that the function concat() is executed before the directory update and not after, and therefore it is trying to cat a file that just have been deleted. Why ?
I know that nodejs is executing functions in a synchronous way but I can't find a way to solve this problem. I tried async but I can't declare a variable between two functions and I couldn't manage to make it work.
If it cannot exist in a callback, using the setTimeout(fn, 0) trick may help make sure it's executed after the variable assignment.
var goconcat =
fs.readdir(paths, function (err, files) {
if (err) {console.log(err);}
fs.unlink(paths + 'concat.css', function (err) {
if (err) throw err;
var list = files.map(function (files) {
return path.join(paths, files);
});
setTimeout(function() {
concat(list, paths + 'concat.css', function(err) {
if (err) throw err
})}, 0);
});
});
The problem you're having is that your concat function is being invoked before the file is deleted by invoking unlink. You can prevent this by having nested callbacks; however, you can probably have better control flow if you use a module like async, and prevent yourself from dealing with Callback Hell.
Below is an example on how you can use the async module.
var fs = require('fs');
var async = require('async');
var myDir = __dirname + '/data';
async.waterfall([function(callback) {
fs.readdir(myDir, 'utf-8', function(error, files) {
if (error) {
return callback(error);
}
return callback(null, files);
});
}, function(files, callback) {
fs.open(myDir + '/myFile', 'wx', function(error, f) {
if (error && error.code === 'EEXIST') {
return callback(null, 'EEXIST');
}
return callback(null, 'CREATE');
});
}, function(fileStatus, callback) {
if (fileStatus === 'EEXIST') {
console.log('File exists. Deleting file...');
fs.unlink(myDir + '/myFile', function(error) {
if (error) {
return callback(error);
} else {
return callback(null);
}
});
} else {
console.log('File does not exist...');
return callback(null);
}
}, function(callback) {
fs.writeFile(myDir + '/myFile', "Hello World", function(err) {
if(err) {
return callback(error);
}
return callback(null, 'File Created');
});
}], function(error, results) {
console.error(error);
console.log(results);
});
The waterfall function runs the tasks array of functions in series,
each passing their results to the next in the array. However, if any
of the tasks pass an error to their own callback, the next function is
not executed, and the main callback is immediately called with the
error.
I have this async.parallel functionality inside an aysnc.eachSeries call.
I hardcoded an error so I could pass it, to see if it was behaving the way I thought. For some reason, when I pass an error, it doesn't get thrown in the final callback named "doneWithSeries".
async.eachSeries(jsonDataArr, function iterator(item, callback) {
async.parallel([
function (cb) {
if (item.hasOwnProperty('event.type')) {
var event_type = item['event.type'];
delete item['event.type'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + event_type + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
},
function (cb) {
if (item.hasOwnProperty('status_desc')) {
var status_desc = item['status_desc'];
delete item['status_desc'];
try {
var json = JSON.stringify(item);
}
catch (err) {
throw err;
}
fs.writeFile('./enriched_data/' + status_desc + '.json', json, function (err) {
if (err) {
cb(err);
}
else {
cb(null);
}
});
}
}
],
function doneWithParallel(err) {
callback(new Error('throw this baby')); //shouldn't the first incident of error pass the error straight to the doneWithSeries callback below?
})
},
function doneWithSeries(err) {
if (err) {
throw err;
}
else {
console.log('success');
}
});
here is a distilled version of the code without anything unnecessary:
var async = require('async');
async.eachSeries(['1', '2'], function (item, callback) {
async.parallel([
function (cb) {
setTimeout(function () {
cb(null, 'one');
}, 200);
},
function (cb) {
setTimeout(function () {
cb(null, 'two');
}, 100);
}
],
function doneWithParallel(err, results) {
console.log('results', results);
callback(new Error('duh'));
})
},
function doneWithSeries(err) {
if (err)
throw err;
});
indeed that works. can't figure out why my code above doesn't, accept perhaps that the array could be empty even though when I run my code the success message gets logged...weird.
I think that's expected behavior if your list is empty. async will always call the final callback with no error even if there is no input list.
I have a requirement to make several API requests and then do some processing on the combines result sets. In the example below, you can see that 3 requests are made (to /create) by duplicating the same request code however I would like to be able to specify how many to make. For example, I may wish to run the same API call 50 times.
How can I make n calls without duplicating the API call function n times?
async.parallel([
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/api/store/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
}
],
function(err, results){
if (err) {
console.log(err);
}
// do stuff with results
});
First, wrap the code that you want to call many times in a function:
var doRequest = function (callback) {
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err);
}
callback(null, res.body.id);
});
}
Then, use the async.times function:
async.times(50, function (n, next) {
doRequest(function (err, result) {
next(err, result);
});
}, function (error, results) {
// do something with your results
}
Create an array with as many references to the function as you need tasks in your workload. Then pass them to async.parallel. For example:
var async = require("async");
var slowone = function (callback) {
setTimeout(function () {
callback(null, 1);
}, 1000);
};
async.parallel(
dd(slowone, 100),
function (err, r) {
console.log(JSON.stringify(r));
}
);
// Returns an array with count instances of value.
function dd(value, count) {
var result = [];
for (var i=0; i<count; i++) {
result.push(value);
}
return result;
}
Note again that there is only one instance of the slow running function, in spite of there being many references to it.
When I run collection.find() in MongoDB/Node/Express, I need to return value for my array like this but iam in callback hell;
foursquare.getVenues(params,function(error, venues) {
if (!error) {
var places = [];
venues.response.venues.forEach(function(e) {
places.push(
{
obj_id:e.id,
name:e.name,
distance:e.distance,
here_now:req.collection.findById(e.id) //count- i want need this value
}
);
});
res.send(places);
}
});
You can try to use Async https://github.com/caolan/async#each
var async = require('async');
...
foursquare.getVenues(params, function (error, venues) {
if (!error) {
throw err;
}
var places = [];
async.each(venues.response.venues, function (e, callback) {
db.collection.findById(e.id, function (err, res) {
places.push({
obj_id: e.id,
name: e.name,
distance: e.distance,
here_now: res
});
callback()
});
}, function (err) {
if (err) {
console.log('A file failed to process');
} else {
console.log('All files have been processed successfully');
res.send(places);
}
});
});
or Using async.map
var async = require('async');
var createArray = function (e, cb) {
db.collection.findById(e.id,function(err,res){
var obj = {
obj_id: e.id,
name: e.name,
distance: e.distance,
here_now: res
}
cb(null, obj);
});
}
async.map(venues.response.venues, createArray, function (err, places) {
if(err){
throw err;
}
console.log(places);
});
I'm trying to design a webpage. I have a function that I call to get all info needed for an individual's home page. A snippet of the code is:
exports.getHomePageData = function(userId, cb) {
var pageData = {};
pageData.userFullName = dbUtil.findNameByUserId(userId, function(err){
if (err) cb(err);
});
pageData.classes = dbUtil.findUserClassesByUserId(userId, function(err){
if (err) cb(err);
});
cb(pageData);
}
The problem I'm having is that the cb(pageData) is being called before I even finish setting the elements.
I've seen that people use the async library to solve this, but I was wondering if there was any other way for me to do it without needing more modules.
One possible approach:
exports.getHomePageData = function(userId, cb) {
var pageData = {},
filler = function() {
if ('userFullName' in pageData
&& 'classes' in pageData)
cb(null, pageData);
};
dbUtil.findNameByUserId(userId, function(err, name) {
if (err) {
cb(err);
return;
}
pageData.userFullName = name;
filler();
});
dbUtil.findUserClassesByUserId(userId, function(err, classes) {
if (err) {
cb(err);
return;
}
pageData.classes = classes;
filler();
});
}
It looks like dbUtil.findUserClassesByUserId and dbUtil.findNameByUserId are asynchronous methods; that usually indicates that they do not return a value, and instead use the callback to give you the data.
Both functions are most likely expecting a signature like follows:
function(err, data) {
// if err is set, an error occurred, otherwise data is set with the result
}
Thus, your function should look like this instead:
exports.getHomePageData = function(userId, cb) {
dbUtil.findNameByUserId(userId, function(err, userFullName){
if (err) {
cb(err);
return;
}
dbUtil.findUserClassesByUserId(userId, function(err, classes){
if (err) {
cb(err);
return;
}
var pageData = {
userFullName: userFullName,
classes: classes
};
cb(pageData);
});
});
}