Lets say, for example, I want to write a nodejs program where I have two or three independent parts like fs.readdir, fs.copy, etc. on different locations, but the result all three actions is to be sent to a json file like this:
var fs = require('fs-extra');
var jsd = {
"act1" : false,
"act2" : false,
"act3" : false
}
fs.readdir(path1, function (err, files) {
if (err) jsd.act1 = err;
for (x in files) console.log(files[x]);
jsd.act1 = true;
});
fs.copy(path2, path3, function (err) {
if (err) jsd.act2 = err;
jsd.act2 = true;
});
fs.remove(path4, function (err) {
if (err) jsd.act3 = err;
jsd.act3 = true;
});
// all three of the above actions are independent, so it makes sense that all of them are executed asynchronously.
// Now we write jsd object to a json file; jsd's contents are dependent on the above actions though
fs.writeJson("./data.json", jsd, function (err, files) {
if (err) return console.error(err);
});
How do I make sure that the correct data is entered into the file data.json, i.e fs.writeJson executes after the actions previous to it are executed first?
I know one way is to nest all of them, i.e,
readdir() {
copy() {
remove() {
writeJson();
}
}
}
But this may result in callback hell, so is there a better way to do this?
you can use Promise or module async,
if you use promise, first you must convert all callback function into Promise like this:
const reddir = function(path) {
return new Promise((resolve, reject) => {
fs.readdir(path, (err, files) => {
if (err) return reject(err);
for (x in files) console.log(files[x]);
resolve(true);
});
})
}
then you can use
Promise.all([reddir(path1), copy(path2, path3), remove(path4)])
.spread((act1, act2, act3) => { //.spread is bluebird feature
return writeJson(./data.json);
})
.catch(e => {
// all error can handled in this
})
if you use async module, you can write like this:
async.parallel({
act1: function(cb){
fs.reddir(path1, (err, files) => {
if (err) return cb(err);
for (x in files) console.log(files[x]);
cb(true);
})
},
act2: ...
},(err, jsd) => { // jsd will be {act1: true, act2: ...}
if (err) return console.error(err); // handle all above error here;
fs.writeJson("./data.json", jsd, function (err, files) {
if (err) return console.error(err);
});
})
Related
Today, when I was working with node, I met some special async functions with "overloads" that accept both promises and callbacks. Like this:
doSomething(result => {
console.log(result)
})
doSomething()
.then(result => console.log(result))
And probably this:
const result = await doSomething()
console.log(result)
I tried to implement this in my code but was unsuccessful. Any help would be appreciated.
You can make a function like this by creating a promise, then chaining on that promise with the argument if there is one, and then returning that chained promise. That will make it call the callback at the appropriate time, as well as giving you access to the promise that will complete when the callback completes. If you want the original promise even when there's a callback (not the chained version), then you can return that instead, by still chaining but then returning the original promise instead.
function f(cb) {
let promise = new Promise((resolve, reject) => {
setTimeout(() => resolve(123), 1000);
});
if (cb) {
return promise.then(cb);
} else {
return promise;
}
}
// usage 1
f(console.log)
// usage 2
f().then(console.log)
Let's say you had a function that was going to read your config file and parse it and you wanted to support both versions. You could do that like this with two separate implementation inside. Note, this has full error handling and uses the nodejs calling convention for the callback that passes parameters (err, result):
function getConfigData(filename, callback) {
if (typeof callback === "function") {
fs.readFile(filename, function(err, data) {
try {
if (err) throw err;
let result = JSON.parse(data);
callback(null, result);
} catch(e) {
callback(err);
}
});
} else {
return fs.promises.readFile(filename).then(data => {
return JSON.parse(data);
}).
}
}
This could then be used as either:
getConfigData('./config.json').then(result => {
console.log(result);
}).catch(err => {
console.log(err);
});
configData('./config.json', (err, data) => {
if (err) {
console.log(err);
} else {
console.log(data);
}
});
Depending upon the specific asynchronous operation, it may be better or more efficient to have two separate implementations internally or to have one implementation that you adapt at the end to either a callback or a promise.
And, there's a useful helper function if you adapt a promise to a callback in multiple places like this:
function callbackHelper(p, callback) {
if (typeof callback === "function") {
// use nodejs calling convention for callbacks
p.then(result => {
callback(null, result);
}, err => {
callback(err);
});
} else {
return p;
}
}
That lets you work up a simpler shared implementation:
function getConfigData(filename, callback) {
let p = fs.promises.readFile(filename).then(data => {
return JSON.parse(data);
});
return callbackHelper(p, callback);
}
I am new to nodejs and trying to cat multiple css files on-the-fly while coding. The package chokidar allow me to call a function when a file is modified, however I have a problem with the execution.
var goconcat =
fs.readdir(paths, function (err, files) {
if (err) {console.log(err);}
fs.unlink(paths + 'concat.css', function (err) {
if (err) throw err;
var list = files.map(function (files) {
return path.join(paths, files);
});
concat(list, paths + 'concat.css', function(err) {
if (err) throw err
});
});
});
I want to first delete the previous file, then read the directory and then write a new "concat.css". However I have an error;
Error: ENOENT: no such file or directory, open 'public/css/concat.css'
at error (native)
It appears that the function concat() is executed before the directory update and not after, and therefore it is trying to cat a file that just have been deleted. Why ?
I know that nodejs is executing functions in a synchronous way but I can't find a way to solve this problem. I tried async but I can't declare a variable between two functions and I couldn't manage to make it work.
If it cannot exist in a callback, using the setTimeout(fn, 0) trick may help make sure it's executed after the variable assignment.
var goconcat =
fs.readdir(paths, function (err, files) {
if (err) {console.log(err);}
fs.unlink(paths + 'concat.css', function (err) {
if (err) throw err;
var list = files.map(function (files) {
return path.join(paths, files);
});
setTimeout(function() {
concat(list, paths + 'concat.css', function(err) {
if (err) throw err
})}, 0);
});
});
The problem you're having is that your concat function is being invoked before the file is deleted by invoking unlink. You can prevent this by having nested callbacks; however, you can probably have better control flow if you use a module like async, and prevent yourself from dealing with Callback Hell.
Below is an example on how you can use the async module.
var fs = require('fs');
var async = require('async');
var myDir = __dirname + '/data';
async.waterfall([function(callback) {
fs.readdir(myDir, 'utf-8', function(error, files) {
if (error) {
return callback(error);
}
return callback(null, files);
});
}, function(files, callback) {
fs.open(myDir + '/myFile', 'wx', function(error, f) {
if (error && error.code === 'EEXIST') {
return callback(null, 'EEXIST');
}
return callback(null, 'CREATE');
});
}, function(fileStatus, callback) {
if (fileStatus === 'EEXIST') {
console.log('File exists. Deleting file...');
fs.unlink(myDir + '/myFile', function(error) {
if (error) {
return callback(error);
} else {
return callback(null);
}
});
} else {
console.log('File does not exist...');
return callback(null);
}
}, function(callback) {
fs.writeFile(myDir + '/myFile', "Hello World", function(err) {
if(err) {
return callback(error);
}
return callback(null, 'File Created');
});
}], function(error, results) {
console.error(error);
console.log(results);
});
The waterfall function runs the tasks array of functions in series,
each passing their results to the next in the array. However, if any
of the tasks pass an error to their own callback, the next function is
not executed, and the main callback is immediately called with the
error.
I would like to return the results from the npm async in nodejs, it returns undefined when I call the function 'findRequestAgent'
findRequestAgent : (agentId) => {
var queries = [];
var datum;
queries.push((callback) => {
MovingAgent.findById(agentId,(err,doc) => {
if (err) {
console.log("There was an error finding the moving agent");
throw err;
}
callback(null, doc)
})
});
queries.push((callback) => {
SelfStander.findOne({ user_id: agentId}, (err, doc) => {
if (err) {
console.log("There was an error finding the self stander");
throw err;
}
callback(null, doc)
})
});
queries.push((callback) =>{
MovingCompany.findOne({custom_id: agentId}, (err, doc) => {
if (err) {
console.log("There was an error finding the self stander");
throw err;
}
callback(null, doc);
})
});
async.parallel(queries, (err, results) => {
if (err) {
console.log("There was an error perfoming async");
throw err;
}
datum = results;
console.log(datum);
});
return datum;
}
What can I do so that when I call the above function 'findRequestAgent' it returns the results
All of the Mongo queries return promises.
var queries = [];
queries.push(
MovingAgent.findById(agentId,(err,doc) => {
if (err) {
console.log("There was an error finding the moving agent");
throw err;
}
return (null, doc);
})
);
...
return Promise.all(queries).then(
(results) => results; //array of results
).catch(
(err) => console.log(err);
)
No need for the async part
As a general rule, asynchronous functions don't return values directly. You need to use another mechanism such as a callback, events, promises.
Without knowing more about your use case, it's hard to know what the right solution is. But you should definitely stop and make sure you understand how asynchronous functions work in JavaScript before continuing.
The easiest solution is to do what you need to do in side the callback passed to .parallel() but again, without knowing more, it can't be said for certain that that will work for your use case.
var shortLinks = [];
Link.find({}, function (err, links) {
if (err) {
console.log(err);
} else {
links.map(link => {
shortLinks.push(link.shortLink);
});
}
console.log(shortLinks);//shortLinks has values, all okey
});
console.log(shortLinks); //shortLinks is empty
i need to use shortLinks after Link.find({}) but array is empty.
Need to return shortLinks.
Callbacks. The function(err, links) is called asynchronously, so shortLinks isn't populated until that function is called. Your bottom console.log is being called first, due to how callbacks work.
https://developer.mozilla.org/en-US/docs/Mozilla/js-ctypes/Using_js-ctypes/Declaring_and_Using_Callbacks
need to use promise:
const shortLinks = [];
const getShortLinks = Link.find({}, function (err, links) {
if (err) {
console.log(err);
} else {
links.map(link => {
shortLinks.push(link.shortLink);
});
}
});
getShortLinks.then(function(links){
console.log(shortLinks);
}, function(err){
console.log(err);
});
I have a requirement to make several API requests and then do some processing on the combines result sets. In the example below, you can see that 3 requests are made (to /create) by duplicating the same request code however I would like to be able to specify how many to make. For example, I may wish to run the same API call 50 times.
How can I make n calls without duplicating the API call function n times?
async.parallel([
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
},
function(callback){
request.post('http://localhost:3000/api/store/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err, null);
}
callback(null, res.body.id);
});
}
],
function(err, results){
if (err) {
console.log(err);
}
// do stuff with results
});
First, wrap the code that you want to call many times in a function:
var doRequest = function (callback) {
request.post('http://localhost:3000/create')
.send(conf)
.end(function (err, res) {
if (err) {
callback(err);
}
callback(null, res.body.id);
});
}
Then, use the async.times function:
async.times(50, function (n, next) {
doRequest(function (err, result) {
next(err, result);
});
}, function (error, results) {
// do something with your results
}
Create an array with as many references to the function as you need tasks in your workload. Then pass them to async.parallel. For example:
var async = require("async");
var slowone = function (callback) {
setTimeout(function () {
callback(null, 1);
}, 1000);
};
async.parallel(
dd(slowone, 100),
function (err, r) {
console.log(JSON.stringify(r));
}
);
// Returns an array with count instances of value.
function dd(value, count) {
var result = [];
for (var i=0; i<count; i++) {
result.push(value);
}
return result;
}
Note again that there is only one instance of the slow running function, in spite of there being many references to it.