Nodejs FileReads Sync to Async - node.js

I basically I have two functions :
One to read a file (post) : loadPost(name)
One to read all files : loadPosts()
Obviously loadPosts() calls loadPost(name).
Both return the final html.
Ideally I should write it asynchronously.
The problem is : I don't know how to make that asynchronously, as I need to wait until the file is completely read before I can continue.
Here is my synchronous solution:
function loadPost(name){
var post = fs.readFileSync("_posts/"+name,'utf8');
// Convert Markdown to html
var marked = mark(post);
return marked;
}
function loadPosts(){
var files = fs.readdirSync("_posts/");
var html;
for(var i = 0; i < files.length ; i++){
html += loadPost(files[i]);
}
return html;
}

Something like this, no third party libs necessary. Fairly simple really.
/*jshint node:true */
function loadPosts(callback) {
'use strict';
var allHtml = ''; //Declare an html results variable within the closure, so we can collect it all within the functions defined within this function
fs.readdir("_posts/", function (err, files) {
function loadPost(name) {
fs.read("_posts/" + name, 'utf8', function (err, data) {
allHtml += mark(data);//Append the data from the file to our html results variable
if (files.length) {
loadPost(files.pop()); //If we have more files, launch another async read.
} else {
callback(allHtml); //If we're out of files to read, send us back to program flow, using the html we gathered as a function parameter
}
});
}
loadPost(files.pop());
});
}
function doSomethingWithHtml(html) {
'use strict';
console.log(html);
}
loadPosts(doSomethingWithHtml);

You'll want to use the async.js package to make your life easier.
function loadPost(name, callback) {
fs.readFile("_posts/+"name,{encoding:'utf8'},function(err, data) {
if(err) return callback(err);
// convert markdown to html
var marked = mark(post);
callback(false, marked);
});
}
function loadPosts(cb) {
fs.readdir("_posts/", function(err, dir) {
if(err) return cb(err);
var html;
async.eachSeries(dir, function(one, callback) {
loadPost(one, function(err, post) {
if(err) return cb(err);
html += post;
callback(false);
});
},
// final callback
function(err) {
if(err) { console.log(err); cb(true); return; }
cb(false, html);
});
});
}

Related

unable to return data from module in express node js

I have written below code in one file:
models/exported.js
module.exports = {
processedList: function(store_name) {
var t;
var tradeIds = exported.find({storename: store_name}, function (err, value) {
if (err) return console.error(err);
return value;
}).select('tid -_id');
}, // Export connection here
};
I have another file in routes
routes/exported.js
var exported = require('../models/exported.js');
var tradeIds = exported.processedList(storename);
console.log('simer'+tradeIds);
}
but I get undefined in console.log. If instead of return statement in processedlist I write console.log then the result gets console. But my requirement is to return data from model file to route file.
I am new to express and node js.
I guidance would be highly appreciated.
Acoording to your question, you want calling a function from route and get return response from your function to route. simple use callback functions.
models/exported.js
module.exports = {
processedList: function (store_name, callback) {
var t;
var tradeIds = exported.find({storename: store_name}, function (err, value) {
if (err) {
callback("error", err)
} else {
callback("success", value)
}
}).select('tid -_id');
}
}
routes/exported.js
var exported = require('../models/exported.js');
exported.processedList('storename', function (err, results) {
if (err == 'error') {
console.log(err);
} else {
console.log(results);
}
});
You are trying sync operation in async environment. processedList may or may not have completed when you try to console log tradeIds. NodeJS would not wait for it to complete because it is asynchronous in nature (by design and it is not a bug). You can pass callback rather than executing this way.
models/exported.js
module.exports = {
processedList: function(store_name, cb) {
var t;
var tradeIds = exported.find({storename: store_name}, function (err, value) {
if (err) return cb(err);
cb(null, value);
}).select('tid -_id');
}, // Export connection here
};
routes/exported.js
var exported = require('../models/exported.js');
exported.processedList(storename, function(err, results) {
if (err) { console.log(err); }
console.log(results);
});
This makes sure that console.log happens only when processedList finishes execution.

How to use a nested loop in request.head - Node JS

I am trying to move download images from parse and save it to my local. I have this piece of code that does the job for me. This works well when there is only one request but when I put in a loop, it doesn't hold good.
`for(var i = 0; i < 5; i++) {
console.log(i);//to debug
var filename = results_jsonObj[i].imageFile.name;
var uri = results_jsonObj[i].imageFile.url;
request.head(uri, function(err, res, body){
if (err){
console.log(err);
console.log(item);
return;
}else {
console.log(i); //to debug
var stream = request(uri);
stream.pipe(
fs.createWriteStream("images/"+filename)
.on('error', function(err){
callback(error, filename);
stream.read();
})
)
}
});
}`
Irrespective of the loop condition I have, only one image downloads to the mentioned directory.
Below is the op
The input is from a Json file and I have the request, fs, parse module included in the node js program.
Any help on how to go about this?
I have got this fixed now. As advised in the comments it was async which helped me do the trick.
for(var i = 0; i < 900; i++) {
async.forEachOf(results_jsonObj[i], function(value, key, callback){
var image = {};
image.key = key;
image.value = value;
if(image.key == 'imageFile')
{
var filename = image.value.name;
var uri = image.value.url;
// console.log(filename, uri);
}
request.head(uri, function(err, res, body){
if (err){
console.log(err);
// console.log(item);
return;
}else {
// console.log(i,res.headers['content-type']); //to debug
var stream = request(uri);
stream.pipe(
fs.createWriteStream("images/"+filename)
.on('error', function(err){
callback(error, filename);
stream.read();
})
)
}
});
callback();
}, function(err){
if (err) {
console.log('one of the api failed, the whole thing will fail now');
}
});
}

Nodejs asynchronus callbacks and recursion

I would like to do something like this
function scan(apath){
var files = fs.readdirSync(apath);
for(var i=0; i<files.length;i++){
var stats = fs.statSync(path.join(apath,files[i]))
if(stats.isDirectory()){
results.push(path.join(apath, files[i]))
scan(path.join(apath,files[i]))
}
if(stats.isFile()){
results.push(path.join(apath,files[i]))
}
}
}
but asynchronously.
Trying this with asynchronous functions led me to a nightmare with something like this.
function scan(apath){
fs.readdir(apath, function(err, files)){
var counter = files.length;
files.forEach(function(file){
var newpath = path.join(apath, file)
fs.stat(newpath, function(err, stat){
if(err) return callback(err)
if(stat.isFile())
results.push(newpath)
if(stat.isDirectory()){
results.push(newpath)
scan(newpath)
}
if(--counter <=0) return
})
})
}
}
All hell breaks loose in node's stack because things don't happen in logical succession as they do in synchronous methods.
you can try async module, and use like this:
function scan(apath, callback) {
fs.readdir(apath, function(err, files) {
var counter = 0;
async.whilst(
function() {
return counter < files.length;
},
function(cb) {
var file = files[counter++];
var newpath = path.join(apath, file);
fs.stat(newpath, function(err, stat) {
if (err) return cb(err);
if (stat.isFile()) {
results.push(newpath);
cb(); // asynchronously call the loop
}
if (stat.isDirectory()) {
results.push(newpath);
scan(newpath, cb); // recursion loop
}
});
},
function(err) {
callback(err); // loop over, come out
}
);
});
}
look for more about async.whilst

How to set variable before callback is called?

I'm trying to design a webpage. I have a function that I call to get all info needed for an individual's home page. A snippet of the code is:
exports.getHomePageData = function(userId, cb) {
var pageData = {};
pageData.userFullName = dbUtil.findNameByUserId(userId, function(err){
if (err) cb(err);
});
pageData.classes = dbUtil.findUserClassesByUserId(userId, function(err){
if (err) cb(err);
});
cb(pageData);
}
The problem I'm having is that the cb(pageData) is being called before I even finish setting the elements.
I've seen that people use the async library to solve this, but I was wondering if there was any other way for me to do it without needing more modules.
One possible approach:
exports.getHomePageData = function(userId, cb) {
var pageData = {},
filler = function() {
if ('userFullName' in pageData
&& 'classes' in pageData)
cb(null, pageData);
};
dbUtil.findNameByUserId(userId, function(err, name) {
if (err) {
cb(err);
return;
}
pageData.userFullName = name;
filler();
});
dbUtil.findUserClassesByUserId(userId, function(err, classes) {
if (err) {
cb(err);
return;
}
pageData.classes = classes;
filler();
});
}
It looks like dbUtil.findUserClassesByUserId and dbUtil.findNameByUserId are asynchronous methods; that usually indicates that they do not return a value, and instead use the callback to give you the data.
Both functions are most likely expecting a signature like follows:
function(err, data) {
// if err is set, an error occurred, otherwise data is set with the result
}
Thus, your function should look like this instead:
exports.getHomePageData = function(userId, cb) {
dbUtil.findNameByUserId(userId, function(err, userFullName){
if (err) {
cb(err);
return;
}
dbUtil.findUserClassesByUserId(userId, function(err, classes){
if (err) {
cb(err);
return;
}
var pageData = {
userFullName: userFullName,
classes: classes
};
cb(pageData);
});
});
}

How to create callback post saving an array of objects?

Assuming I have the following in a function:
exports.addnames = function(req, res) {
var names = ["Kelley", "Amy", "Mark"];
for(var i = 0; i < names.length; i++) {
(function (name_now) {
Person.findOne({ name: name_now},
function(err, doc) {
if(!err && !doc) {
var personDoc = new PersonDoc();
personDoc.name = name_now;
console.log(personDoc.name);
personDoc.save(function(err) {});
} else if(!err) {
console.log("Person is in the system");
} else {
console.log("ERROR: " + err);
}
}
);
)(names[i]);
}
My issue is after I save the names, I want to return the results:
Person.find({}, function(err, doc) {
res.json(200, doc);
})
Though I have a callback for names, it appears that the last block of code (Persons.find({})) gets executed before the calls to save all the names is complete... thusly when the user goes to the url in the browser, "doc" is empty... Is there some way I can ensure that the Persons.find({}) is called after the for loop completes?
The easiest way to do things like this is to use an async library like the aptly named async which can be found at https://github.com/caolan/async.
If you have a list of names that you want to save and then return when complete, it would look like:
// save each of the names asynchronously
async.forEach(names, function(name, done) {
Person.findOne({name: name},
function(err, doc) {
// return immediately if there was an error
if(err) return done(err);
// save the person if it doesn't already exist
if(!doc) {
var personDoc = new PersonDoc();
personDoc.name = name;
console.log(personDoc.name);
// the async call is complete after the save completes
return personDoc.save(done);
}
// or if the name is already there, just return successfully
console.log("Person is in the system");
done();
}
);
},
// this function is called after all of the names have been saved
// or as soon as an error occurs
function(err) {
if(err) return console.log('ERROR: ' + err);
Person.find({}, function(err, doc) {
res.json(200, doc);
})
});

Resources