I wrote a function to copy a directory to another... But there's a problem : I use callback function to send the copied size. This callback comes too early (before the end of the copy). I think the problem is that the process is asynchronous. Can you help me?
var fs=require('fs');
var copyDir=function copyDir(from, to, callback){
if(!fs.existsSync(to)){
fs.mkdirSync(to);
}
console.log(from+" ==> "+to);
var count = 0;
fs.readdir(from, function(err,files){
for(var i=0;i<files.length;i++){
var f = from+"/"+files[i];
var d = f.replace(from, to);
console.log(f+" ("+i+")"+ " : "+d);
if(!fs.existsSync(d)){
if(!fs.statSync(f).isFile()){
//fs.mkdirSync(f.replace(from, to));
count += fs.statSync(f).size;
console.log(f + " will make an inception!")
copyDir(f, f.replace(from, to), function(err, cp){callback(err, cp)});
}else{
var size = fs.statSync(f).size;
copyFile(f, f.replace(from, to), function(err){
if(err) callback(err, count)
});
count += size;
callback(null, count);
}
}
}
});
}
function copyFile(source, target, cb) {
fs.readFile(source, function (err, data) {
if (err) throw err;
fs.writeFileSync(target, data, function (err, data){
if(err) throw err;
cb(null, fs.statSync(source).size); //This callback comes before the copy end.
});
});
}
exports.copyDir = copyDir;
copyDir is called by:
io.sockets.on('connection', function(socket){
console.log('connection');
socket.on('startCopy', function(data){
sizeDir('templates', function(e, r){
copyDir('templates', 'tmp', function(err, cp){
console.log("copy % " + Math.round(100*cp/r));
socket.emit('copy', {prog: Math.round(100*cp/r)});
});
});
});
});
You can rewrite your else code with following:
(function() {
var size = fs.statSync(f).size;
copyFile(f, f.replace(from, to), function(err){
if(err) {
callback(err, count);
return;
}
count += size;
callback(null, count);
});
})();
But, you have a lot of synchronous function in your code. You should know about all caveats of this approach. This article may be helpful
Related
I am trying to move download images from parse and save it to my local. I have this piece of code that does the job for me. This works well when there is only one request but when I put in a loop, it doesn't hold good.
`for(var i = 0; i < 5; i++) {
console.log(i);//to debug
var filename = results_jsonObj[i].imageFile.name;
var uri = results_jsonObj[i].imageFile.url;
request.head(uri, function(err, res, body){
if (err){
console.log(err);
console.log(item);
return;
}else {
console.log(i); //to debug
var stream = request(uri);
stream.pipe(
fs.createWriteStream("images/"+filename)
.on('error', function(err){
callback(error, filename);
stream.read();
})
)
}
});
}`
Irrespective of the loop condition I have, only one image downloads to the mentioned directory.
Below is the op
The input is from a Json file and I have the request, fs, parse module included in the node js program.
Any help on how to go about this?
I have got this fixed now. As advised in the comments it was async which helped me do the trick.
for(var i = 0; i < 900; i++) {
async.forEachOf(results_jsonObj[i], function(value, key, callback){
var image = {};
image.key = key;
image.value = value;
if(image.key == 'imageFile')
{
var filename = image.value.name;
var uri = image.value.url;
// console.log(filename, uri);
}
request.head(uri, function(err, res, body){
if (err){
console.log(err);
// console.log(item);
return;
}else {
// console.log(i,res.headers['content-type']); //to debug
var stream = request(uri);
stream.pipe(
fs.createWriteStream("images/"+filename)
.on('error', function(err){
callback(error, filename);
stream.read();
})
)
}
});
callback();
}, function(err){
if (err) {
console.log('one of the api failed, the whole thing will fail now');
}
});
}
I have problem with mongoosastic it not update Elasticsearch from first update. I update 1 time and nothing happend and then update 2 time and in elasticsearch make update from what I updated 1st time...
Here is my code:
var User = system.mongoose.model('User', userSchema, 'user');
User.createMapping(function (err, mapping) {
if (err) {
console.log("error creating mapping");
console.log(err);
} else {
console.log("Mapping created");
console.log(mapping);
}
});
//{$in: ['doctor','nurse']}
var stream = User.synchronize();
var count = 0;
stream.on('data', function (data) {
count++;
});
stream.on('close', function () {
console.log("Indexed " + count + " documents");
});
stream.on('error', function (err) {
console.log(err);
});
Anyone know what is problem?
It is not clear from your question what you expected, and what you got.
But I am going to guess that you are experiencing a race condition because you started User.synchronize() before User.createMapping() finished.
If so, this might work better:
var User = system.mongoose.model('User', userSchema, 'user');
User.createMapping(function (err, mapping) {
if (err) {
console.log("error creating mapping");
console.log(err);
} else {
console.log("Mapping created");
console.log(mapping);
doTheNextThing();
}
});
function doTheNextThing () {
//{$in: ['doctor','nurse']}
var stream = User.synchronize();
var count = 0;
stream.on('data', function (data) {
count++;
});
stream.on('close', function () {
console.log("Indexed " + count + " documents");
});
stream.on('error', function (err) {
console.log(err);
});
}
I would like to do something like this
function scan(apath){
var files = fs.readdirSync(apath);
for(var i=0; i<files.length;i++){
var stats = fs.statSync(path.join(apath,files[i]))
if(stats.isDirectory()){
results.push(path.join(apath, files[i]))
scan(path.join(apath,files[i]))
}
if(stats.isFile()){
results.push(path.join(apath,files[i]))
}
}
}
but asynchronously.
Trying this with asynchronous functions led me to a nightmare with something like this.
function scan(apath){
fs.readdir(apath, function(err, files)){
var counter = files.length;
files.forEach(function(file){
var newpath = path.join(apath, file)
fs.stat(newpath, function(err, stat){
if(err) return callback(err)
if(stat.isFile())
results.push(newpath)
if(stat.isDirectory()){
results.push(newpath)
scan(newpath)
}
if(--counter <=0) return
})
})
}
}
All hell breaks loose in node's stack because things don't happen in logical succession as they do in synchronous methods.
you can try async module, and use like this:
function scan(apath, callback) {
fs.readdir(apath, function(err, files) {
var counter = 0;
async.whilst(
function() {
return counter < files.length;
},
function(cb) {
var file = files[counter++];
var newpath = path.join(apath, file);
fs.stat(newpath, function(err, stat) {
if (err) return cb(err);
if (stat.isFile()) {
results.push(newpath);
cb(); // asynchronously call the loop
}
if (stat.isDirectory()) {
results.push(newpath);
scan(newpath, cb); // recursion loop
}
});
},
function(err) {
callback(err); // loop over, come out
}
);
});
}
look for more about async.whilst
I'm making a function that will check if two files contain a given string. If both files DON'T contain the string, it should return undefined :(
here is my code:
var fs = require("fs");
function get_uniq(string, file1, file2, callback){
fs.readFile(file1, 'utf8', function(err, data1) {
if (err) throw err;
i = data1.search(string);
console.log(i);
if(i == -1){
fs.readFile(file2, 'utf8', function(err, data2) {
if (err) throw err;
j = data2.search(string);
if(j == -1){
return 1;
}
});
}
});
callback();
}
var i = get_uniq("stringThatFilesDoesntContainin", "somefile.txt", "anotherfile.txt", function(){
console.log(i);
});
Any idea what the problem is?
You should not rely on returning a computed value. In node functions can be executed asynchronously so it can return before the function can finish. To execute when the function completes a callback is given. For e.g.
fs.readFile(file1, 'utf8', function(err, data1) {...});
The function that is passed as the last argument is the callback. It is executed when the file has been read. Trying to return the data will result in undefined value.
In your case the returned values will be undefined for all cases. And callback will be executed in parallel with readFile.
callback must be called from inside readFile for file1 or file2, wherever it can finish logically. To give all the places where callback can be added are :
function get_uniq(string, file1, file2, callback){
fs.readFile(file1, 'utf8', function(err, data1) {
if (err)
{
throw err;
callback(err);
}
else
{
i = data1.search(string);
console.log(i);
if(i == -1){
fs.readFile(file2, 'utf8', function(err, data2) {
if (err)
{
throw err;
callback(err);
}
else
{
j = data2.search(string);
if(j == -1){
callback(false);
}
else
callback(true);
}
});
}
else
callback(false);
}
});
}
You can put you return value (true/false) as the argument to callback. Or catch error from inside it. How you will execute the above function will be like :
get_uniq("stringThatFilesDoesntContainin", "somefile.txt", "anotherfile.txt", function(value){
console.log(value);
});
Assuming I have the following in a function:
exports.addnames = function(req, res) {
var names = ["Kelley", "Amy", "Mark"];
for(var i = 0; i < names.length; i++) {
(function (name_now) {
Person.findOne({ name: name_now},
function(err, doc) {
if(!err && !doc) {
var personDoc = new PersonDoc();
personDoc.name = name_now;
console.log(personDoc.name);
personDoc.save(function(err) {});
} else if(!err) {
console.log("Person is in the system");
} else {
console.log("ERROR: " + err);
}
}
);
)(names[i]);
}
My issue is after I save the names, I want to return the results:
Person.find({}, function(err, doc) {
res.json(200, doc);
})
Though I have a callback for names, it appears that the last block of code (Persons.find({})) gets executed before the calls to save all the names is complete... thusly when the user goes to the url in the browser, "doc" is empty... Is there some way I can ensure that the Persons.find({}) is called after the for loop completes?
The easiest way to do things like this is to use an async library like the aptly named async which can be found at https://github.com/caolan/async.
If you have a list of names that you want to save and then return when complete, it would look like:
// save each of the names asynchronously
async.forEach(names, function(name, done) {
Person.findOne({name: name},
function(err, doc) {
// return immediately if there was an error
if(err) return done(err);
// save the person if it doesn't already exist
if(!doc) {
var personDoc = new PersonDoc();
personDoc.name = name;
console.log(personDoc.name);
// the async call is complete after the save completes
return personDoc.save(done);
}
// or if the name is already there, just return successfully
console.log("Person is in the system");
done();
}
);
},
// this function is called after all of the names have been saved
// or as soon as an error occurs
function(err) {
if(err) return console.log('ERROR: ' + err);
Person.find({}, function(err, doc) {
res.json(200, doc);
})
});