I am working through smashing node.js - the first book example shows a function that brings in all the current directory files into a list, then file(i) is run, without it being called. I don't know why? - the input parameter to the function is used within the program and incremented as well, but where does this value come from? How is this function called in the first place?
/**
* Module dependencies.
*/
var fs = require('fs')
, stdin = process.stdin
, stdout = process.stdout
/**
* Read the current directory.
*/
fs.readdir(__dirname, function (err, files) {
console.log('');
if (!files.length) {
return console.log(' \033[31m No files to show!\033[39m\n');
}
console.log(' Select which file or directory you want to see\n');
// called for each file walked in the directory
var stats = {};
function file(i) {
var filename = files[i];
fs.stat(__dirname + '/' + filename, function (err, stat) {
stats[i] = stat;
if (stat.isDirectory()) {
console.log(' '+i+' \033[36m' + filename + '/\033[39m');
} else {
console.log(' '+i+' \033[90m' + filename + '\033[39m');
}
if (++i == files.length) {
read();
} else {
file(i);
}
});
}
// read user input when files are shown
function read () {
console.log('');
stdout.write(' \033[33mEnter your choice: \033[39m');
stdin.resume();
stdin.setEncoding('utf8');
stdin.on('data', option);
}
// called with the option supplied by the user
function option (data) {
var filename = files[Number(data)];
if (!filename) {
stdout.write(' \033[31mEnter your choice: \033[39m');
} else {
stdin.pause();
if (stats[Number(data)].isDirectory()) {
fs.readdir(__dirname + '/' + filename, function (err, files) {
console.log('');
console.log(' (' + files.length + ' files)');
files.forEach(function (file) {
console.log(' - ' + file);
});
console.log('');
});
} else {
fs.readFile(__dirname + '/' + filename, 'utf8', function (err, data) {
console.log('');
console.log('\033[90m' + data.replace(/(.*)/g, ' $1') + '\033[39m');
});
}
}
}
// start by walking the first file
file(0);
});
I must be misunderstanding your question... cuz it seems to straightforward.
The line at the bottom:
// start by walking the first file
file(0);
Is the line the "kicks-off" the file(i) chain. That line is at the bottom of the callback of the fs.readdir and is called when it gets to that point after defining the other functions and variables in the main body of the fs.readdir callback.
Am I missing something to your question?
Related
So I have the following code
var processed;
fs.readFile(path, 'utf-8', function(err, data) {
processed = false;
//checking if text is in file and setting flag
processed = true;
});
if (processed == true) {
try {
var fname = path.substring(path.lastIndexOf("\\") + 1);
fs.moveSync(path, './processedxml/' + fname, {
overwrite: true
})
} catch (err) {
console.log("Error while moving file to processed folder " + err);
}
}
But I don't get the desired output. Because looks like the readfile is executed by a separate thread and so the value of "processed" is not reliable.
I am not very familiar with nodejs so any help will be greatly appreciated.
Yes, you are right, your executions are performed by different threads.
In this scenario, you'll need to use promises.
You can solve your need easily by using "Promise FS" (you can use any other promise solution anyway).
Your code would be something like the following:
fs = require('promise-fs');
var fname = 'test.txt' ;
var toMove = false ;
fs.readFile('test.txt','utf8')
.then (function (content) {
if(content.indexOf('is VALID') !== -1) {
console.log('pattern found!');
toMove = true ;
}
else { toMove = false
}
return toMove ;
}).
then (function (toMove) {
if(toMove) {
var oldPath = 'test.txt'
var newPath = '/tmp/moved/file.txt'
fs.rename(oldPath, newPath, function (err) {
if (err) throw err
console.log('Successfully renamed - moved!')
}) ;
}
})
.catch (function (err) {
console.log(err);
})
Create a file "test.txt" and add the following contents:
this is text.file contents
token is VALID
The code above will evaluate if "is VALID" is present as content and if it does then it will move the file "test.txt" from your current folder to a new one called "moved" in "/tmp" directory. It will also rename the file as "file.txt" file name.
Hope it helps you.
Regards
It looks like you're shadowing path, trying to use it as a variable and as a node module. The easiest way to make this work is to choose a different variable name for the file and move the processing logic into the callback of fs.readFile.
var path = require('path');
var fs = require('fs-extra');
var file = 'some/file/path/foo.xml';
var text = 'search text';
fs.readFile(file, 'utf-8', function (err, data) {
if (err) {
console.error(err);
} else {
//checking if text is in file and setting flag
if (data.indexOf(text) > -1) {
try {
var fname = path.basename(file);
fs.moveSync(file, './processedxml/' + fname, {
overwrite: true
})
} catch (err) {
console.log("Error while moving file to processed folder " + err);
}
}
}
});
I have an array with some IDs like a = [abc,cde,efg]. I pass this array and a second one containing the names with fit to the IDs to a child process in Node.
I want to wait until the child process finished and processes the next array member after that.
The code to achieve it is (with the suggested Edit):
function downloadSentinel(promObj) {
return new Promise((resolve,reject) => {
function makeRequest(url, i, callback) {
url = promObj.ID;
name = promObj.Name;
var sys = require('util'),
exec = require('child_process').exec,
child;
var directory = __dirname.substring(0, __dirname.indexOf("\\app_api"));
console.log(directory);
child = exec(directory + '\\downloadProducts.sh' + promObj.ID[i] + ' ' + promObj.Name[i], function (error, stdout, stderr) {
child.on("error", function (error) {
console.log(error);
})
child.stdout.on('data', function (data) {
console.log(data.toString());
});
child.on('exit', function(exit) {
console.log(exit);
callback();
})
})
}
async.eachOfLimit(promObj.requestURLS, 2, makeRequest, function (err) {
if (err) reject(promObj)
else {
resolve(promObj);
}
});
});
}
I am using npm-async to control the concurrency flow because I want to limit the curl requests I do inside the shell script. The shell sript works without an error. Now the script is only called twice because of the async.eachOfLimit limit. Then the other array IDs are not processed anymore.
EDIT
This is the code I tried at last, but then all possible urls are evoked instead of only 2. I found this here Another Stackoverflow question. I also tried with async.timesLimit
function downloadSentinel(promObj,req,res) {
async.eachOfLimit(promObj.requestURLS, 2,function (value,i,callback) {
console.log('I am here ' + i + 'times');
url = promObj.requestURLS;
name = promObj.Name;
console.log(promObj.requestURLS);
var sys = require('util'),
exec = require('child_process').exec,
child;
var directory = __dirname.substring(0, __dirname.indexOf("\\app_api"));
console.log(directory);
console.log("executing:", directory + '\\downloadProducts.sh ' + promObj.requestURLS[i] + ' ' + promObj.Name[i]);
child = exec(directory + '\\downloadProducts.sh' + ' ' + promObj.requestURLS[i] + ' ' + promObj.Name[i], function (error, stdout, stderr) {
if (error != null){
console.log(error);
}
// this task is resolved
return callback(error, stdout);
});
}, function (err) {
if (err) console.log('error')
else {
console.log('Done');
}
});
}
What have I missed? Thanks in advance.
exec is asynchronous, it does not wait for the launched process to finish.
Move the call to callback() inside the child.on('exit' ...) handler or use execSync
I have created a function to populate a list of objects but they are not being written.
if (stats.isDirectory()) {
objlist=[];
//do something here.. the path was correct
fs.readdir(path, (err, files) => {
objlist=[];
for(file in files){
//console.log(files[file])
objlist[file]={};
fs.stat(path + '\\' + files[file], function(err,stats){
if(err)
throw err;
if(stats.isDirectory()){
objlist[file]['file'] = 'folder'
}
if(stats.isFile()){
objlist[file]['file'] = 'file'
}
objlist[file]['name'] = files[file]
objlist[file]['size'] = stats.size
//console.log(objlist)
//console.log(stats)
});
}
});
console.log(objlist);
return objlist;
}
However the function returns an empty objlist; Can you please suggest what I am doing wrong
this code will be helphul. use let file in files
objlist=[];
//do something here.. the path was correct
fs.readdir(path, (err, files) => {
objlist=[];
for(let file in files){
objlist[file]={};
fs.stat(path + '\\' + files[file], function(err,stats){
if(err)
throw err;
if(stats.isDirectory()){
objlist[file]['file'] = 'folder'
}
if(stats.isFile()){
objlist[file]['file'] = 'file'
}
objlist[file]['name'] = files[file]
objlist[file]['size'] = stats.size
if(file == files.length-1) {
console.log(objlist);
}
});
}
});
You must add the objects to the array with the method push, like this:
objlist=[];
for(file in files){
obj = {};
obj.file = file;
//... do all stuff with obj
objlist.push(obj);
}
Also your function is asynchronous inside the fs.readdir, so if you have to build and populate in the function or do an await to receive the result.
Otherwise you will call the function, it will be executed asynchronously, you will continue at the main function without the wait of the response of the fs.readdir and the array will continue empty because you async function may or may not have been executed.
There are two mistakes in your code:-
1) You are using an array as an object
2) Since for loop is synchronous loop and you are trying asynchronous task inside it and hence either use synchronous version of fs or change your for loop as follows:-
for(file in files) {
obj = {};
obj[file]={};
(function(f){
fs.stat(path + '\\' + files[f], function(err,stats){
if(err)
throw err;
if(stats.isDirectory()){
obj[file]['file'] = 'folder'
}
if(stats.isFile()){
obj[file]['file'] = 'file'
}
obj[file]['name'] = files[file]
obj[file]['size'] = stats.size
objlist.push(obj)
//console.log(objlist)
//console.log(stats)
});
}
}(file))
}
I am using async.each to check through an array of 180 filenames, using fs.stat to check that the physical file exists. At present, all the files do exist, and yet every time I call this code, 3 of the 180 images are found by fs.stat to be non-existent. The images which it fails to find vary slightly on each call.
The first image which fails to be found is always around the 70th filename in the array. If I reduce the size of the array to a list of (for example) 50 names, then all the images are found.
// ImagesToCheck is an array of 180 filepaths, all of which exist on the server
var index = 0
async.each(ImagesToCheck,
function(item, cb) {
fs.stat(item, function(error, value) {
if (error) {
// Logs "ENOENT: no such file or directory"
console.log("error is "+error)
// this is logging the correct filename, which DOES exist
console.log("filename is "+ImagesToCheck[index])
Speakers[index].speaker_image_url = null
index++
cb()
}
else {
// Image exists
// console.log("image exists " + index)
Speakers[index].speaker_image_url = "http://www.apiurl.co.uk/images/seminar_image-" + rows[index].seminar_id + ".jpg"
index++
cb()
}
}) // close fs.stat
},
function(err) {
// sends the array of URLs
callBack(Speakers)
})
// Now using fs.open instead of fs.stat, with async.forEachOf, and getting basically the same result:
var index = 0
async.forEachOf(ImagesToCheck,
function(item, key, cb) {
fs.open(item, 'r', function(error, value) {
if (error) {
// Logs "ENOENT: no such file or directory"
console.log("error is " + error)
// this is logging the correct filename, which DOES exist
console.log("index is " + index + ", filename is " + ImagesToCheck[index])
Speakers[index].speaker_image_url = null
index++
return cb()
}
// if no error
Speakers[index].speaker_image_url = "http://www.apiurl.co.uk/images/seminar_image-" + rows[index].seminar_id + ".jpg"
index++
cb()
})
},
function(err) {
// send the array of urls
callBack(Speakers)
}
)
Note: This is just to show a test implementation. This is not intended as a answer
Just tried to generate 200 files (0.txt, 1.txt... 200.txt) in a folder named files, and tried this code:
'use strict';
const fs = require('fs'),
async = require('async');
let files = [],
results = [],
i = 0;
while(i < 200) {
files.push('./files/' + i + '.txt');
i = i + 1;
}
async.eachOf(files, (file, i, callback) => {
fs.stat(file, (err, stats) => {
files[i] = !err;
callback();
});
}, () => {
let notFound = files.filter((f) => {
return !f;
});
console.log('fs.stat didn\'t found ' + notFound.length + '/200 files');
files.map((f, i) => {
if(!f) {
console.log('File ' + i + ' wasn\'t found');
}
})
});
No error found, it seems the problem is not fs.stat
I'm wondering what is the proper way to ensure that all folder in a path exist before to write a new file.
In the following example, the code fails because the folder cache doesn't exists.
fs.writeFile(__config.path.base + '.tmp/cache/shared.cache', new Date().getTime(), function(err) {
if (err){
consoleDev('Unable to write the new cache timestamp in ' + __filename + ' at ' + __line, 'error');
}else{
consoleDev('Cache process done!');
}
callback ? callback() : '';
});
Solution:
// Ensure the path exists with mkdirp, if it doesn't, create all missing folders.
mkdirp(path.resolve(__config.path.base, path.dirname(__config.cache.lastCacheTimestampFile)), function(err){
if (err){
consoleDev('Unable to create the directories "' + __config.path.base + '" in' + __filename + ' at ' + __line + '\n' + err.message, 'error');
}else{
fs.writeFile(__config.path.base + filename, new Date().getTime(), function(err) {
if (err){
consoleDev('Unable to write the new cache timestamp in ' + __filename + ' at ' + __line + '\n' + err.message, 'error');
}else{
consoleDev('Cache process done!');
}
callback ? callback() : '';
});
}
});
Thanks!
Use mkdirp.
If you really want to do it yourself (recursively):
var pathToFile = 'the/file/sits/in/this/dir/myfile.txt';
pathToFile.split('/').slice(0,-1).reduce(function(prev, curr, i) {
if(fs.existsSync(prev) === false) {
fs.mkdirSync(prev);
}
return prev + '/' + curr;
});
You need the slice to omit the file itself.
fs-extra has a function I found useful for doing this.
The code would be something like:
var fs = require('fs-extra');
fs.ensureFile(path, function(err) {
if (!err) {
fs.writeFile(path, ...);
}
});
You can use fs.existsSync to check if your dir exists and create it if not :
if (fs.existsSync(pathToFolder)===false) {
fs.mkdirSync(pathToFolder,0777);
}