Here's the code I use to browse a directory :
var path = 'D:/Syslog/live';
listDir(path);
function listDir (path) {
fs.readdir(path, function(err, files)
{
console.log(files);
for( i = 0; i < files.length; i++)
{
var fullPath = path + "/" + files[i];
fs.stat(fullPath, function(err, stats){
if (stats.isDirectory())
{
listDir(fullPath);
} else
{
console.log(files[i]);
}
});
}
});
}
When I debug and use the variable fullPath it works fine, if I use files[i] (which is declared in the level, i is undefined
As it's an asynchronous loop, i is iterating much faster than the rest of the code. You can use an anonymous closure function to freeze i within the loop.
Have a look at closures.
The popular async library is good for this sort of stuff too.
function listDir(path) {
fs.readdir(path, function(err, files) {
console.log(files);
for (i = 0; i < files.length; i++) {
(function(i) {
var fullPath = path + "/" + files[i];
fs.stat(fullPath, function(err, stats) {
if (stats.isDirectory()) {
listDir(fullPath);
} else {
console.log(files[i]);
}
});
})(i);
}
});
}
If you were using the async library, it'd look similar to this
var async = require('async');
function listDir(path) {
fs.readdir(path, function(err, files) {
console.log(files);
async.each(files, function(file, callback) {
var fullPath = path + "/" + file;
fs.stat(fullPath, function(err, stats) {
if (stats.isDirectory()) {
listDir(fullPath);
} else {
console.log(file);
}
callback();
});
});
});
}
Both tested and working.
Related
I want to read the all (text) files from a specific directory and it's all subdirecoty recursively.. I am able to read the file and append the result to a global variable. but i want to access the variable at the end of all operation. I am trying with promises but i am unable to access it. please help
var file_path = `C:\\Users\\HP\\Desktop\\test_folder`;
const fs = require('fs');
var final_array = [];
let getFolderTree = function(file_path) {
return new Promise(function(resolve, reject) {
fs.readdir(file_path, function(err, folders) {
if (err) {
console.log("error reading folder :: " + err);
} else {
if (folders.length !== 0) {
for (let i = 0; i < folders.length; i++) {
if (folders[i].endsWith("txt")) {
let text_file_path = file_path + `\\` + folders[i];
fs.readFile(text_file_path, function(error_read, data) {
if (error_read) {
console.log("error reading " + error_read);
} else {
return resolve(final_array.push(data));// want to access final_array at the end of all operations
}
});
} else {
let current_path = file_path + `\\` + folders[i];
getFolderTree(current_path);
}
}
}
}
});
});
}
getFolderTree(file_path).then(function() {
console.log(final_array); // this is not working
});
I think i have found the solution but I am still confused about how it works.
I took reference from another code and able to figure out some how.
var fs = require('fs');
var path = require('path');
let root_path = "C:\\Users\\HP\\Desktop\\test_folder";
function getAllDirectoriesPath(current_path) {
var results = [];
return new Promise(function (resolve, reject) {
fs.readdir(current_path, function (erro, sub_dirs) {
if (erro) {
console.log(error);
} else {
let no_of_subdir = sub_dirs.length;
if (!no_of_subdir) {
return resolve(results);
} else {
sub_dirs.forEach(function (dir) {
dir = path.resolve(current_path, dir);
fs.stat(dir, function (err, stat) {
if (stat && stat.isDirectory()) {
getAllDirectoriesPath(dir).then(function (res) {
results = results.concat(res);
if (!--no_of_subdir) {
resolve(results);
}
});
} else {
fs.readFile(dir, function (err, data) {
results.push(data.toString());
if (!--no_of_subdir) {
resolve(results);
}
});
}
});
});
}
}
});
});
}
getAllDirectoriesPath(root_path).then(function (results) {
console.log(results);
});
I want to make a function to list the file in some directory, and filtered it by extension:
here is my code look like:
var fs = require('fs'),
path = require('path');
var filterList = function (path, ext) {
return fs.readdir(path, function (err, files) {
if (err) {
console.log(err);
} else {
for (var i = 0; i < files.length; i++) {
if (path.extname(files[i]) === '.' + ext) {
return console.log(files[i]);
}
}
}
});
};
filterList(process.argv[2], process.argv[3]);
but it returns this error:
if (path.extname(files[i]) === '.' + ext) {
^
TypeError: path.extname is not a function
Any idea why?
Change name of arg from path to _path, its conflicting
var fs = require('fs'),
path = require('path');
var filterList = function (_path, ext) {
return fs.readdir(_path, function (err, files) {
if (err) {
console.log(err);
} else {
for (var i = 0; i < files.length; i++) {
if (path.extname(files[i]) === '.' + ext) {
return console.log(files[i]);
}
}
}
});
};
filterList(process.argv[2], process.argv[3]);
I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();
I'm using the npm module node-glob.
This snippet returns recursively all files in the current working directory.
var glob = require('glob');
glob('**/*', function(err, files) {
console.log(files);
});
sample output:
[ 'index.html', 'js', 'js/app.js', 'js/lib.js' ]
I want to exclude index.html and js/lib.js.
I tried to exclude these files with negative pattern '!' but without luck.
Is there a way to achieve this only by using a pattern?
I suppose it's not actual anymore but I got stuck with the same question and found an answer.
This can be done using only glob module.
We need to use options as a second parameter to glob function
glob('pattern', {options}, cb)
There is an options.ignore pattern for your needs.
var glob = require('glob');
glob("**/*",{"ignore":['index.html', 'js', 'js/app.js', 'js/lib.js']}, function (err, files) {
console.log(files);
})
Check out globby, which is pretty much glob with support for multiple patterns and a Promise API:
const globby = require('globby');
globby(['**/*', '!index.html', '!js/lib.js']).then(paths => {
console.log(paths);
});
You can use node-globule for that:
var globule = require('globule');
var result = globule.find(['**/*', '!index.html', '!js/lib.js']);
console.log(result);
Or without an external dependency:
/**
Walk directory,
list tree without regex excludes
*/
var fs = require('fs');
var path = require('path');
var walk = function (dir, regExcludes, done) {
var results = [];
fs.readdir(dir, function (err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function (file) {
file = path.join(dir, file);
var excluded = false;
var len = regExcludes.length;
var i = 0;
for (; i < len; i++) {
if (file.match(regExcludes[i])) {
excluded = true;
}
}
// Add if not in regExcludes
if(excluded === false) {
results.push(file);
// Check if its a folder
fs.stat(file, function (err, stat) {
if (stat && stat.isDirectory()) {
// If it is, walk again
walk(file, regExcludes, function (err, res) {
results = results.concat(res);
if (!--pending) { done(null, results); }
});
} else {
if (!--pending) { done(null, results); }
}
});
} else {
if (!--pending) { done(null, results); }
}
});
});
};
var regExcludes = [/index\.html/, /js\/lib\.js/, /node_modules/];
walk('.', regExcludes, function(err, results) {
if (err) {
throw err;
}
console.log(results);
});
Here is what I wrote for my project:
var glob = require('glob');
var minimatch = require("minimatch");
function globArray(patterns, options) {
var i, list = [];
if (!Array.isArray(patterns)) {
patterns = [patterns];
}
patterns.forEach(pattern => {
if (pattern[0] === "!") {
i = list.length-1;
while( i > -1) {
if (!minimatch(list[i], pattern)) {
list.splice(i,1);
}
i--;
}
}
else {
var newList = glob.sync(pattern, options);
newList.forEach(item => {
if (list.indexOf(item)===-1) {
list.push(item);
}
});
}
});
return list;
}
And call it like this (Using an array):
var paths = globArray(["**/*.css","**/*.js","!**/one.js"], {cwd: srcPath});
or this (Using a single string):
var paths = globArray("**/*.js", {cwd: srcPath});
A samples example with gulp:
gulp.task('task_scripts', function(done){
glob("./assets/**/*.js", function (er, files) {
gulp.src(files)
.pipe(gulp.dest('./public/js/'))
.on('end', done);
});
});
How to get totalsize of files in directory ? Best way ?
Here is a simple solution using the core Nodejs fs libraries combined with the async library. It is fully asynchronous and should work just like the 'du' command.
var fs = require('fs'),
path = require('path'),
async = require('async');
function readSizeRecursive(item, cb) {
fs.lstat(item, function(err, stats) {
if (!err && stats.isDirectory()) {
var total = stats.size;
fs.readdir(item, function(err, list) {
if (err) return cb(err);
async.forEach(
list,
function(diritem, callback) {
readSizeRecursive(path.join(item, diritem), function(err, size) {
total += size;
callback(err);
});
},
function(err) {
cb(err, total);
}
);
});
}
else {
cb(err);
}
});
}
I tested the following code and it works perfectly fine.
Please do let me know if there is anything that you don't understand.
var util = require('util'),
spawn = require('child_process').spawn,
size = spawn('du', ['-sh', '/path/to/dir']);
size.stdout.on('data', function (data) {
console.log('size: ' + data);
});
// --- Everything below is optional ---
size.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
size.on('exit', function (code) {
console.log('child process exited with code ' + code);
});
Courtesy Link
2nd method:
var util = require('util'), exec = require('child_process').exec, child;
child = exec('du -sh /path/to/dir', function(error, stdout, stderr){
console.log('stderr: ' + stderr);
if (error !== null){
console.log('exec error: ' + error);
}
});
You might want to refer the Node.js API for child_process
Use du : https://www.npmjs.org/package/du
require('du')('/home/rvagg/.npm/', function (err, size) {
console.log('The size of /home/rvagg/.npm/ is:', size, 'bytes')
})
ES6 variant:
import path_module from 'path'
import fs from 'fs'
// computes a size of a filesystem folder (or a file)
export function fs_size(path, callback)
{
fs.lstat(path, function(error, stats)
{
if (error)
{
return callback(error)
}
if (!stats.isDirectory())
{
return callback(undefined, stats.size)
}
let total = stats.size
fs.readdir(path, function(error, names)
{
if (error)
{
return callback(error)
}
let left = names.length
if (left === 0)
{
return callback(undefined, total)
}
function done(size)
{
total += size
left--
if (left === 0)
{
callback(undefined, total)
}
}
for (let name of names)
{
fs_size(path_module.join(path, name), function(error, size)
{
if (error)
{
return callback(error)
}
done(size)
})
}
})
})
}
Review the node.js File System functions. It looks like you can use a combination of fs.readdir(path, [cb]), and fs.stat(file, [cb]) to list the files in a directory and sum their sizes.
Something like this (totally untested):
var fs = require('fs');
fs.readdir('/path/to/dir', function(err, files) {
var i, totalSizeBytes=0;
if (err) throw err;
for (i=0; i<files.length; i++) {
fs.stat(files[i], function(err, stats) {
if (err) { throw err; }
if (stats.isFile()) { totalSizeBytes += stats.size; }
});
}
});
// Figure out how to wait for all callbacks to complete
// e.g. by using a countdown latch, and yield total size
// via a callback.
Note that this solution only considers the plain files stored directly in the target directory and performs no recursion. A recursive solution would come naturally by checking stats.isDirectory() and entering, although it likely complicates the "wait for completion" step.
'use strict';
const async = require('async');
const fs = require('fs');
const path = require('path')
const getSize = (item, callback) => {
let totalSize = 0;
fs.lstat(item, (err, stats) => {
if (err) return callback(err);
if (stats.isDirectory()) {
fs.readdir(item, (err, list) => {
if (err) return callback(err);
async.each(list, (listItem, cb) => {
getSize(path.join(item, listItem), (err, size) => {
totalSize += size;
cb();
});
},
(err) => {
if (err) return callback(err);
callback(null, totalSize);
});
});
} else {
// Ensure fully asynchronous API
process.nextTick(function() {
callback(null, (totalSize += stats.size))
});
}
});
}
getSize('/Applications', (err, totalSize) => { if (!err) console.log(totalSize); });
I know I'm a bit late to the part but I though I'd include my solution which uses promises based on #maerics answer:
const fs = require('fs');
const Promise = require('bluebird');
var totalSizeBytes=0;
fs.readdir('storage', function(err, files) {
if (err) throw err;
Promise.mapSeries(files, function(file){
return new Promise((resolve, reject) => {
fs.stat('storage/' + file,function(err, stats) {
if (err) { throw err; }
if (stats.isFile()) { totalSizeBytes += stats.size; resolve(); }
});
})
}).then(()=>{
console.log(totalSizeBytes);
});
});
function readSizeRecursive(folder, nested = 0) {
return new Promise(function(resolve, reject) {
const stats = fs.lstatSync(path.resolve(__dirname, '../projects/', folder));
var total = stats.size;
const list = fs.readdirSync(path.resolve(__dirname, '../projects/', folder));
if(list.length > 0){
Promise.all(list.map(async li => {
const stat = await fs.lstatSync(path.resolve(__dirname, '../projects/', folder, li));
if(stat.isDirectory() && nested == 0){
const tt = await readSizeRecursive(folder, 1);
total += tt;
} else {
total += stat.size;
}
})).then(() => resolve(convertBytes(total)));
} else {
resolve(convertBytes(total));
}
});
}
const convertBytes = function(bytes) {
const sizes = ["Bytes", "KB", "MB", "GB", "TB"]
if (bytes == 0) {
return "n/a"
}
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)))
if (i == 0) {
return bytes + " " + sizes[i]
}
// return (bytes / Math.pow(1024, i)).toFixed(1) + " " + sizes[i]
return parseFloat((bytes / Math.pow(1024, i)).toFixed(1));
}
This combines async/await and the fs Promises API introduced in Node.js v14.0.0 for a clean, readable implementation:
const { readdir, stat } = require('fs/promises');
const dirSize = async directory => {
const files = await readdir( directory );
const stats = files.map( file => stat( path.join( directory, file ) ) );
let size = 0;
for await ( const stat of stats ) size += stat.size;
return size;
};
Usage:
const size = await dirSize( '/path/to/directory' );
console.log( size );
An shorter-but-less-readable alternative of the dirSize function would be:
const dirSize = async directory => {
const files = await readdir( directory );
const stats = files.map( file => stat( path.join( directory, file ) ) );
return ( await Promise.all( stats ) ).reduce( ( accumulator, { size } ) => accumulator + size, 0 );
}
A very simple synchronous solution that I implemented.
const fs = require("fs");
function getSize(path){
// Get the size of a file or folder recursively
let size = 0;
if(fs.statSync(path).isDirectory()){
const files = fs.readdirSync(path);
files.forEach(file => {
size += getSize(path + "/" + file);
});
}
else{
size += fs.statSync(path).size;
}
return size;
}