node.js glob pattern for excluding multiple files - node.js

I'm using the npm module node-glob.
This snippet returns recursively all files in the current working directory.
var glob = require('glob');
glob('**/*', function(err, files) {
console.log(files);
});
sample output:
[ 'index.html', 'js', 'js/app.js', 'js/lib.js' ]
I want to exclude index.html and js/lib.js.
I tried to exclude these files with negative pattern '!' but without luck.
Is there a way to achieve this only by using a pattern?

I suppose it's not actual anymore but I got stuck with the same question and found an answer.
This can be done using only glob module.
We need to use options as a second parameter to glob function
glob('pattern', {options}, cb)
There is an options.ignore pattern for your needs.
var glob = require('glob');
glob("**/*",{"ignore":['index.html', 'js', 'js/app.js', 'js/lib.js']}, function (err, files) {
console.log(files);
})

Check out globby, which is pretty much glob with support for multiple patterns and a Promise API:
const globby = require('globby');
globby(['**/*', '!index.html', '!js/lib.js']).then(paths => {
console.log(paths);
});

You can use node-globule for that:
var globule = require('globule');
var result = globule.find(['**/*', '!index.html', '!js/lib.js']);
console.log(result);

Or without an external dependency:
/**
Walk directory,
list tree without regex excludes
*/
var fs = require('fs');
var path = require('path');
var walk = function (dir, regExcludes, done) {
var results = [];
fs.readdir(dir, function (err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function (file) {
file = path.join(dir, file);
var excluded = false;
var len = regExcludes.length;
var i = 0;
for (; i < len; i++) {
if (file.match(regExcludes[i])) {
excluded = true;
}
}
// Add if not in regExcludes
if(excluded === false) {
results.push(file);
// Check if its a folder
fs.stat(file, function (err, stat) {
if (stat && stat.isDirectory()) {
// If it is, walk again
walk(file, regExcludes, function (err, res) {
results = results.concat(res);
if (!--pending) { done(null, results); }
});
} else {
if (!--pending) { done(null, results); }
}
});
} else {
if (!--pending) { done(null, results); }
}
});
});
};
var regExcludes = [/index\.html/, /js\/lib\.js/, /node_modules/];
walk('.', regExcludes, function(err, results) {
if (err) {
throw err;
}
console.log(results);
});

Here is what I wrote for my project:
var glob = require('glob');
var minimatch = require("minimatch");
function globArray(patterns, options) {
var i, list = [];
if (!Array.isArray(patterns)) {
patterns = [patterns];
}
patterns.forEach(pattern => {
if (pattern[0] === "!") {
i = list.length-1;
while( i > -1) {
if (!minimatch(list[i], pattern)) {
list.splice(i,1);
}
i--;
}
}
else {
var newList = glob.sync(pattern, options);
newList.forEach(item => {
if (list.indexOf(item)===-1) {
list.push(item);
}
});
}
});
return list;
}
And call it like this (Using an array):
var paths = globArray(["**/*.css","**/*.js","!**/one.js"], {cwd: srcPath});
or this (Using a single string):
var paths = globArray("**/*.js", {cwd: srcPath});

A samples example with gulp:
gulp.task('task_scripts', function(done){
glob("./assets/**/*.js", function (er, files) {
gulp.src(files)
.pipe(gulp.dest('./public/js/'))
.on('end', done);
});
});

Related

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

Node js - Wierd variable scope

Here's the code I use to browse a directory :
var path = 'D:/Syslog/live';
listDir(path);
function listDir (path) {
fs.readdir(path, function(err, files)
{
console.log(files);
for( i = 0; i < files.length; i++)
{
var fullPath = path + "/" + files[i];
fs.stat(fullPath, function(err, stats){
if (stats.isDirectory())
{
listDir(fullPath);
} else
{
console.log(files[i]);
}
});
}
});
}
When I debug and use the variable fullPath it works fine, if I use files[i] (which is declared in the level, i is undefined
As it's an asynchronous loop, i is iterating much faster than the rest of the code. You can use an anonymous closure function to freeze i within the loop.
Have a look at closures.
The popular async library is good for this sort of stuff too.
function listDir(path) {
fs.readdir(path, function(err, files) {
console.log(files);
for (i = 0; i < files.length; i++) {
(function(i) {
var fullPath = path + "/" + files[i];
fs.stat(fullPath, function(err, stats) {
if (stats.isDirectory()) {
listDir(fullPath);
} else {
console.log(files[i]);
}
});
})(i);
}
});
}
If you were using the async library, it'd look similar to this
var async = require('async');
function listDir(path) {
fs.readdir(path, function(err, files) {
console.log(files);
async.each(files, function(file, callback) {
var fullPath = path + "/" + file;
fs.stat(fullPath, function(err, stats) {
if (stats.isDirectory()) {
listDir(fullPath);
} else {
console.log(file);
}
callback();
});
});
});
}
Both tested and working.

Coffeescript --watch switch not working with node 0.4.7

The --watch directive to the coffeescript compiler doesn't work with node.js 0.4.7. How to fix?
(Node 0.4.7 is currently required for deployment on Heroku.)
There is no fs.watch method in node.js 0.4.7. Replace the watch function at line 198 in command.js with the following:
watch = function(source, base) {
return fs.stat(source, function(err, prevStats) {
if (err) throw err;
return fs.watchFile(source, function(curr, prev) {
return fs.stat(source, function(err, stats) {
if (err) throw err;
if (stats.size === prevStats.size && stats.mtime.getTime() === prevStats.mtime.getTime()) {
return;
}
prevStats = stats;
return fs.readFile(source, function(err, code) {
if (err) throw err;
return compileScript(source, code.toString(), base);
});
});
});
});
};
If you can't upgrade to Node 0.6, try Jitter. It's a single-purpose command line utility that does what coffee -cw did before CoffeeScript 1.1.3. Plus it has a few little extras, like Growl notifications when compilation fails.
Here's a watchscript I wrote after having same experience with coffeescript -w option not working. It finds and watches .coffee, .less, .jade files for change and triggers their corresponding compilers in response. Generates colored output for easy detection of when the compilation failed.
/*
* Tested on node 0.6, Ubuntu
*/
#!/usr/bin/node
COLOR_GREEN="\033[01;32m"
COLOR_RED="\033[01;31m"
COLOR_YELLOW="\033[01;33m"
COLOR_BLUE="\033[01;34m"
COLOR_NORMAL="\033[00m"
var fs = require('fs');
var coffeescript, jade, less;
try {
coffeescript = require('/usr/lib/node_modules/coffee-script'); // after npm -g install coffee-script
} catch(e) {
console.log(COLOR_RED,String(e),COLOR_NORMAL);
}
try {
jade = require('/usr/lib/node_modules/jade'); // after npm -g install jade
} catch(e) {
console.log(COLOR_RED,String(e),COLOR_NORMAL);
}
try {
less = require('/usr/lib/node_modules/less'); // after npm -g install less
} catch(e) {
console.log(COLOR_RED,String(e),COLOR_NORMAL);
}
console.log(
'Watch Support',
(coffeescript ? COLOR_GREEN : COLOR_RED),' coffeescript ',COLOR_NORMAL,
(jade ? COLOR_GREEN : COLOR_RED),' jade ',COLOR_NORMAL,
(less ? COLOR_GREEN : COLOR_RED),' less ',COLOR_NORMAL
);
var compile = function (fname) {
var source = fs.readFileSync(fname).toString();
try {
if(/\.jade$/.test(fname)) {
var html = jade.compile(source,{pretty:true})();
var htmlfname = fname.replace(/\.jade$/,'.html');
fs.writeFileSync(htmlfname, html);
console.log(COLOR_GREEN,'Success ',fname,new Date(),COLOR_NORMAL)
} else if(/\.coffee$/.test(fname)) {
var js = coffeescript.compile(source);
var jsfname = fname.replace(/\.coffee$/,'.js');
fs.writeFileSync(jsfname, js);
console.log(COLOR_GREEN,'Success ',fname,new Date(),COLOR_NORMAL)
} else if(/\.less$/.test(fname)) {
var lessparser = new(less.Parser);
var cssfname = fname.replace(/\.less$/,'.css');
lessparser.parse(source, function (e, tree) {
if(e) {
console.log(COLOR_RED,'Failed ',fname,String(e),COLOR_NORMAL);
} else {
fs.writeFileSync(cssfname, tree.toCSS());
console.log(COLOR_GREEN,'Success ',fname,new Date(),COLOR_NORMAL)
}
});
}
} catch(e) {
console.log(COLOR_RED,'Failed ',fname,String(e),COLOR_NORMAL)
}
}
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function(file) {
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
results.push(file);
if (!--pending) done(null, results);
}
});
});
});
};
function main() {
if(process.argv.length > 2) {
dirName = process.argv[2];
} else {
dirName = '.';
}
stats = fs.statSync(dirName);
if(stats.isDirectory()) {
walk(dirName,function (err, files) {
var watching = 0;
files.forEach(function (fname) {
if(/\.coffee$|\.less$|\.jade$/.test(fname)) {
fs.watchFile(fname,{persistent:true,interval:500},
function (cur,prev) {
if(cur.mtime.getTime() != prev.mtime.getTime()) {
compile(fname);
}
}
);
watching++;
}
});
if(watching) {
console.log('Watching '+watching+' file(s) in '+
COLOR_YELLOW+dirName+COLOR_NORMAL);
} else {
console.log('No coffee/jade/less files found to watch');
}
});
} else {
// It's a file
var fname = dirName;
if(/\.coffee$|\.less$|\.jade$/.test(fname)) {
fs.watchFile(fname,{persistent:true,interval:500},
function (cur,prev) {
if(cur.mtime.getTime() != prev.mtime.getTime()) {
compile(fname);
}
}
);
console.log('Watching '+COLOR_YELLOW+fname+COLOR_NORMAL);
} else {
console.log('No coffee/jade/less files found to watch');
}
}
}
main();
If you prefer gist: https://gist.github.com/3083080

How to get totalsize of files in directory?

How to get totalsize of files in directory ? Best way ?
Here is a simple solution using the core Nodejs fs libraries combined with the async library. It is fully asynchronous and should work just like the 'du' command.
var fs = require('fs'),
path = require('path'),
async = require('async');
function readSizeRecursive(item, cb) {
fs.lstat(item, function(err, stats) {
if (!err && stats.isDirectory()) {
var total = stats.size;
fs.readdir(item, function(err, list) {
if (err) return cb(err);
async.forEach(
list,
function(diritem, callback) {
readSizeRecursive(path.join(item, diritem), function(err, size) {
total += size;
callback(err);
});
},
function(err) {
cb(err, total);
}
);
});
}
else {
cb(err);
}
});
}
I tested the following code and it works perfectly fine.
Please do let me know if there is anything that you don't understand.
var util = require('util'),
spawn = require('child_process').spawn,
size = spawn('du', ['-sh', '/path/to/dir']);
size.stdout.on('data', function (data) {
console.log('size: ' + data);
});
// --- Everything below is optional ---
size.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
size.on('exit', function (code) {
console.log('child process exited with code ' + code);
});
Courtesy Link
2nd method:
var util = require('util'), exec = require('child_process').exec, child;
child = exec('du -sh /path/to/dir', function(error, stdout, stderr){
console.log('stderr: ' + stderr);
if (error !== null){
console.log('exec error: ' + error);
}
});
You might want to refer the Node.js API for child_process
Use du : https://www.npmjs.org/package/du
require('du')('/home/rvagg/.npm/', function (err, size) {
console.log('The size of /home/rvagg/.npm/ is:', size, 'bytes')
})
ES6 variant:
import path_module from 'path'
import fs from 'fs'
// computes a size of a filesystem folder (or a file)
export function fs_size(path, callback)
{
fs.lstat(path, function(error, stats)
{
if (error)
{
return callback(error)
}
if (!stats.isDirectory())
{
return callback(undefined, stats.size)
}
let total = stats.size
fs.readdir(path, function(error, names)
{
if (error)
{
return callback(error)
}
let left = names.length
if (left === 0)
{
return callback(undefined, total)
}
function done(size)
{
total += size
left--
if (left === 0)
{
callback(undefined, total)
}
}
for (let name of names)
{
fs_size(path_module.join(path, name), function(error, size)
{
if (error)
{
return callback(error)
}
done(size)
})
}
})
})
}
Review the node.js File System functions. It looks like you can use a combination of fs.readdir(path, [cb]), and fs.stat(file, [cb]) to list the files in a directory and sum their sizes.
Something like this (totally untested):
var fs = require('fs');
fs.readdir('/path/to/dir', function(err, files) {
var i, totalSizeBytes=0;
if (err) throw err;
for (i=0; i<files.length; i++) {
fs.stat(files[i], function(err, stats) {
if (err) { throw err; }
if (stats.isFile()) { totalSizeBytes += stats.size; }
});
}
});
// Figure out how to wait for all callbacks to complete
// e.g. by using a countdown latch, and yield total size
// via a callback.
Note that this solution only considers the plain files stored directly in the target directory and performs no recursion. A recursive solution would come naturally by checking stats.isDirectory() and entering, although it likely complicates the "wait for completion" step.
'use strict';
const async = require('async');
const fs = require('fs');
const path = require('path')
const getSize = (item, callback) => {
let totalSize = 0;
fs.lstat(item, (err, stats) => {
if (err) return callback(err);
if (stats.isDirectory()) {
fs.readdir(item, (err, list) => {
if (err) return callback(err);
async.each(list, (listItem, cb) => {
getSize(path.join(item, listItem), (err, size) => {
totalSize += size;
cb();
});
},
(err) => {
if (err) return callback(err);
callback(null, totalSize);
});
});
} else {
// Ensure fully asynchronous API
process.nextTick(function() {
callback(null, (totalSize += stats.size))
});
}
});
}
getSize('/Applications', (err, totalSize) => { if (!err) console.log(totalSize); });
I know I'm a bit late to the part but I though I'd include my solution which uses promises based on #maerics answer:
const fs = require('fs');
const Promise = require('bluebird');
var totalSizeBytes=0;
fs.readdir('storage', function(err, files) {
if (err) throw err;
Promise.mapSeries(files, function(file){
return new Promise((resolve, reject) => {
fs.stat('storage/' + file,function(err, stats) {
if (err) { throw err; }
if (stats.isFile()) { totalSizeBytes += stats.size; resolve(); }
});
})
}).then(()=>{
console.log(totalSizeBytes);
});
});
function readSizeRecursive(folder, nested = 0) {
return new Promise(function(resolve, reject) {
const stats = fs.lstatSync(path.resolve(__dirname, '../projects/', folder));
var total = stats.size;
const list = fs.readdirSync(path.resolve(__dirname, '../projects/', folder));
if(list.length > 0){
Promise.all(list.map(async li => {
const stat = await fs.lstatSync(path.resolve(__dirname, '../projects/', folder, li));
if(stat.isDirectory() && nested == 0){
const tt = await readSizeRecursive(folder, 1);
total += tt;
} else {
total += stat.size;
}
})).then(() => resolve(convertBytes(total)));
} else {
resolve(convertBytes(total));
}
});
}
const convertBytes = function(bytes) {
const sizes = ["Bytes", "KB", "MB", "GB", "TB"]
if (bytes == 0) {
return "n/a"
}
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)))
if (i == 0) {
return bytes + " " + sizes[i]
}
// return (bytes / Math.pow(1024, i)).toFixed(1) + " " + sizes[i]
return parseFloat((bytes / Math.pow(1024, i)).toFixed(1));
}
This combines async/await and the fs Promises API introduced in Node.js v14.0.0 for a clean, readable implementation:
const { readdir, stat } = require('fs/promises');
const dirSize = async directory => {
const files = await readdir( directory );
const stats = files.map( file => stat( path.join( directory, file ) ) );
let size = 0;
for await ( const stat of stats ) size += stat.size;
return size;
};
Usage:
const size = await dirSize( '/path/to/directory' );
console.log( size );
An shorter-but-less-readable alternative of the dirSize function would be:
const dirSize = async directory => {
const files = await readdir( directory );
const stats = files.map( file => stat( path.join( directory, file ) ) );
return ( await Promise.all( stats ) ).reduce( ( accumulator, { size } ) => accumulator + size, 0 );
}
A very simple synchronous solution that I implemented.
const fs = require("fs");
function getSize(path){
// Get the size of a file or folder recursively
let size = 0;
if(fs.statSync(path).isDirectory()){
const files = fs.readdirSync(path);
files.forEach(file => {
size += getSize(path + "/" + file);
});
}
else{
size += fs.statSync(path).size;
}
return size;
}

Node.js fs.readdir recursive directory search

Any ideas on an async directory search using fs.readdir? I realize that we could introduce recursion and call the read directory function with the next directory to read, but I'm a little worried about it not being async...
Any ideas? I've looked at node-walk which is great, but doesn't give me just the files in an array, like readdir does. Although
Looking for output like...
['file1.txt', 'file2.txt', 'dir/file3.txt']
There are basically two ways of accomplishing this. In an async environment you'll notice that there are two kinds of loops: serial and parallel. A serial loop waits for one iteration to complete before it moves onto the next iteration - this guarantees that every iteration of the loop completes in order. In a parallel loop, all the iterations are started at the same time, and one may complete before another, however, it is much faster than a serial loop. So in this case, it's probably better to use a parallel loop because it doesn't matter what order the walk completes in, just as long as it completes and returns the results (unless you want them in order).
A parallel loop would look like this:
var fs = require('fs');
var path = require('path');
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function(file) {
file = path.resolve(dir, file);
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
if (!--pending) done(null, results);
});
} else {
results.push(file);
if (!--pending) done(null, results);
}
});
});
});
};
A serial loop would look like this:
var fs = require('fs');
var path = require('path');
var walk = function(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = path.resolve(dir, file);
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walk(file, function(err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
};
And to test it out on your home directory (WARNING: the results list will be huge if you have a lot of stuff in your home directory):
walk(process.env.HOME, function(err, results) {
if (err) throw err;
console.log(results);
});
EDIT: Improved examples.
This one uses the maximum amount of new, buzzwordy features available in node 8, including Promises, util/promisify, destructuring, async-await, map+reduce and more, making your co-workers scratch their heads as they try to figure out what is going on.
Node 8+
No external dependencies.
const { promisify } = require('util');
const { resolve } = require('path');
const fs = require('fs');
const readdir = promisify(fs.readdir);
const stat = promisify(fs.stat);
async function getFiles(dir) {
const subdirs = await readdir(dir);
const files = await Promise.all(subdirs.map(async (subdir) => {
const res = resolve(dir, subdir);
return (await stat(res)).isDirectory() ? getFiles(res) : res;
}));
return files.reduce((a, f) => a.concat(f), []);
}
Usage
getFiles(__dirname)
.then(files => console.log(files))
.catch(e => console.error(e));
Node 10.10+
Updated for node 10+ with even more whizbang:
const { resolve } = require('path');
const { readdir } = require('fs').promises;
async function getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
const files = await Promise.all(dirents.map((dirent) => {
const res = resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
}));
return Array.prototype.concat(...files);
}
Note that starting with node 11.15.0 you can use files.flat() instead of Array.prototype.concat(...files) to flatten the files array.
Node 11+
If you want to blow everybody's head up completely, you can use the following version using async iterators. In addition to being really cool, it also allows consumers to pull results one-at-a-time, making it better suited for really large directories.
const { resolve } = require('path');
const { readdir } = require('fs').promises;
async function* getFiles(dir) {
const dirents = await readdir(dir, { withFileTypes: true });
for (const dirent of dirents) {
const res = resolve(dir, dirent.name);
if (dirent.isDirectory()) {
yield* getFiles(res);
} else {
yield res;
}
}
}
Usage has changed because the return type is now an async iterator instead of a promise
;(async () => {
for await (const f of getFiles('.')) {
console.log(f);
}
})()
In case somebody is interested, I've written more about async iterators here: https://qwtel.com/posts/software/async-generators-in-the-wild/
Just in case anyone finds it useful, I also put together a synchronous version.
var walk = function(dir) {
var results = [];
var list = fs.readdirSync(dir);
list.forEach(function(file) {
file = dir + '/' + file;
var stat = fs.statSync(file);
if (stat && stat.isDirectory()) {
/* Recurse into a subdirectory */
results = results.concat(walk(file));
} else {
/* Is a file */
results.push(file);
}
});
return results;
}
Tip: To use less resources when filtering. Filter within this function itself. E.g. Replace results.push(file); with below code. Adjust as required:
file_type = file.split(".").pop();
file_name = file.split(/(\\|\/)/g).pop();
if (file_type == "json") results.push(file);
A. Have a look at the file module. It has a function called walk:
file.walk(start, callback)
Navigates a file tree, calling callback for each directory, passing in
(null, dirPath, dirs, files).
This may be for you! And yes, it is async. However, I think you would have to aggregate the full path's yourself, if you needed them.
B. An alternative, and even one of my favourites: use the unix find for that. Why do something again, that has already been programmed? Maybe not exactly what you need, but still worth checking out:
var execFile = require('child_process').execFile;
execFile('find', [ 'somepath/' ], function(err, stdout, stderr) {
var file_list = stdout.split('\n');
/* now you've got a list with full path file names */
});
Find has a nice build-in caching mechanism that makes subsequent searches very fast, as long as only few folder have changed.
I recommend using node-glob to accomplish that task.
var glob = require( 'glob' );
glob( 'dirname/**/*.js', function( err, files ) {
console.log( files );
});
Another nice npm package is glob.
npm install glob
It is very powerful and should cover all your recursing needs.
Edit:
I actually wasn't perfectly happy with glob, so I created readdirp.
I'm very confident that its API makes finding files and directories recursively and applying specific filters very easy.
Read through its documentation to get a better idea of what it does and install via:
npm install readdirp
Short, Modern and Efficient:
import {readdir} from 'node:fs/promises'
import {join} from 'node:path'
const deepReadDir = async (dirPath) => await Promise.all(
(await readdir(dirPath, {withFileTypes: true})).map(async (dirent) => {
const path = join(dirPath, dirent.name)
return dirent.isDirectory() ? await deepReadDir(path) : path
}),
)
Special thank to Function for hinting use of {withFileTypes: true}.
This auto folds each nested path into a new nested array. For example if:
await deepReadDir('src')
returns something like this:
[
[
'src/client/api.js',
'src/client/http-constants.js',
'src/client/index.html',
'src/client/index.js',
[ 'src/client/res/favicon.ico' ],
'src/client/storage.js'
],
[ 'src/crypto/keygen.js' ],
'src/discover.js',
[
'src/mutations/createNewMutation.js',
'src/mutations/newAccount.js',
'src/mutations/transferCredit.js',
'src/mutations/updateApp.js'
],
[
'src/server/authentication.js',
'src/server/handlers.js',
'src/server/quick-response.js',
'src/server/server.js',
'src/server/static-resources.js'
],
[ 'src/util/prompt.js', 'src/util/safeWriteFile.js' ],
'src/util.js'
]
But you can easily flat it, if you want:
(await deepReadDir('src')).flat(Number.POSITIVE_INFINITY)
[
'src/client/api.js',
'src/client/http-constants.js',
'src/client/index.html',
'src/client/index.js',
'src/client/res/favicon.ico',
'src/client/storage.js',
'src/crypto/keygen.js',
'src/discover.js',
'src/mutations/createNewMutation.js',
'src/mutations/newAccount.js',
'src/mutations/transferCredit.js',
'src/mutations/updateApp.js',
'src/server/authentication.js',
'src/server/handlers.js',
'src/server/quick-response.js',
'src/server/server.js',
'src/server/static-resources.js',
'src/util/prompt.js',
'src/util/safeWriteFile.js',
'src/util.js'
]
If you want to use an npm package, wrench is pretty good.
var wrench = require("wrench");
var files = wrench.readdirSyncRecursive("directory");
wrench.readdirRecursive("directory", function (error, files) {
// live your dreams
});
EDIT (2018):
Anyone reading through in recent time: The author deprecated this package in 2015:
wrench.js is deprecated, and hasn't been updated in quite some time. I heavily recommend using fs-extra to do any extra filesystem operations.
Async
const fs = require('fs')
const path = require('path')
const readdir = (p, done, a = [], i = 0) => fs.readdir(p, (e, d = []) =>
d.map(f => readdir(a[a.push(path.join(p, f)) - 1], () =>
++i == d.length && done(a), a)).length || done(a))
readdir(__dirname, console.log)
Sync
const fs = require('fs')
const path = require('path')
const readdirSync = (p, a = []) => {
if (fs.statSync(p).isDirectory())
fs.readdirSync(p).map(f => readdirSync(a[a.push(path.join(p, f)) - 1], a))
return a
}
console.log(readdirSync(__dirname))
Async readable
function readdir (currentPath, done, allFiles = [], i = 0) {
fs.readdir(currentPath, function (e, directoryFiles = []) {
if (!directoryFiles.length)
return done(allFiles)
directoryFiles.map(function (file) {
var joinedPath = path.join(currentPath, file)
allFiles.push(joinedPath)
readdir(joinedPath, function () {
i = i + 1
if (i == directoryFiles.length)
done(allFiles)}
, allFiles)
})
})
}
readdir(__dirname, console.log)
Note: both versions will follow symlinks (same as the original fs.readdir)
With Recursion
var fs = require('fs')
var path = process.cwd()
var files = []
var getFiles = function(path, files){
fs.readdirSync(path).forEach(function(file){
var subpath = path + '/' + file;
if(fs.lstatSync(subpath).isDirectory()){
getFiles(subpath, files);
} else {
files.push(path + '/' + file);
}
});
}
Calling
getFiles(path, files)
console.log(files) // will log all files in directory
I loved the answer from chjj above and would not have been able to create my version of the parallel loop without that start.
var fs = require("fs");
var tree = function(dir, done) {
var results = {
"path": dir
,"children": []
};
fs.readdir(dir, function(err, list) {
if (err) { return done(err); }
var pending = list.length;
if (!pending) { return done(null, results); }
list.forEach(function(file) {
fs.stat(dir + '/' + file, function(err, stat) {
if (stat && stat.isDirectory()) {
tree(dir + '/' + file, function(err, res) {
results.children.push(res);
if (!--pending){ done(null, results); }
});
} else {
results.children.push({"path": dir + "/" + file});
if (!--pending) { done(null, results); }
}
});
});
});
};
module.exports = tree;
I created a Gist as well. Comments welcome. I am still starting out in the NodeJS realm so that is one way I hope to learn more.
Use node-dir to produce exactly the output you like
var dir = require('node-dir');
dir.files(__dirname, function(err, files) {
if (err) throw err;
console.log(files);
//we have an array of files now, so now we can iterate that array
files.forEach(function(path) {
action(null, path);
})
});
Here is a simple synchronous recursive solution
const fs = require('fs')
const getFiles = path => {
const files = []
for (const file of fs.readdirSync(path)) {
const fullPath = path + '/' + file
if(fs.lstatSync(fullPath).isDirectory())
getFiles(fullPath).forEach(x => files.push(file + '/' + x))
else files.push(file)
}
return files
}
Usage:
const files = getFiles(process.cwd())
console.log(files)
You could write it asynchronously, but there is no need. Just make sure that the input directory exists and is accessible.
Modern promise based read dir recursive version:
const fs = require('fs');
const path = require('path');
const readDirRecursive = async (filePath) => {
const dir = await fs.promises.readdir(filePath);
const files = await Promise.all(dir.map(async relativePath => {
const absolutePath = path.join(filePath, relativePath);
const stat = await fs.promises.lstat(absolutePath);
return stat.isDirectory() ? readDirRecursive(absolutePath) : absolutePath;
}));
return files.flat();
}
qwtel's answer variant, in TypeScript
import { resolve } from 'path';
import { readdir } from 'fs/promises';
async function* getFiles(dir: string): AsyncGenerator<string> {
const entries = await readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const res = resolve(dir, entry.name);
if (entry.isDirectory()) {
yield* getFiles(res);
} else {
yield res;
}
}
}
Vanilla ES6 + async/await + small & readable
I didn't find the answer I was looking for in this thread; there were a few similar elements spread across different answers, but I just wanted something simple and readable.
Just in case it helps anyone in the future (i.e. myself in a couple of months), this I what I ended up using:
const { readdir } = require('fs/promises');
const { join } = require('path');
const readdirRecursive = async dir => {
const files = await readdir( dir, { withFileTypes: true } );
const paths = files.map( async file => {
const path = join( dir, file.name );
if ( file.isDirectory() ) return await readdirRecursive( path );
return path;
} );
return ( await Promise.all( paths ) ).flat( Infinity );
}
module.exports = {
readdirRecursive,
}
I've coded this recently, and thought it would make sense to share this here. The code makes use of the async library.
var fs = require('fs');
var async = require('async');
var scan = function(dir, suffix, callback) {
fs.readdir(dir, function(err, files) {
var returnFiles = [];
async.each(files, function(file, next) {
var filePath = dir + '/' + file;
fs.stat(filePath, function(err, stat) {
if (err) {
return next(err);
}
if (stat.isDirectory()) {
scan(filePath, suffix, function(err, results) {
if (err) {
return next(err);
}
returnFiles = returnFiles.concat(results);
next();
})
}
else if (stat.isFile()) {
if (file.indexOf(suffix, file.length - suffix.length) !== -1) {
returnFiles.push(filePath);
}
next();
}
});
}, function(err) {
callback(err, returnFiles);
});
});
};
You can use it like this:
scan('/some/dir', '.ext', function(err, files) {
// Do something with files that ends in '.ext'.
console.log(files);
});
A library called Filehound is another option. It will recursively search a given directory (working directory by default). It supports various filters, callbacks, promises and sync searches.
For example, search the current working directory for all files (using callbacks):
const Filehound = require('filehound');
Filehound.create()
.find((err, files) => {
if (err) {
return console.error(`error: ${err}`);
}
console.log(files); // array of files
});
Or promises and specifying a specific directory:
const Filehound = require('filehound');
Filehound.create()
.paths("/tmp")
.find()
.each(console.log);
Consult the docs for further use cases and examples of usage: https://github.com/nspragg/filehound
Disclaimer: I'm the author.
Using async/await, this should work:
const FS = require('fs');
const readDir = promisify(FS.readdir);
const fileStat = promisify(FS.stat);
async function getFiles(dir) {
let files = await readDir(dir);
let result = files.map(file => {
let path = Path.join(dir,file);
return fileStat(path).then(stat => stat.isDirectory() ? getFiles(path) : path);
});
return flatten(await Promise.all(result));
}
function flatten(arr) {
return Array.prototype.concat(...arr);
}
You can use bluebird.Promisify or this:
/**
* Returns a function that will wrap the given `nodeFunction`. Instead of taking a callback, the returned function will return a promise whose fate is decided by the callback behavior of the given node function. The node function should conform to node.js convention of accepting a callback as last argument and calling that callback with error as the first argument and success value on the second argument.
*
* #param {Function} nodeFunction
* #returns {Function}
*/
module.exports = function promisify(nodeFunction) {
return function(...args) {
return new Promise((resolve, reject) => {
nodeFunction.call(this, ...args, (err, data) => {
if(err) {
reject(err);
} else {
resolve(data);
}
})
});
};
};
Node 8+ has Promisify built-in
See my other answer for a generator approach that can give results even faster.
Simple, Async Promise Based
const fs = require('fs/promises');
const getDirRecursive = async (dir) => {
try {
const items = await fs.readdir(dir);
let files = [];
for (const item of items) {
if ((await fs.lstat(`${dir}/${item}`)).isDirectory()) files = [...files, ...(await getDirRecursive(`${dir}/${item}`))];
else files.push({file: item, path: `${dir}/${item}`, parents: dir.split("/")});
}
return files;
} catch (e) {
return e
}
};
Usage: await getDirRecursive("./public");
Check out the final-fs library. It provides a readdirRecursive function:
ffs.readdirRecursive(dirPath, true, 'my/initial/path')
.then(function (files) {
// in the `files` variable you've got all the files
})
.otherwise(function (err) {
// something went wrong
});
Standalone promise implementation
I am using the when.js promise library in this example.
var fs = require('fs')
, path = require('path')
, when = require('when')
, nodefn = require('when/node/function');
function walk (directory, includeDir) {
var results = [];
return when.map(nodefn.call(fs.readdir, directory), function(file) {
file = path.join(directory, file);
return nodefn.call(fs.stat, file).then(function(stat) {
if (stat.isFile()) { return results.push(file); }
if (includeDir) { results.push(file + path.sep); }
return walk(file, includeDir).then(function(filesInDir) {
results = results.concat(filesInDir);
});
});
}).then(function() {
return results;
});
};
walk(__dirname).then(function(files) {
console.log(files);
}).otherwise(function(error) {
console.error(error.stack || error);
});
I've included an optional parameter includeDir which will include directories in the file listing if set to true.
klaw and klaw-sync are worth considering for this sort of thing. These were part of node-fs-extra.
For Node 10.3+, here is a for-await solution:
#!/usr/bin/env node
const FS = require('fs');
const Util = require('util');
const readDir = Util.promisify(FS.readdir);
const Path = require('path');
async function* readDirR(path) {
const entries = await readDir(path,{withFileTypes:true});
for(let entry of entries) {
const fullPath = Path.join(path,entry.name);
if(entry.isDirectory()) {
yield* readDirR(fullPath);
} else {
yield fullPath;
}
}
}
async function main() {
const start = process.hrtime.bigint();
for await(const file of readDirR('/mnt/home/media/Unsorted')) {
console.log(file);
}
console.log((process.hrtime.bigint()-start)/1000000n);
}
main().catch(err => {
console.error(err);
});
The benefit of this solution is that you can start processing the results immediately; e.g. it takes 12 seconds to read all the files in my media directory, but if I do it this way I can get the first result within a few milliseconds.
Here's yet another implementation. None of the above solutions have any limiters, and so if your directory structure is large, they're all going to thrash and eventually run out of resources.
var async = require('async');
var fs = require('fs');
var resolve = require('path').resolve;
var scan = function(path, concurrency, callback) {
var list = [];
var walker = async.queue(function(path, callback) {
fs.stat(path, function(err, stats) {
if (err) {
return callback(err);
} else {
if (stats.isDirectory()) {
fs.readdir(path, function(err, files) {
if (err) {
callback(err);
} else {
for (var i = 0; i < files.length; i++) {
walker.push(resolve(path, files[i]));
}
callback();
}
});
} else {
list.push(path);
callback();
}
}
});
}, concurrency);
walker.push(path);
walker.drain = function() {
callback(list);
}
};
Using a concurrency of 50 works pretty well, and is almost as fast as simpler implementations for small directory structures.
The recursive-readdir module has this functionality.
I modified Trevor Senior's Promise based answer to work with Bluebird
var fs = require('fs'),
path = require('path'),
Promise = require('bluebird');
var readdirAsync = Promise.promisify(fs.readdir);
var statAsync = Promise.promisify(fs.stat);
function walkFiles (directory) {
var results = [];
return readdirAsync(directory).map(function(file) {
file = path.join(directory, file);
return statAsync(file).then(function(stat) {
if (stat.isFile()) {
return results.push(file);
}
return walkFiles(file).then(function(filesInDir) {
results = results.concat(filesInDir);
});
});
}).then(function() {
return results;
});
}
//use
walkDir(__dirname).then(function(files) {
console.log(files);
}).catch(function(e) {
console.error(e); {
});
For fun, here is a flow based version that works with highland.js streams library. It was co-authored by Victor Vu.
###
directory >---m------> dirFilesStream >---------o----> out
| |
| |
+--------< returnPipe <-----------+
legend: (m)erge (o)bserve
+ directory has the initial file
+ dirListStream does a directory listing
+ out prints out the full path of the file
+ returnPipe runs stat and filters on directories
###
_ = require('highland')
fs = require('fs')
fsPath = require('path')
directory = _(['someDirectory'])
mergePoint = _()
dirFilesStream = mergePoint.merge().flatMap((parentPath) ->
_.wrapCallback(fs.readdir)(parentPath).sequence().map (path) ->
fsPath.join parentPath, path
)
out = dirFilesStream
# Create the return pipe
returnPipe = dirFilesStream.observe().flatFilter((path) ->
_.wrapCallback(fs.stat)(path).map (v) ->
v.isDirectory()
)
# Connect up the merge point now that we have all of our streams.
mergePoint.write directory
mergePoint.write returnPipe
mergePoint.end()
# Release backpressure. This will print files as they are discovered
out.each H.log
# Another way would be to queue them all up and then print them all out at once.
# out.toArray((files)-> console.log(files))
Using Promises (Q) to solve this in a Functional style:
var fs = require('fs'),
fsPath = require('path'),
Q = require('q');
var walk = function (dir) {
return Q.ninvoke(fs, 'readdir', dir).then(function (files) {
return Q.all(files.map(function (file) {
file = fsPath.join(dir, file);
return Q.ninvoke(fs, 'lstat', file).then(function (stat) {
if (stat.isDirectory()) {
return walk(file);
} else {
return [file];
}
});
}));
}).then(function (files) {
return files.reduce(function (pre, cur) {
return pre.concat(cur);
});
});
};
It returns a promise of an array, so you can use it as:
walk('/home/mypath').then(function (files) { console.log(files); });
I must add the Promise-based sander library to the list.
var sander = require('sander');
sander.lsr(directory).then( filenames => { console.log(filenames) } );

Resources