fs-extra: Copy files without overwrite - node.js

I am using copy method of fs-extra to copy files from a source to destination. My use case is to create a copy of the file with a name like if a file of the same name exists in the destination. The copy method of fs-extra module overwrites the destination file.

You could try something like this:
const fs = require('fs-extra');
async function copy(src, dest) {
try {
await fs.copy(src, dest, { overwrite: false, errorOnExist: true });
return true;
} catch (error) {
if (error.message.includes('already exists')) {
return false;
}
throw error;
}
}
async function copyWithoutOverwrite(src, dest, maxAttempts) {
try {
if (!await copy(src, dest)); {
for (let i = 1; i <= maxAttempts; i++) {
if (await copy(src, `${dest}_copy${i}`)) {
return;
}
}
}
} catch (error) {
console.error(error);
}
}
const src = '/tmp/testfile';
const dest = '/tmp/mynewfile';
const maxAttempts = 10;
copyWithoutOverwrite(src, dest, maxAttempts);

So basically you need to implement custom copy procedure, that you can interrupt and alter at any moment. fs-jetpack does this very well.
const pathUtil = require("path");
const jetpack = require("fs-jetpack");
const src = jetpack.cwd("path/to/source/folder");
const dst = jetpack.cwd("path/to/destination/folder");
const findUnusedPath = path => {
const pathDir = pathUtil.dirname(path);
const extension = pathUtil.extname(path);
const fileName = pathUtil.basename(path, extension);
const pathCandidate = pathUtil.join(pathDir, `${fileName}-duplicate${extension}`);
if (dst.exists(pathCandidate)) {
// Will just add "-duplicate-duplicate-duplicate..." to name
// of the file until finds unused name.
return findUnusedPath(pathCandidate);
} else {
return pathCandidate;
}
};
// Iterate over all files in source directory and manually decide
// where to put them in destination directory.
src.find().forEach(filePath => {
const content = src.read(filePath, "buffer");
let destPath = filePath;
if (dst.exists(destPath)) {
destPath = findUnusedPath(destPath);
}
dst.write(destPath, content);
});
Here using file read/write instead of copy, because it's faster in this particular case. Copy needs to check first what the path is: directory or file. If we already know it is a file there is no need for this extra work.

Related

How to read folders which only have specific subfolder named in node js?

I have the following directory structure. The main directory has folders (ex. main_A) and some of them have a subfolder named "foo" but the other doesn't.
main
|__main_A/foo
|__main_B/a
|__main_C/b
|__main_D/foo
|__main_E/foo
|__main_F/c
Currently, I am using fsPromise.readdir() to read the files. My question is simply how to read and dive into only folders including foo subfolder?
Eventually, I need to read all files in the foo folder.
async function readFooFolder() {
try {
const pathProd = 'C:\\main';
const fileList = await fsPromise.readdir(pathProd);
console.log(fileList);
} catch (err) {
console.error(err);
}
}
You can attempt to read the "foo" subdir for each directory in fileList. If the subddir does not exist for that directory, then an error with code ENOENT will be thrown.
const path = require('path');
async function readFooFolder() {
try {
const pathProd = 'C:\\main';
const fileList = await fsPromise.readdir(pathProd);
for (const subDir of fileList) {
const fooDir = path.join(pathProd, subDir, 'foo');
try {
const fooFiles = await fsPromise.readdir(fooDir);
// do something with fooFiles...
} catch (fooErr) {
if (fooErr.code != 'ENOENT') {
console.error(fooErr);
}
}
}
} catch (err) {
console.error(err);
}
}

Is there a way in nodejs to get all the files in a directory including files that are nested in folders [duplicate]

I have a little problem with my function. I would like to get all files in many directories. Currently, I can retrieve the files in the file passed in parameters. I would like to retrieve the html files of each folder in the folder passed as a parameter. I will explain if I put in parameter "test" I retrieve the files in "test" but I would like to retrieve "test / 1 / *. Html", "test / 2 / . /.html ":
var srcpath2 = path.join('.', 'diapo', result);
function getDirectories(srcpath2) {
return fs.readdirSync(srcpath2).filter(function (file) {
return fs.statSync(path.join(srcpath2, file)).isDirectory();
});
}
The result :
[1,2,3]
thanks !
It looks like the glob npm package would help you. Here is an example of how to use it:
File hierarchy:
test
├── one.html
└── test-nested
└── two.html
JS code:
const glob = require("glob");
var getDirectories = function (src, callback) {
glob(src + '/**/*', callback);
};
getDirectories('test', function (err, res) {
if (err) {
console.log('Error', err);
} else {
console.log(res);
}
});
which displays:
[ 'test/one.html',
'test/test-nested',
'test/test-nested/two.html' ]
I've seen many very long answers, and it's kinda a waste of memory space. Some also use packages like glob, but if you don't want to depend on any package, here's my solution.
const Path = require("path");
const FS = require("fs");
let Files = [];
function ThroughDirectory(Directory) {
FS.readdirSync(Directory).forEach(File => {
const Absolute = Path.join(Directory, File);
if (FS.statSync(Absolute).isDirectory()) return ThroughDirectory(Absolute);
else return Files.push(Absolute);
});
}
ThroughDirectory("./input/directory/");
It's pretty self-explanatory. There's an input directory, and it iterates through that. If one of the items is also a directory, go through that and so on. If it's a file, add the absolute path to the array.
Hope this helped :]
Using ES6 yield
const fs = require('fs');
const path = require('path');
function *walkSync(dir) {
const files = fs.readdirSync(dir, { withFileTypes: true });
for (const file of files) {
if (file.isDirectory()) {
yield* walkSync(path.join(dir, file.name));
} else {
yield path.join(dir, file.name);
}
}
}
for (const filePath of walkSync(__dirname)) {
console.log(filePath);
}
I really liked Smally's Solution but didn't like the Syntax.
Same solution but slightly easier to read:
const fs = require("fs");
const path = require("path");
let files = [];
const getFilesRecursively = (directory) => {
const filesInDirectory = fs.readdirSync(directory);
for (const file of filesInDirectory) {
const absolute = path.join(directory, file);
if (fs.statSync(absolute).isDirectory()) {
getFilesRecursively(absolute);
} else {
files.push(absolute);
}
}
};
Here's mine. Like all good answers it's hard to understand:
const isDirectory = path => statSync(path).isDirectory();
const getDirectories = path =>
readdirSync(path).map(name => join(path, name)).filter(isDirectory);
const isFile = path => statSync(path).isFile();
const getFiles = path =>
readdirSync(path).map(name => join(path, name)).filter(isFile);
const getFilesRecursively = (path) => {
let dirs = getDirectories(path);
let files = dirs
.map(dir => getFilesRecursively(dir)) // go through each directory
.reduce((a,b) => a.concat(b), []); // map returns a 2d array (array of file arrays) so flatten
return files.concat(getFiles(path));
};
With modern JavaScript (NodeJs 10) you can use async generator function and loop through them using for-await...of
// ES modules syntax that is included by default in NodeJS 14.
// For earlier versions, use `--experimental-modules` flag
import fs from "fs/promises"
// or, without ES modules, use this:
// const fs = require('fs').promises
async function run() {
for await (const file of getFiles()) {
console.log(file.path)
}
}
async function* getFiles(path = `./`) {
const entries = await fs.readdir(path, { withFileTypes: true })
for (let file of entries) {
if (file.isDirectory()) {
yield* getFiles(`${path}${file.name}/`)
} else {
yield { ...file, path: path + file.name }
}
}
}
run()
Packed into library:
https://www.npmjs.com/package/node-recursive-directory
https://github.com/vvmspace/node-recursive-directory
List of files:
const getFiles = require('node-recursive-directory');
(async () => {
const files = await getFiles('/home');
console.log(files);
})()
List of files with parsed data:
const getFiles = require('node-resursive-directory');
(async () => {
const files = await getFiles('/home', true); // add true
console.log(files);
})()
You will get something like that:
[
...,
{
fullpath: '/home/vvm/Downloads/images/Some/Some Image.jpg',
filepath: '/home/vvm/Downloads/images/Some/',
filename: 'Some Image.jpg',
dirname: 'Some'
},
]
You can also write your own code like below to traverse the directory as shown below :
var fs = require('fs');
function traverseDirectory(dirname, callback) {
var directory = [];
fs.readdir(dirname, function(err, list) {
dirname = fs.realpathSync(dirname);
if (err) {
return callback(err);
}
var listlength = list.length;
list.forEach(function(file) {
file = dirname + '\\' + file;
fs.stat(file, function(err, stat) {
directory.push(file);
if (stat && stat.isDirectory()) {
traverseDirectory(file, function(err, parsed) {
directory = directory.concat(parsed);
if (!--listlength) {
callback(null, directory);
}
});
} else {
if (!--listlength) {
callback(null, directory);
}
}
});
});
});
}
traverseDirectory(__dirname, function(err, result) {
if (err) {
console.log(err);
}
console.log(result);
});
You can check more information about it here : http://www.codingdefined.com/2014/09/how-to-navigate-through-directories-in.html
I needed to so something similar, in an Electron app: get all subfolders in a given base folder, using TypeScript, and came up with this:
import { readdirSync, statSync, existsSync } from "fs";
import * as path from "path";
// recursive synchronous "walk" through a folder structure, with the given base path
getAllSubFolders = (baseFolder, folderList = []) => {
let folders:string[] = readdirSync(baseFolder).filter(file => statSync(path.join(baseFolder, file)).isDirectory());
folders.forEach(folder => {
folderList.push(path.join(baseFolder,folder));
this.getAllSubFolders(path.join(baseFolder,folder), folderList);
});
}
const fs = require('fs');
const path = require('path');
var filesCollection = [];
const directoriesToSkip = ['bower_components', 'node_modules', 'www', 'platforms'];
function readDirectorySynchronously(directory) {
var currentDirectorypath = path.join(__dirname + directory);
var currentDirectory = fs.readdirSync(currentDirectorypath, 'utf8');
currentDirectory.forEach(file => {
var fileShouldBeSkipped = directoriesToSkip.indexOf(file) > -1;
var pathOfCurrentItem = path.join(__dirname + directory + '/' + file);
if (!fileShouldBeSkipped && fs.statSync(pathOfCurrentItem).isFile()) {
filesCollection.push(pathOfCurrentItem);
}
else if (!fileShouldBeSkipped) {
var directorypath = path.join(directory + '\\' + file);
readDirectorySynchronously(directorypath);
}
});
}
readDirectorySynchronously('');
This will fill filesCollection with all the files in the directory and its subdirectories (it's recursive). You have the option to skip some directory names in the directoriesToSkip array.
Speaking of npm packages - another short option is to use fs-readdir-recursive:
const read = require("fs-readdir-recursive");
const foundFiles = read("test");
console.log(foundFiles);
Output:
[ 'one.html', 'test-nested/some_text.txt', 'test-nested/two.html' ]
If you're interested only in files with specific extension (like .html mentioned in the question) you can filter them using .endsWith():
const filteredFiles = read("test").filter(item => item.endsWith(".html"));
The accepted answer needs to install a package.
If you want a native option that is ES6:
import { readdirSync } from 'fs'
import { join } from 'path'
function walk(dir) {
return readdirSync(dir, { withFileTypes: true }).flatMap((file) => file.isDirectory() ? walk(join(dir, file.name)) : join(dir, file.name))
}
This works for me.
Read root directory with readdirSync
Then map over but flatten as we go
if it's a directory, go recursive; else return the filename
If you rather work synchronously with glob, use the glob.sync() function as mentioned in their documentation. Here's the equivalent example provided by #Paul Mougel but written synchronously:
const glob = require("glob");
var getDirectories = function (src) {
return glob.sync(src + '/**/*');
};
var rest = getDirectories('test');
console.log(res);
A solution with Promises based on globby:
import { globby } from 'globby';
(async () => {
const path = '/path/to/dir';
const files = await globby([`${path}/**/*`]);
console.log(files);
// [
// '/path/to/dir/file1.txt',
// '/path/to/dir/subdir/file2.txt',
// ...
// ]
})()
Synchrone method with two option, simple and efficacy.
const path = require('path');const fs = require('fs');
function toHierarchie_files(pathDir, output_normalize=false, removeEmpty=true)
{
var result = {}, enqueue = [pathDir];
//normalize slash separator if output_normalize is true or just return val
output_normalize = output_normalize == false?val => {return val}:val => {return path.normalize(val)};
//allows absolute or relative path with extended resolution. Returns path normalize absolute to work with or 'none' string.
const path_exist = (path_test) => {var tmpTab = fs.existsSync(path.normalize(path.resolve(path_test))) == true?[path.normalize(path.resolve(path_test))]:['', '../', '../../'].map(val => path.normalize(path.resolve(__dirname, val+path_test))).filter((val, index) => fs.existsSync(path.normalize(path.resolve(__dirname, val+path_test))) == true);return tmpTab.length > 0?tmpTab[0]:'none'};
//Check if file exist and return her type or 'none' string
const getType = (path_test) => {path_test = path_exist(path_test);return path_test == 'none'?'none':fs.lstatSync(path_test).isDirectory() == true?'dir':fs.lstatSync(path_test).isFile() == true?'file':'none';};
function recursive()
{
//init new entrie
var parentDir = enqueue.pop();result[parentDir]=[];
//read dir
fs.readdirSync(path_exist(parentDir)).forEach((file, index) =>{
switch(getType(parentDir+'/'+file))
{
//if detect dir push in queue
case 'dir': enqueue.push(output_normalize(parentDir+'/'+file)); break;
//if file, add in entrie
case 'file': result[parentDir].push(file); break;
//else done
default: break;
};
});
//if optional arg remove empty is true, delete entries if not contains files
if(result[parentDir].length == 0 && removeEmpty == true){Reflect.deleteProperty(result, parentDir);}
//if queue is not empty continue processing
if(enqueue.length > 0){recursive();}
};
//if dir renseign exist, go recusive
if(getType(pathDir) == 'dir'){recursive();}
return result;
};
Result:
{
"public/assets": [
"favicon.ico"
],
"public/assets/js": [
"dede.js",
"test.js"
],
"public/assets/js/css/secure": [
"config.json",
"index.js"
],
"public/assets/css": [
"style.css"
]
}
You can use loop through all the files and directories of the root folder, if it's a directory, then get inside it and repeat the process.
Consider the code below:
const fs = require('fs');
const path = require('path');
const target = './'; // choose the directory to target
var result = []
var filePaths = []
var tempFolder = []
const targetPath = fs.readdirSync(target);
function hit(mainPath = targetPath) {
mainPath.forEach((file) => {
let check = fs.statSync(file);
if (!check.isDirectory()) {
filePaths.push(file)
}
else {
if (file[0] != '.') {
tempFolder.push(file)
}
}
});
// get files from folder
if (tempFolder.length > 0) {
tempFolder.forEach((dir) => {
getFiles(dir)
})
}
// filePaths contains path to every file
}
function getFiles(dir) {
var paths = fs.readdirSync(dir);
var files = [];
paths.forEach(function (file) {
var fullPath = dir + '/' + file;
files.push(fullPath);
});
files.forEach((tempFile) => {
let check = fs.statSync(tempFile);
if (check.isDirectory()) {
getFiles(tempFile)
} else {
filePaths.push(tempFile)
}
})
}
hit(); // main function
Although not perfect in some scenarios, it must be helpful in many.
const getAllFilePath = (path: string) => {
const addData = (_paths: string[]) => {
const newFoldersToScrape: string[] = [];
_paths.forEach(_path => {
fs.readdirSync(_path).forEach((file: string) => {
if (file.indexOf(".") === -1) {
newFoldersToScrape.push(`${_path}/${file}`);
} else {
filePaths.push(`${_path}/${file}`);
}
});
});
foldersToScrape = newFoldersToScrape;
};
const baseDirPath = `<YOUR BASE PATH HERE>/${path}`;
let foldersToScrape: string[] = [];
const filePaths: string[] = [];
addData([baseDirPath]);
while (foldersToScrape.length !== 0) {
addData(foldersToScrape);
}
return filePaths;
};
This is how I did it, I think it is similar to yet simpler than most of the other answers here.
const fs = require('fs')
let files = []
const getFiles = (path) => {
if (fs.lstatSync(path).isDirectory()) { // is this a folder?
fs.readdirSync(path).forEach(f => { // for everything in this folder
getFiles(path + '/' + f) // process it recursively
})
} else if (path.endsWith(".ts")) { // is this a file we are searching for?
files.push(path) // record it
}
}
getFiles("src")
It fills the "files" array with every .ts file under the "src/" directory.
Slightly modified version of #Stephen's response (https://stackoverflow.com/a/66083078/4421370) above that returns the files' path relative to the directory you are searching. Or any arbitrary base path you supply to the function call in-place of the default base. If you want the full path just call it as walkSync(dir, dir).
Search Path is: c:\tmp,
File path is c:\tmp\test\myfile.txt,
Result is test\myfile.txt
Hopefully helpful to some.
const fs = require('fs');
const path = require('path');
function *walkSync(dir, base="") {
const files = fs.readdirSync(dir, { withFileTypes: true })
for (const file of files) {
if (file.isDirectory()) {
yield* walkSync(path.join(dir, file.name), path.join(base, file.name));
} else {
yield path.join(base, file.name);
}
}
}
for (const filePath of walkSync(__dirname)) {
console.log(filePath);
}
Here is a compact pure function that returns all the paths (relatives) in the directory.
import path from 'path'
const getFilesPathsRecursively = (directory: string, origin?: string): string[] =>
fs.readdirSync(directory).reduce((files, file) => {
const absolute = path.join(directory, file)
return [
...files,
...(fs.statSync(absolute).isDirectory()
? getFilesPathsRecursively(absolute, origin || directory)
: [path.relative(origin || directory, absolute)]),
]
}, [])
The solution is written in TypeScript.
modern solution with async/await
No external dependencies.
Asynchronous function (non-blocking like other solutions with readdirSync and statSync)
It is extremely fast because multiple processes work in parallel (it is not waiting for a response from every file in the list).
It has also some naive error handling (if something happens with one file or folder it will not blow whole process)
import path from "path";
import fs from "fs/promises"
export default async function readDirectory(directory: string): Promise<string[]> {
const files = await fs.readdir(directory)
const filesPromises = files.map(async (file) => {
try {
const absolutePath = path.join(directory, file);
const fileStat = await fs.stat(absolutePath)
if (fileStat.isDirectory()) {
return await readDirectory(absolutePath);
} else {
return absolutePath;
}
} catch (err) {
// error handling
return [];
}
});
const filesWithArrays = await Promise.all(filesPromises)
const flatArray = filesWithArrays.reduce<string[]>((acc, fileOrArray) => acc.concat(fileOrArray), []);
return flatArray;
}
usage (if this is a separate file please remember to import)
const results = await readDirectory('some/path');
I did mine with typescript works well fairly easy to understand
import * as fs from 'fs';
import * as path from 'path';
export const getAllSubFolders = (
baseFolder: string,
folderList: string[] = []
) => {
const folders: string[] = fs
.readdirSync(baseFolder)
.filter(file => fs.statSync(path.join(baseFolder, file)).isDirectory());
folders.forEach(folder => {
folderList.push(path.join(baseFolder, folder));
getAllSubFolders(path.join(baseFolder, folder), folderList);
});
return folderList;
};
export const getFilesInFolder = (rootPath: string) => {
return fs
.readdirSync(rootPath)
.filter(
filePath => !fs.statSync(path.join(rootPath, filePath)).isDirectory()
)
.map(filePath => path.normalize(path.join(rootPath, filePath)));
};
export const getFilesRecursively = (rootPath: string) => {
const subFolders: string[] = getAllSubFolders(rootPath);
const allFiles: string[][] = subFolders.map(folder =>
getFilesInFolder(folder)
);
return [].concat.apply([], allFiles);
};

Get all files recursively in directories NodejS

I have a little problem with my function. I would like to get all files in many directories. Currently, I can retrieve the files in the file passed in parameters. I would like to retrieve the html files of each folder in the folder passed as a parameter. I will explain if I put in parameter "test" I retrieve the files in "test" but I would like to retrieve "test / 1 / *. Html", "test / 2 / . /.html ":
var srcpath2 = path.join('.', 'diapo', result);
function getDirectories(srcpath2) {
return fs.readdirSync(srcpath2).filter(function (file) {
return fs.statSync(path.join(srcpath2, file)).isDirectory();
});
}
The result :
[1,2,3]
thanks !
It looks like the glob npm package would help you. Here is an example of how to use it:
File hierarchy:
test
├── one.html
└── test-nested
└── two.html
JS code:
const glob = require("glob");
var getDirectories = function (src, callback) {
glob(src + '/**/*', callback);
};
getDirectories('test', function (err, res) {
if (err) {
console.log('Error', err);
} else {
console.log(res);
}
});
which displays:
[ 'test/one.html',
'test/test-nested',
'test/test-nested/two.html' ]
I've seen many very long answers, and it's kinda a waste of memory space. Some also use packages like glob, but if you don't want to depend on any package, here's my solution.
const Path = require("path");
const FS = require("fs");
let Files = [];
function ThroughDirectory(Directory) {
FS.readdirSync(Directory).forEach(File => {
const Absolute = Path.join(Directory, File);
if (FS.statSync(Absolute).isDirectory()) return ThroughDirectory(Absolute);
else return Files.push(Absolute);
});
}
ThroughDirectory("./input/directory/");
It's pretty self-explanatory. There's an input directory, and it iterates through that. If one of the items is also a directory, go through that and so on. If it's a file, add the absolute path to the array.
Hope this helped :]
Using ES6 yield
const fs = require('fs');
const path = require('path');
function *walkSync(dir) {
const files = fs.readdirSync(dir, { withFileTypes: true });
for (const file of files) {
if (file.isDirectory()) {
yield* walkSync(path.join(dir, file.name));
} else {
yield path.join(dir, file.name);
}
}
}
for (const filePath of walkSync(__dirname)) {
console.log(filePath);
}
I really liked Smally's Solution but didn't like the Syntax.
Same solution but slightly easier to read:
const fs = require("fs");
const path = require("path");
let files = [];
const getFilesRecursively = (directory) => {
const filesInDirectory = fs.readdirSync(directory);
for (const file of filesInDirectory) {
const absolute = path.join(directory, file);
if (fs.statSync(absolute).isDirectory()) {
getFilesRecursively(absolute);
} else {
files.push(absolute);
}
}
};
Here's mine. Like all good answers it's hard to understand:
const isDirectory = path => statSync(path).isDirectory();
const getDirectories = path =>
readdirSync(path).map(name => join(path, name)).filter(isDirectory);
const isFile = path => statSync(path).isFile();
const getFiles = path =>
readdirSync(path).map(name => join(path, name)).filter(isFile);
const getFilesRecursively = (path) => {
let dirs = getDirectories(path);
let files = dirs
.map(dir => getFilesRecursively(dir)) // go through each directory
.reduce((a,b) => a.concat(b), []); // map returns a 2d array (array of file arrays) so flatten
return files.concat(getFiles(path));
};
With modern JavaScript (NodeJs 10) you can use async generator function and loop through them using for-await...of
// ES modules syntax that is included by default in NodeJS 14.
// For earlier versions, use `--experimental-modules` flag
import fs from "fs/promises"
// or, without ES modules, use this:
// const fs = require('fs').promises
async function run() {
for await (const file of getFiles()) {
console.log(file.path)
}
}
async function* getFiles(path = `./`) {
const entries = await fs.readdir(path, { withFileTypes: true })
for (let file of entries) {
if (file.isDirectory()) {
yield* getFiles(`${path}${file.name}/`)
} else {
yield { ...file, path: path + file.name }
}
}
}
run()
Packed into library:
https://www.npmjs.com/package/node-recursive-directory
https://github.com/vvmspace/node-recursive-directory
List of files:
const getFiles = require('node-recursive-directory');
(async () => {
const files = await getFiles('/home');
console.log(files);
})()
List of files with parsed data:
const getFiles = require('node-resursive-directory');
(async () => {
const files = await getFiles('/home', true); // add true
console.log(files);
})()
You will get something like that:
[
...,
{
fullpath: '/home/vvm/Downloads/images/Some/Some Image.jpg',
filepath: '/home/vvm/Downloads/images/Some/',
filename: 'Some Image.jpg',
dirname: 'Some'
},
]
You can also write your own code like below to traverse the directory as shown below :
var fs = require('fs');
function traverseDirectory(dirname, callback) {
var directory = [];
fs.readdir(dirname, function(err, list) {
dirname = fs.realpathSync(dirname);
if (err) {
return callback(err);
}
var listlength = list.length;
list.forEach(function(file) {
file = dirname + '\\' + file;
fs.stat(file, function(err, stat) {
directory.push(file);
if (stat && stat.isDirectory()) {
traverseDirectory(file, function(err, parsed) {
directory = directory.concat(parsed);
if (!--listlength) {
callback(null, directory);
}
});
} else {
if (!--listlength) {
callback(null, directory);
}
}
});
});
});
}
traverseDirectory(__dirname, function(err, result) {
if (err) {
console.log(err);
}
console.log(result);
});
You can check more information about it here : http://www.codingdefined.com/2014/09/how-to-navigate-through-directories-in.html
I needed to so something similar, in an Electron app: get all subfolders in a given base folder, using TypeScript, and came up with this:
import { readdirSync, statSync, existsSync } from "fs";
import * as path from "path";
// recursive synchronous "walk" through a folder structure, with the given base path
getAllSubFolders = (baseFolder, folderList = []) => {
let folders:string[] = readdirSync(baseFolder).filter(file => statSync(path.join(baseFolder, file)).isDirectory());
folders.forEach(folder => {
folderList.push(path.join(baseFolder,folder));
this.getAllSubFolders(path.join(baseFolder,folder), folderList);
});
}
const fs = require('fs');
const path = require('path');
var filesCollection = [];
const directoriesToSkip = ['bower_components', 'node_modules', 'www', 'platforms'];
function readDirectorySynchronously(directory) {
var currentDirectorypath = path.join(__dirname + directory);
var currentDirectory = fs.readdirSync(currentDirectorypath, 'utf8');
currentDirectory.forEach(file => {
var fileShouldBeSkipped = directoriesToSkip.indexOf(file) > -1;
var pathOfCurrentItem = path.join(__dirname + directory + '/' + file);
if (!fileShouldBeSkipped && fs.statSync(pathOfCurrentItem).isFile()) {
filesCollection.push(pathOfCurrentItem);
}
else if (!fileShouldBeSkipped) {
var directorypath = path.join(directory + '\\' + file);
readDirectorySynchronously(directorypath);
}
});
}
readDirectorySynchronously('');
This will fill filesCollection with all the files in the directory and its subdirectories (it's recursive). You have the option to skip some directory names in the directoriesToSkip array.
Speaking of npm packages - another short option is to use fs-readdir-recursive:
const read = require("fs-readdir-recursive");
const foundFiles = read("test");
console.log(foundFiles);
Output:
[ 'one.html', 'test-nested/some_text.txt', 'test-nested/two.html' ]
If you're interested only in files with specific extension (like .html mentioned in the question) you can filter them using .endsWith():
const filteredFiles = read("test").filter(item => item.endsWith(".html"));
The accepted answer needs to install a package.
If you want a native option that is ES6:
import { readdirSync } from 'fs'
import { join } from 'path'
function walk(dir) {
return readdirSync(dir, { withFileTypes: true }).flatMap((file) => file.isDirectory() ? walk(join(dir, file.name)) : join(dir, file.name))
}
This works for me.
Read root directory with readdirSync
Then map over but flatten as we go
if it's a directory, go recursive; else return the filename
If you rather work synchronously with glob, use the glob.sync() function as mentioned in their documentation. Here's the equivalent example provided by #Paul Mougel but written synchronously:
const glob = require("glob");
var getDirectories = function (src) {
return glob.sync(src + '/**/*');
};
var rest = getDirectories('test');
console.log(res);
A solution with Promises based on globby:
import { globby } from 'globby';
(async () => {
const path = '/path/to/dir';
const files = await globby([`${path}/**/*`]);
console.log(files);
// [
// '/path/to/dir/file1.txt',
// '/path/to/dir/subdir/file2.txt',
// ...
// ]
})()
Synchrone method with two option, simple and efficacy.
const path = require('path');const fs = require('fs');
function toHierarchie_files(pathDir, output_normalize=false, removeEmpty=true)
{
var result = {}, enqueue = [pathDir];
//normalize slash separator if output_normalize is true or just return val
output_normalize = output_normalize == false?val => {return val}:val => {return path.normalize(val)};
//allows absolute or relative path with extended resolution. Returns path normalize absolute to work with or 'none' string.
const path_exist = (path_test) => {var tmpTab = fs.existsSync(path.normalize(path.resolve(path_test))) == true?[path.normalize(path.resolve(path_test))]:['', '../', '../../'].map(val => path.normalize(path.resolve(__dirname, val+path_test))).filter((val, index) => fs.existsSync(path.normalize(path.resolve(__dirname, val+path_test))) == true);return tmpTab.length > 0?tmpTab[0]:'none'};
//Check if file exist and return her type or 'none' string
const getType = (path_test) => {path_test = path_exist(path_test);return path_test == 'none'?'none':fs.lstatSync(path_test).isDirectory() == true?'dir':fs.lstatSync(path_test).isFile() == true?'file':'none';};
function recursive()
{
//init new entrie
var parentDir = enqueue.pop();result[parentDir]=[];
//read dir
fs.readdirSync(path_exist(parentDir)).forEach((file, index) =>{
switch(getType(parentDir+'/'+file))
{
//if detect dir push in queue
case 'dir': enqueue.push(output_normalize(parentDir+'/'+file)); break;
//if file, add in entrie
case 'file': result[parentDir].push(file); break;
//else done
default: break;
};
});
//if optional arg remove empty is true, delete entries if not contains files
if(result[parentDir].length == 0 && removeEmpty == true){Reflect.deleteProperty(result, parentDir);}
//if queue is not empty continue processing
if(enqueue.length > 0){recursive();}
};
//if dir renseign exist, go recusive
if(getType(pathDir) == 'dir'){recursive();}
return result;
};
Result:
{
"public/assets": [
"favicon.ico"
],
"public/assets/js": [
"dede.js",
"test.js"
],
"public/assets/js/css/secure": [
"config.json",
"index.js"
],
"public/assets/css": [
"style.css"
]
}
You can use loop through all the files and directories of the root folder, if it's a directory, then get inside it and repeat the process.
Consider the code below:
const fs = require('fs');
const path = require('path');
const target = './'; // choose the directory to target
var result = []
var filePaths = []
var tempFolder = []
const targetPath = fs.readdirSync(target);
function hit(mainPath = targetPath) {
mainPath.forEach((file) => {
let check = fs.statSync(file);
if (!check.isDirectory()) {
filePaths.push(file)
}
else {
if (file[0] != '.') {
tempFolder.push(file)
}
}
});
// get files from folder
if (tempFolder.length > 0) {
tempFolder.forEach((dir) => {
getFiles(dir)
})
}
// filePaths contains path to every file
}
function getFiles(dir) {
var paths = fs.readdirSync(dir);
var files = [];
paths.forEach(function (file) {
var fullPath = dir + '/' + file;
files.push(fullPath);
});
files.forEach((tempFile) => {
let check = fs.statSync(tempFile);
if (check.isDirectory()) {
getFiles(tempFile)
} else {
filePaths.push(tempFile)
}
})
}
hit(); // main function
Although not perfect in some scenarios, it must be helpful in many.
const getAllFilePath = (path: string) => {
const addData = (_paths: string[]) => {
const newFoldersToScrape: string[] = [];
_paths.forEach(_path => {
fs.readdirSync(_path).forEach((file: string) => {
if (file.indexOf(".") === -1) {
newFoldersToScrape.push(`${_path}/${file}`);
} else {
filePaths.push(`${_path}/${file}`);
}
});
});
foldersToScrape = newFoldersToScrape;
};
const baseDirPath = `<YOUR BASE PATH HERE>/${path}`;
let foldersToScrape: string[] = [];
const filePaths: string[] = [];
addData([baseDirPath]);
while (foldersToScrape.length !== 0) {
addData(foldersToScrape);
}
return filePaths;
};
This is how I did it, I think it is similar to yet simpler than most of the other answers here.
const fs = require('fs')
let files = []
const getFiles = (path) => {
if (fs.lstatSync(path).isDirectory()) { // is this a folder?
fs.readdirSync(path).forEach(f => { // for everything in this folder
getFiles(path + '/' + f) // process it recursively
})
} else if (path.endsWith(".ts")) { // is this a file we are searching for?
files.push(path) // record it
}
}
getFiles("src")
It fills the "files" array with every .ts file under the "src/" directory.
Slightly modified version of #Stephen's response (https://stackoverflow.com/a/66083078/4421370) above that returns the files' path relative to the directory you are searching. Or any arbitrary base path you supply to the function call in-place of the default base. If you want the full path just call it as walkSync(dir, dir).
Search Path is: c:\tmp,
File path is c:\tmp\test\myfile.txt,
Result is test\myfile.txt
Hopefully helpful to some.
const fs = require('fs');
const path = require('path');
function *walkSync(dir, base="") {
const files = fs.readdirSync(dir, { withFileTypes: true })
for (const file of files) {
if (file.isDirectory()) {
yield* walkSync(path.join(dir, file.name), path.join(base, file.name));
} else {
yield path.join(base, file.name);
}
}
}
for (const filePath of walkSync(__dirname)) {
console.log(filePath);
}
Here is a compact pure function that returns all the paths (relatives) in the directory.
import path from 'path'
const getFilesPathsRecursively = (directory: string, origin?: string): string[] =>
fs.readdirSync(directory).reduce((files, file) => {
const absolute = path.join(directory, file)
return [
...files,
...(fs.statSync(absolute).isDirectory()
? getFilesPathsRecursively(absolute, origin || directory)
: [path.relative(origin || directory, absolute)]),
]
}, [])
The solution is written in TypeScript.
modern solution with async/await
No external dependencies.
Asynchronous function (non-blocking like other solutions with readdirSync and statSync)
It is extremely fast because multiple processes work in parallel (it is not waiting for a response from every file in the list).
It has also some naive error handling (if something happens with one file or folder it will not blow whole process)
import path from "path";
import fs from "fs/promises"
export default async function readDirectory(directory: string): Promise<string[]> {
const files = await fs.readdir(directory)
const filesPromises = files.map(async (file) => {
try {
const absolutePath = path.join(directory, file);
const fileStat = await fs.stat(absolutePath)
if (fileStat.isDirectory()) {
return await readDirectory(absolutePath);
} else {
return absolutePath;
}
} catch (err) {
// error handling
return [];
}
});
const filesWithArrays = await Promise.all(filesPromises)
const flatArray = filesWithArrays.reduce<string[]>((acc, fileOrArray) => acc.concat(fileOrArray), []);
return flatArray;
}
usage (if this is a separate file please remember to import)
const results = await readDirectory('some/path');
I did mine with typescript works well fairly easy to understand
import * as fs from 'fs';
import * as path from 'path';
export const getAllSubFolders = (
baseFolder: string,
folderList: string[] = []
) => {
const folders: string[] = fs
.readdirSync(baseFolder)
.filter(file => fs.statSync(path.join(baseFolder, file)).isDirectory());
folders.forEach(folder => {
folderList.push(path.join(baseFolder, folder));
getAllSubFolders(path.join(baseFolder, folder), folderList);
});
return folderList;
};
export const getFilesInFolder = (rootPath: string) => {
return fs
.readdirSync(rootPath)
.filter(
filePath => !fs.statSync(path.join(rootPath, filePath)).isDirectory()
)
.map(filePath => path.normalize(path.join(rootPath, filePath)));
};
export const getFilesRecursively = (rootPath: string) => {
const subFolders: string[] = getAllSubFolders(rootPath);
const allFiles: string[][] = subFolders.map(folder =>
getFilesInFolder(folder)
);
return [].concat.apply([], allFiles);
};

Node fs copy a folder

I am trying to copy a folder using Node fs module. I am familiar with readFileSync() and writeFileSync() methods but I am wondering what method I should use to copy a specified folder?
You can use fs-extra to copy contents of one folder to another like this
var fs = require("fs-extra");
fs.copy('/path/to/source', '/path/to/destination', function (err) {
if (err) return console.error(err)
console.log('success!')
});
There's also a synchronous version.
fs.copySync('/path/to/source', '/path/to/destination')
Save yourself the extra dependency with just 10 lines of native node functions
Add the following copyDir function:
const { promises: fs } = require("fs")
const path = require("path")
async function copyDir(src, dest) {
await fs.mkdir(dest, { recursive: true });
let entries = await fs.readdir(src, { withFileTypes: true });
for (let entry of entries) {
let srcPath = path.join(src, entry.name);
let destPath = path.join(dest, entry.name);
entry.isDirectory() ?
await copyDir(srcPath, destPath) :
await fs.copyFile(srcPath, destPath);
}
}
And then use like this:
copyDir("./inputFolder", "./outputFolder")
Further Reading
Copy folder recursively in node.js
fsPromises.copyFile (added in v10.11.0)
fsPromises.readdir (added in v10.0)
fsPromises.mkdir (added in v10.0)
You might want to check out the ncp package. It does exactly what you're trying to do; Recursively copy files from a path to another.
Here's something to get your started :
const fs = require("fs");
const path = require("path");
const ncp = require("ncp").ncp;
// No limit, because why not?
ncp.limit = 0;
var thePath = "./";
var folder = "testFolder";
var newFolder = "newTestFolder";
ncp(path.join(thePath, folder), path.join(thePath, newFolder), function (err) {
if (err) {
return console.error(err);
}
console.log("Done !");
});
I liked KyleMit's answer, but thought a parallel version would be preferable.
The code is in TypeScript. If you need JavaScript, just delete the : string type annotations on the line of the declaration of copyDirectory.
import { promises as fs } from "fs"
import path from "path"
export const copyDirectory = async (src: string, dest: string) => {
const [entries] = await Promise.all([
fs.readdir(src, { withFileTypes: true }),
fs.mkdir(dest, { recursive: true }),
])
await Promise.all(
entries.map((entry) => {
const srcPath = path.join(src, entry.name)
const destPath = path.join(dest, entry.name)
return entry.isDirectory()
? copyDirectory(srcPath, destPath)
: fs.copyFile(srcPath, destPath)
})
)
}
Here's the synchronous version of #KyleMit answer
copyDirectory(source, destination) {
fs.mkdirSync(destination, { recursive: true });
fs.readdirSync(source, { withFileTypes: true }).forEach((entry) => {
let sourcePath = path.join(source, entry.name);
let destinationPath = path.join(destination, entry.name);
entry.isDirectory()
? copyDirectory(sourcePath, destinationPath)
: fs.copyFileSync(sourcePath, destinationPath);
});
}
There is an elegant syntax. You can use the pwd-fs module.
const FileSystem = require('pwd-fs');
const pfs = new FileSystem();
async () => {
await pfs.copy('./path', './dest');
}

Get the most recent file in a directory, Node.js

I am trying to find the most recently created file in a directory using Node.js and cannot seem to find a solution. The following code seemed to be doing the trick on one machine but on another it was just pulling a random file from the directory - as I figured it might. Basically, I need to find the newest file and ONLY that file.
var fs = require('fs'); //File System
var audioFilePath = 'C:/scanner/audio/'; //Location of recorded audio files
var audioFile = fs.readdirSync(audioFilePath)
.slice(-1)[0]
.replace('.wav', '.mp3');
Many thanks!
Assuming availability of underscore (http://underscorejs.org/) and taking synchronous approach (which doesn't utilize the node.js strengths, but is easier to grasp):
var fs = require('fs'),
path = require('path'),
_ = require('underscore');
// Return only base file name without dir
function getMostRecentFileName(dir) {
var files = fs.readdirSync(dir);
// use underscore for max()
return _.max(files, function (f) {
var fullpath = path.join(dir, f);
// ctime = creation time is used
// replace with mtime for modification time
return fs.statSync(fullpath).ctime;
});
}
While not the most efficient approach, this should be conceptually straight forward:
var fs = require('fs'); //File System
var audioFilePath = 'C:/scanner/audio/'; //Location of recorded audio files
fs.readdir(audioFilePath, function(err, files) {
if (err) { throw err; }
var audioFile = getNewestFile(files, audioFilePath).replace('.wav', '.mp3');
//process audioFile here or pass it to a function...
console.log(audioFile);
});
function getNewestFile(files, path) {
var out = [];
files.forEach(function(file) {
var stats = fs.statSync(path + "/" +file);
if(stats.isFile()) {
out.push({"file":file, "mtime": stats.mtime.getTime()});
}
});
out.sort(function(a,b) {
return b.mtime - a.mtime;
})
return (out.length>0) ? out[0].file : "";
}
BTW, there is no obvious reason in the original post to use synchronous file listing.
Another approach:
const fs = require('fs')
const glob = require('glob')
const newestFile = glob.sync('input/*xlsx')
.map(name => ({name, ctime: fs.statSync(name).ctime}))
.sort((a, b) => b.ctime - a.ctime)[0].name
A more functional version might look like:
import { readdirSync, lstatSync } from "fs";
const orderReccentFiles = (dir: string) =>
readdirSync(dir)
.filter(f => lstatSync(f).isFile())
.map(file => ({ file, mtime: lstatSync(file).mtime }))
.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
const getMostRecentFile = (dir: string) => {
const files = orderReccentFiles(dir);
return files.length ? files[0] : undefined;
};
First, you need to order files (newest at the begin)
Then, get the first element of an array for the most recent file.
I have modified code from #mikeysee to avoid the path exception so that I use the full path to fix them.
The snipped codes of 2 functions are shown below.
const fs = require('fs');
const path = require('path');
const getMostRecentFile = (dir) => {
const files = orderReccentFiles(dir);
return files.length ? files[0] : undefined;
};
const orderReccentFiles = (dir) => {
return fs.readdirSync(dir)
.filter(file => fs.lstatSync(path.join(dir, file)).isFile())
.map(file => ({ file, mtime: fs.lstatSync(path.join(dir, file)).mtime }))
.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
};
const dirPath = '<PATH>';
getMostRecentFile(dirPath)
this should do the trick ("dir" is the directory you use fs.readdir over to get the "files" array):
function getNewestFile(dir, files, callback) {
if (!callback) return;
if (!files || (files && files.length === 0)) {
callback();
}
if (files.length === 1) {
callback(files[0]);
}
var newest = { file: files[0] };
var checked = 0;
fs.stat(dir + newest.file, function(err, stats) {
newest.mtime = stats.mtime;
for (var i = 0; i < files.length; i++) {
var file = files[i];
(function(file) {
fs.stat(file, function(err, stats) {
++checked;
if (stats.mtime.getTime() > newest.mtime.getTime()) {
newest = { file : file, mtime : stats.mtime };
}
if (checked == files.length) {
callback(newest);
}
});
})(dir + file);
}
});
}
[Extended umair's answer to correct a bug with current working directory]
function getNewestFile(dir, regexp) {
var fs = require("fs"),
path = require('path'),
newest = null,
files = fs.readdirSync(dir),
one_matched = 0,
i
for (i = 0; i < files.length; i++) {
if (regexp.test(files[i]) == false)
continue
else if (one_matched == 0) {
newest = files[i];
one_matched = 1;
continue
}
f1_time = fs.statSync(path.join(dir, files[i])).mtime.getTime()
f2_time = fs.statSync(path.join(dir, newest)).mtime.getTime()
if (f1_time > f2_time)
newest[i] = files[i]
}
if (newest != null)
return (path.join(dir, newest))
return null
}
Using pure JavaScript and easy to understand structure :
function getLatestFile(dirpath) {
// Check if dirpath exist or not right here
let latest;
const files = fs.readdirSync(dirpath);
files.forEach(filename => {
// Get the stat
const stat = fs.lstatSync(path.join(dirpath, filename));
// Pass if it is a directory
if (stat.isDirectory())
return;
// latest default to first file
if (!latest) {
latest = {filename, mtime: stat.mtime};
return;
}
// update latest if mtime is greater than the current latest
if (stat.mtime > latest.mtime) {
latest.filename = filename;
latest.mtime = stat.mtime;
}
});
return latest.filename;
}
Unfortunately, I don't think the files are guaranteed to be in any particular order.
Instead, you'll need to call fs.stat (or fs.statSync) on each file to get the date it was last modified, then select the newest one once you have all of the dates.
Surprisingly, there is no example in this questions that explicitly uses Array functions, functional programming.
Here is my take on getting the latest file of a directory in nodejs. By default, it will get the latest file by any extension. When passing the extension property, the function will return the latest file for that extension.
The advantage of this code is that its declarative and modular and easy to understand as oppose to using "logic branching/control flows", of course given you understand how these array functions work 😀
const fs = require('fs');
const path = require('path');
function getLatestFile({directory, extension}, callback){
fs.readdir(directory, (_ , dirlist)=>{
const latest = dirlist.map(_path => ({stat:fs.lstatSync(path.join(directory, _path)), dir:_path}))
.filter(_path => _path.stat.isFile())
.filter(_path => extension ? _path.dir.endsWith(`.${extension}`) : 1)
.sort((a, b) => b.stat.mtime - a.stat.mtime)
.map(_path => _path.dir);
callback(latest[0]);
});
}
getLatestFile({directory:process.cwd(), extension:'mp3'}, (filename=null)=>{
console.log(filename);
});
with synchronized version of read directory (fs.readdirSync) and file status (fs.statSync):
function getNewestFile(dir, regexp) {
newest = null
files = fs.readdirSync(dir)
one_matched = 0
for (i = 0; i < files.length; i++) {
if (regexp.test(files[i]) == false)
continue
else if (one_matched == 0) {
newest = files[i]
one_matched = 1
continue
}
f1_time = fs.statSync(files[i]).mtime.getTime()
f2_time = fs.statSync(newest).mtime.getTime()
if (f1_time > f2_time)
newest[i] = files[i]
}
if (newest != null)
return (dir + newest)
return null
}
you can call this function as follows:
var f = getNewestFile("./", new RegExp('.*\.mp3'))
An async version of #pguardiario's functional answer (I did this myself then found theirs halfway down the page when I went to add this).
import {promisify} from 'util';
import _glob from 'glob';
const glob = promisify(_glob);
const newestFile = (await Promise.all(
(await glob(YOUR_GLOB)).map(async (file) => (
{file, mtime:(await fs.stat(file)).mtime}
))
))
.sort(({mtime:a}, {mtime:b}) => ((a < b) ? 1 : -1))
[0]
.file
;
This is a commonplace need - write files out to a temp dir and automatically open the most recent one.
The following works with node version 16:
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const cp = require('child_process');
const fsPromises = fs.promises;
process.on('exit', code => {
console.log('exiting with code:', code);
});
const folder = path.join(process.env.HOME, 'publications/temp');
const c = cp.spawn('sh');
const files = fs.readdirSync(folder).map(v => path.resolve(folder + '/' + v));
const openFile = (file) => {
c.stdin.end(`(open "${file}" &> /dev/null) &> /dev/null &`);
};
if(files.length > 500) {
console.error('too many files, clean this folder up lol');
process.exit(1);
}
const newest = {file: null, mtime: null};
Promise.all(files.map(f => {
return fsPromises.stat(f).then(stats => {
if (!newest.file || (stats.mtime.getTime() > newest.mtime.getTime())) {
newest.file= f;
newest.mtime= stats.mtime;
}
});
})).then(v => {
if(!newest.file){
console.error('could not find the newest file?!');
return;
}
openFile(newest.file);
});
you may want to check for folders instead of files, and you could add something like this towards the beginning:
if (files.length === 1) {
if (fs.statSync(files[0]).isFile()) {
openFile(files[0]);
process.exit(0);
} else {
console.error('folder or symlink where file should be?');
process.exit(1);
}
}

Resources