How to read all file with specified ext in a dir? - node.js

I am new to nodejs and want this to work.
var fs = require('fs');
fs.readdir("dir", function(err, files) {
if (err) return;
files.forEach(function(f) {
data = []
if f.extension = "rtf"
data = data + f.data
});
});

You can try this :
const fs = require('fs');
const path = require('path');
fs.readdir("dir", (err, files) => {
if (err) return;
files.forEach(f => {
let data = []
const ext = path.extname(file)
if (ext == ".rtf") {
fs.readFile(f, function read(err, content) {
if (err) {
throw err;
}
data.push(content);
});
}
});
});
You will have each content of the files under the array data. But it will be better to put it into an object to know where the content come from like this :
const fs = require('fs');
const path = require('path');
fs.readdir("dir", (err, files) => {
if (err) return;
files.forEach(f => {
let data = {}
const ext = path.extname(file)
if (ext == ".rtf") {
fs.readFile(f, function read(err, content) {
if (err) {
throw err;
}
data[f] = content;
});
}
});
});

Related

How to convert all djvu files to pdf

it's answer. Just use nodejs and ddjvu from DJView lib.
There
imports
const fs = require('fs');
const os = require('os');
const {spawn} = require('child_process');
const path = require('path');
const maxProcess = os.cpus().length - 1;// count of procces - 1 for system needs
let nowPlayed = 0;
method for convert file, and delete when converted.
function chpoc(args) {
console.log(args[1] + " start converting");
spawn(`ddjvu`, ["-format=pdf", args[0], args[1] + ".pdf"]).on('close', (data) => {
console.log(args[1] + ".pdf converted");
fs.unlink(args[0], (err) => {
if (err) throw err;
console.log(args[0] + ' successfully deleted!');
nowPlayed--;
})
});
}
queue for optimize max convertions at one time
let queue = [];
function startQueue() {
if (nowPlayed < maxProcess && queue.length) {
nowPlayed++;
queue.pop()();
}
}
setInterval(startQueue, 500)
fillthe queue and start it
function workWithFile(filepath) {
const args = filepath.match(/(.*)\.djvu/)
if (args && args.length) {
queue.push(() => {
chpoc(args);
});
}
}
show errors
const eachCallback = function (err) {
err && console.error(err);
}
catalog three and finde the djvus
let filePaths = [];
function getFiles(dirPath, callback) {
fs.readdir(dirPath, function (err, files) {
if (err) return callback(err);
files.forEach((fileName) => {
setTimeout(() => {
let filePath = path.join(dirPath, fileName);
if (filePath) {
fs.stat(filePath, function (err, stat) {
if (err) return eachCallback(err);
if (stat.isDirectory()) {
getFiles(filePath, callback);
} else if (stat.isFile() && /\.djvu$/.test(filePath)) {
filePaths.push(filePath);
callback(filePath)
}
})
}
});
});
});
}
init from started dir
getFiles(__dirname, function (file) {
workWithFile(file);
});
imports
const fs = require('fs');
const os = require('os');
const {spawn} = require('child_process');
const path = require('path');
const maxProcess = os.cpus().length - 1;// count of procces - 1 for system needs
let nowPlayed = 0;
method for convert file, and delete when converted.
function chpoc(args) {
console.log(args[1] + " start converting");
spawn(`ddjvu`, ["-format=pdf", args[0], args[1] + ".pdf"]).on('close', (data) => {
console.log(args[1] + ".pdf converted");
fs.unlink(args[0], (err) => {
if (err) throw err;
console.log(args[0] + ' successfully deleted!');
nowPlayed--;
})
});
}
queue for optimize max convertions at one time let queue = [];
function startQueue() {
if (nowPlayed < maxProcess && queue.length) {
nowPlayed++;
queue.pop()();
}
}
setInterval(startQueue, 500)
fill the queue and start it
function workWithFile(filepath) {
const args = filepath.match(/(.*)\.djvu/)
if (args && args.length) {
queue.push(() => {
chpoc(args);
});
}
}
show errors
const eachCallback = function (err) {
err && console.error(err);
}
catalog three and finde the djvus
let filePaths = [];
function getFiles(dirPath, callback) {
fs.readdir(dirPath, function (err, files) {
if (err) return callback(err);
files.forEach((fileName) => {
setTimeout(() => {
let filePath = path.join(dirPath, fileName);
if (filePath) {
fs.stat(filePath, function (err, stat) {
if (err) return eachCallback(err);
if (stat.isDirectory()) {
getFiles(filePath, callback);
} else if (stat.isFile() && /\.djvu$/.test(filePath)) {
filePaths.push(filePath);
callback(filePath)
}
})
}
});
});
});
}
init from started dir
getFiles(__dirname, function (file) {
workWithFile(file);
});

Async Recursive File System Traversal

I am trying to compile an array out of data present in some specific files, using a recursive approach to read directories and the file system methods are asynchronous. I am unable to figure out the apt place for the callback invocation.
const fs = require('fs');
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
function doThisOnceDone() {
console.log(nodes);
}
function readFile(path) {
fs.readFile(path + '/' + FILE_NAME,{
encoding:"UTF-8"
}, function(err, data) {
if(err) {
return;
}
nodes.push(data);
});
}
function compileArray(path, callback) {
fs.readdir(path, {
encoding:"UTF-8"
}, function(err, files) {
if(err) {
console.error(err);
return;
}
files.forEach(function(file) {
var nextPath = path + '/' + file;
fs.stat(nextPath, function(err, stats) {
if(err) {
return;
}
if(stats.isDirectory()) {
if(file === 'specific') {
readFile(nextPath);
}
else {
compileArray(nextPath);
}
}
});
});
});
}
compileArray(ENTRY, doThisOnceDone);
When do I know that the recursion tree has been done with , and I can access the nodes array ?
Try this
const util = require('util');
const fs = require('fs');
const stat = util.promisify(fs.stat);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
const compileArray = async (path) => {
try {
const files = await readdir(path);
files.forEach((file) => {
try {
var nextPath = path + '/' + file;
const stats = await stat(nextPath); //check other conditions
if (file === 'specific') {
const data = await readFile(path + '/' + FILE_NAME, {
encoding: "UTF-8"
});
nodes.push(data);
}
} catch (error) {
console.log(error);
}
});
} catch (error) {
console.log(error);
}
}
const util = require('util');
const fs = require('fs');
const stat = util.promisify(fs.stat);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
const compileArray = async (path) => {
try {
const files = await readdir(path);
files.forEach(async (file) => {
try {
var nextPath = path + '/' + file;
const stats = await stat(nextPath); //check other conditions
if (file === 'specific') {
const data = await readFile(path + '/' + FILE_NAME, {
encoding: "UTF-8"
});
nodes.push(data);
}
} catch (error) {
console.log(error);
}
});
} catch (error) {
console.log(error);
}
}
compileArray(ENTRY)

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

node phantomjs seo running via node write the file

I'm wondering if this is the write way:
node.js
var fs = require('fs');
var path = require('path');
var childProcess = require('child_process');
var phantomjs = require('phantomjs');
var binPath = phantomjs.path;
var childArgs = [
path.join(__dirname, 'phantomjs-runner.js'),
'http://localhost:3000'
]
childProcess.execFile(binPath, childArgs, function(err, stdout, stderr) {
if(err){
}
if(stderr){
}
fs.writeFile(__dirname+'/public/snapshots/index.html', stdout, function(err) {
if(err) {
console.log(err);
}
else {
console.log("The file was saved!");
}
});
});
phantomjs-runner.js
var system = require('system');
var url = system.args[1] || '';
if(url.length > 0) {
var page = require('webpage').create();
page.open(url, function (status) {
if (status == 'success') {
var delay, checker = (function() {
var html = page.evaluate(function () {
var body = document.getElementsByTagName('body')[0];
if(body.getAttribute('data-status') == 'ready') {
return document.getElementsByTagName('html')[0].outerHTML;
}
});
if(html) {
clearTimeout(delay);
console.log(html);
phantom.exit();
}
});
delay = setInterval(checker, 100);
}
});
}
may be could be a better way like
clearTimeout(delay);
fs.writeFile
phantom.exit();
How can I manage ie
different urls and different files
I mean
http://localhost:3000 index.html
http://localhost:3000/blog blog.html
http://localhost:3000/blog/postid postid.html
ENDED UP
'use strict';
var fs = require('fs'),
path = require('path'),
childProcess = require('child_process'),
phantomjs = require('phantomjs'),
binPath = phantomjs.path,
config = require('../config/config');
var env = (process.env.NODE_ENV === 'production') ? 'production' : null,
currentPath = (env) ? config.proot + '/build/default/snapshots' : config.proot + '/default/snapshots';
function normalizeUrl(url){
if( (typeof url === 'undefined') || !url){
return '';
}
if ( url.charAt( 0 ) === '/' ){
url = url.substring(1);
}
return '/'+url.replace(/\/$/, "");
}
function normalizePage(route){
if(!route){
return 'index';
}
var chunks = route.substring(1).split('/');
var len = chunks.length;
if(len===1){
return chunks[0];
}
chunks.shift();
//I get the id
return chunks[0];
}
module.exports = function (url) {
var route = normalizeUrl(url);
var page = normalizePage(route);
var childArgs = [
path.join(__dirname, './phantomjs-runner.js'),
config.url+route
];
childProcess.execFile(binPath, childArgs, function(err, stdout, stderr) {
if(err){
}
if(stderr){
}
fs.writeFile(currentPath + '/' + page + '.html', stdout, function(err) {
if(err) {
console.log(err);
}
else {
console.log("The file was saved!");
}
});
});
};
So I worked out for the parameters I'm still
waiting for way to get the output ^^

How to get totalsize of files in directory?

How to get totalsize of files in directory ? Best way ?
Here is a simple solution using the core Nodejs fs libraries combined with the async library. It is fully asynchronous and should work just like the 'du' command.
var fs = require('fs'),
path = require('path'),
async = require('async');
function readSizeRecursive(item, cb) {
fs.lstat(item, function(err, stats) {
if (!err && stats.isDirectory()) {
var total = stats.size;
fs.readdir(item, function(err, list) {
if (err) return cb(err);
async.forEach(
list,
function(diritem, callback) {
readSizeRecursive(path.join(item, diritem), function(err, size) {
total += size;
callback(err);
});
},
function(err) {
cb(err, total);
}
);
});
}
else {
cb(err);
}
});
}
I tested the following code and it works perfectly fine.
Please do let me know if there is anything that you don't understand.
var util = require('util'),
spawn = require('child_process').spawn,
size = spawn('du', ['-sh', '/path/to/dir']);
size.stdout.on('data', function (data) {
console.log('size: ' + data);
});
// --- Everything below is optional ---
size.stderr.on('data', function (data) {
console.log('stderr: ' + data);
});
size.on('exit', function (code) {
console.log('child process exited with code ' + code);
});
Courtesy Link
2nd method:
var util = require('util'), exec = require('child_process').exec, child;
child = exec('du -sh /path/to/dir', function(error, stdout, stderr){
console.log('stderr: ' + stderr);
if (error !== null){
console.log('exec error: ' + error);
}
});
You might want to refer the Node.js API for child_process
Use du : https://www.npmjs.org/package/du
require('du')('/home/rvagg/.npm/', function (err, size) {
console.log('The size of /home/rvagg/.npm/ is:', size, 'bytes')
})
ES6 variant:
import path_module from 'path'
import fs from 'fs'
// computes a size of a filesystem folder (or a file)
export function fs_size(path, callback)
{
fs.lstat(path, function(error, stats)
{
if (error)
{
return callback(error)
}
if (!stats.isDirectory())
{
return callback(undefined, stats.size)
}
let total = stats.size
fs.readdir(path, function(error, names)
{
if (error)
{
return callback(error)
}
let left = names.length
if (left === 0)
{
return callback(undefined, total)
}
function done(size)
{
total += size
left--
if (left === 0)
{
callback(undefined, total)
}
}
for (let name of names)
{
fs_size(path_module.join(path, name), function(error, size)
{
if (error)
{
return callback(error)
}
done(size)
})
}
})
})
}
Review the node.js File System functions. It looks like you can use a combination of fs.readdir(path, [cb]), and fs.stat(file, [cb]) to list the files in a directory and sum their sizes.
Something like this (totally untested):
var fs = require('fs');
fs.readdir('/path/to/dir', function(err, files) {
var i, totalSizeBytes=0;
if (err) throw err;
for (i=0; i<files.length; i++) {
fs.stat(files[i], function(err, stats) {
if (err) { throw err; }
if (stats.isFile()) { totalSizeBytes += stats.size; }
});
}
});
// Figure out how to wait for all callbacks to complete
// e.g. by using a countdown latch, and yield total size
// via a callback.
Note that this solution only considers the plain files stored directly in the target directory and performs no recursion. A recursive solution would come naturally by checking stats.isDirectory() and entering, although it likely complicates the "wait for completion" step.
'use strict';
const async = require('async');
const fs = require('fs');
const path = require('path')
const getSize = (item, callback) => {
let totalSize = 0;
fs.lstat(item, (err, stats) => {
if (err) return callback(err);
if (stats.isDirectory()) {
fs.readdir(item, (err, list) => {
if (err) return callback(err);
async.each(list, (listItem, cb) => {
getSize(path.join(item, listItem), (err, size) => {
totalSize += size;
cb();
});
},
(err) => {
if (err) return callback(err);
callback(null, totalSize);
});
});
} else {
// Ensure fully asynchronous API
process.nextTick(function() {
callback(null, (totalSize += stats.size))
});
}
});
}
getSize('/Applications', (err, totalSize) => { if (!err) console.log(totalSize); });
I know I'm a bit late to the part but I though I'd include my solution which uses promises based on #maerics answer:
const fs = require('fs');
const Promise = require('bluebird');
var totalSizeBytes=0;
fs.readdir('storage', function(err, files) {
if (err) throw err;
Promise.mapSeries(files, function(file){
return new Promise((resolve, reject) => {
fs.stat('storage/' + file,function(err, stats) {
if (err) { throw err; }
if (stats.isFile()) { totalSizeBytes += stats.size; resolve(); }
});
})
}).then(()=>{
console.log(totalSizeBytes);
});
});
function readSizeRecursive(folder, nested = 0) {
return new Promise(function(resolve, reject) {
const stats = fs.lstatSync(path.resolve(__dirname, '../projects/', folder));
var total = stats.size;
const list = fs.readdirSync(path.resolve(__dirname, '../projects/', folder));
if(list.length > 0){
Promise.all(list.map(async li => {
const stat = await fs.lstatSync(path.resolve(__dirname, '../projects/', folder, li));
if(stat.isDirectory() && nested == 0){
const tt = await readSizeRecursive(folder, 1);
total += tt;
} else {
total += stat.size;
}
})).then(() => resolve(convertBytes(total)));
} else {
resolve(convertBytes(total));
}
});
}
const convertBytes = function(bytes) {
const sizes = ["Bytes", "KB", "MB", "GB", "TB"]
if (bytes == 0) {
return "n/a"
}
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)))
if (i == 0) {
return bytes + " " + sizes[i]
}
// return (bytes / Math.pow(1024, i)).toFixed(1) + " " + sizes[i]
return parseFloat((bytes / Math.pow(1024, i)).toFixed(1));
}
This combines async/await and the fs Promises API introduced in Node.js v14.0.0 for a clean, readable implementation:
const { readdir, stat } = require('fs/promises');
const dirSize = async directory => {
const files = await readdir( directory );
const stats = files.map( file => stat( path.join( directory, file ) ) );
let size = 0;
for await ( const stat of stats ) size += stat.size;
return size;
};
Usage:
const size = await dirSize( '/path/to/directory' );
console.log( size );
An shorter-but-less-readable alternative of the dirSize function would be:
const dirSize = async directory => {
const files = await readdir( directory );
const stats = files.map( file => stat( path.join( directory, file ) ) );
return ( await Promise.all( stats ) ).reduce( ( accumulator, { size } ) => accumulator + size, 0 );
}
A very simple synchronous solution that I implemented.
const fs = require("fs");
function getSize(path){
// Get the size of a file or folder recursively
let size = 0;
if(fs.statSync(path).isDirectory()){
const files = fs.readdirSync(path);
files.forEach(file => {
size += getSize(path + "/" + file);
});
}
else{
size += fs.statSync(path).size;
}
return size;
}

Resources