Async Recursive File System Traversal - node.js

I am trying to compile an array out of data present in some specific files, using a recursive approach to read directories and the file system methods are asynchronous. I am unable to figure out the apt place for the callback invocation.
const fs = require('fs');
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
function doThisOnceDone() {
console.log(nodes);
}
function readFile(path) {
fs.readFile(path + '/' + FILE_NAME,{
encoding:"UTF-8"
}, function(err, data) {
if(err) {
return;
}
nodes.push(data);
});
}
function compileArray(path, callback) {
fs.readdir(path, {
encoding:"UTF-8"
}, function(err, files) {
if(err) {
console.error(err);
return;
}
files.forEach(function(file) {
var nextPath = path + '/' + file;
fs.stat(nextPath, function(err, stats) {
if(err) {
return;
}
if(stats.isDirectory()) {
if(file === 'specific') {
readFile(nextPath);
}
else {
compileArray(nextPath);
}
}
});
});
});
}
compileArray(ENTRY, doThisOnceDone);
When do I know that the recursion tree has been done with , and I can access the nodes array ?

Try this
const util = require('util');
const fs = require('fs');
const stat = util.promisify(fs.stat);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
const compileArray = async (path) => {
try {
const files = await readdir(path);
files.forEach((file) => {
try {
var nextPath = path + '/' + file;
const stats = await stat(nextPath); //check other conditions
if (file === 'specific') {
const data = await readFile(path + '/' + FILE_NAME, {
encoding: "UTF-8"
});
nodes.push(data);
}
} catch (error) {
console.log(error);
}
});
} catch (error) {
console.log(error);
}
}

const util = require('util');
const fs = require('fs');
const stat = util.promisify(fs.stat);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
const compileArray = async (path) => {
try {
const files = await readdir(path);
files.forEach(async (file) => {
try {
var nextPath = path + '/' + file;
const stats = await stat(nextPath); //check other conditions
if (file === 'specific') {
const data = await readFile(path + '/' + FILE_NAME, {
encoding: "UTF-8"
});
nodes.push(data);
}
} catch (error) {
console.log(error);
}
});
} catch (error) {
console.log(error);
}
}
compileArray(ENTRY)

Related

Having problem with convert XML to XLSX and download it in Node.js

My problem is that i have to create Converter and this converter takes the XML files and converts it to XLSX and download it. i was able to do the first part where the XML convert to JSON, but i have difficult to do the rest of the code, now i am trying to take the file JSON and convert it to XLSX but my code is not working, i do not know why ?
ControllerXML2Json.js
const { json } = require('body-parser')
const fs = require('fs')
const xml2js = require('xml2js')
let jsonString
async function xml2json() {
try {
const xml = fs.readFileSync('src/data/dataset.xml')
const result = await xml2js.parseStringPromise(xml, { mergeAttrs: true })
jsonString = JSON.stringify(result, null, 4)
//console.log(json)
fs.writeFileSync('./public/data/result.json', jsonString)
//return jsonString
} catch (error) {
console.log(error)
}
}
module.exports = {
convert(req, res) {
let xmlFile = req.files.fileXML
xmlFile.mv('src/data/dataset.xml', function (err) {
if (err) {
return res.status(500).send(err)
} else {
xml2json()
}
})
res.redirect('/json')
}
}
ControllerJson2Xlsx.js
const { json } = require("body-parser");
const fs = require("fs");
const xlsx = require("json-as-xlsx");
const path = require('path');
const fileName = "data.xlsx";
async function generateExcelFromJSON(){
try {
let content = JSON.parse(fs.readFileSync("./data/result.json", "utf-8"));
let newWS = xlsx.utils.book_new();
let newWB = xlsx.utils.json_to_sheet(content);
xlsx.utils.book_append_sheet(newWB, newWS, "data");
await xlsx.writeFile("./public/data/data.xlsx", content);
// xlsx.writeFile(newWB, "./public/data/data.xlsx");
} catch (error) {
console.log(error);
}
}
module.exports = {
download(req, res) {
let jsonFile = req.files.fileJSON;
jsonFile.mv('./data/data.xlsx', function (err) {
if (err) {
return res.status(500).send(err);
} else {
xlsx();
}
});
res.redirect("/xlsx");
},
};

Finding the differences between 2 files in nodejs

I have 2 json files which have static data.
File A.json
{
"data": ["A","B","C","D"]
}
File B.json
{
"data":["A","B","C"]
}
Now I want find the difference between these two files.
I tried using this code to get files by its name
const express = require("express");
const jsonfile = require("jsonfile");
const app = express();
const fs = require("fs");
app.use(express.json());
const BASEPATH = "./data"
app.get("/api/v1/data/:name", async (req,res) => {
const fileName = req.params.name;
const filePath = `${BASEPATH}/${fileName}.json`
try {
const readData = await jsonfile.readFile(filePath);
res.status(200).json(readData);
}catch (e) {
res.status(404).send(e);
}
});
The URL is : localhost:3000/api/v1/data/A
To add data I used,
app.put("/api/v1/data",(req,res) => {
const fileName = req.body.name;
const data = req.body.data;
const filePath = `${BASEPATH}/${fileName}.json`
fs.exists(filePath, async exists => {
if(exists) {
try {
await jsonfile.writeFile(filePath,{data:data});
res.status(200).send();
}catch(e) {
res.send(500).json(e);
}
} else {
try {
await jsonfile.writeFile(filePath,{data:data});
res.status(201).send();
}catch(e) {
res.send(500).json(e);
}
}
})
});
Example of added data:
{ "name":"C", "data":["A","B","Z"]}
URL is: localhost:3000/api/v1/data
you can try like this,
app.get("/api/v1/data/diff", async (req,res) => {
try {
const file1 = req.query.file1;
const file2 = req.query.file2;
console.log(file1,file2)
if(file1 === undefined || file2 === undefined ) {
res.status(401).send("BAD REQUEST. SEND FILE NAMES");
return;
} else {
const filePath1 = `${BASEPATH}/${file1}.json`
const filePath2 = `${BASEPATH}/${file2}.json`
const file1Data = await jsonfile.readFile(filePath1);
const file2Data = await jsonfile.readFile(filePath2);
let difference = file1Data.data.filter(x => !file2Data.data.includes(x));
let difference2 = file2Data.data.filter(x => !file1Data.data.includes(x));
res.status(200).json({diff: [...difference,...difference2]});
}
}catch(e) {
res.status(500).json(e);
}
});
you have to use the url like : localhost:3000/api/v1/data/diff?file1=A&file2=B
the output will be
{
"diff": [
"D"
]
}

How to convert all djvu files to pdf

it's answer. Just use nodejs and ddjvu from DJView lib.
There
imports
const fs = require('fs');
const os = require('os');
const {spawn} = require('child_process');
const path = require('path');
const maxProcess = os.cpus().length - 1;// count of procces - 1 for system needs
let nowPlayed = 0;
method for convert file, and delete when converted.
function chpoc(args) {
console.log(args[1] + " start converting");
spawn(`ddjvu`, ["-format=pdf", args[0], args[1] + ".pdf"]).on('close', (data) => {
console.log(args[1] + ".pdf converted");
fs.unlink(args[0], (err) => {
if (err) throw err;
console.log(args[0] + ' successfully deleted!');
nowPlayed--;
})
});
}
queue for optimize max convertions at one time
let queue = [];
function startQueue() {
if (nowPlayed < maxProcess && queue.length) {
nowPlayed++;
queue.pop()();
}
}
setInterval(startQueue, 500)
fillthe queue and start it
function workWithFile(filepath) {
const args = filepath.match(/(.*)\.djvu/)
if (args && args.length) {
queue.push(() => {
chpoc(args);
});
}
}
show errors
const eachCallback = function (err) {
err && console.error(err);
}
catalog three and finde the djvus
let filePaths = [];
function getFiles(dirPath, callback) {
fs.readdir(dirPath, function (err, files) {
if (err) return callback(err);
files.forEach((fileName) => {
setTimeout(() => {
let filePath = path.join(dirPath, fileName);
if (filePath) {
fs.stat(filePath, function (err, stat) {
if (err) return eachCallback(err);
if (stat.isDirectory()) {
getFiles(filePath, callback);
} else if (stat.isFile() && /\.djvu$/.test(filePath)) {
filePaths.push(filePath);
callback(filePath)
}
})
}
});
});
});
}
init from started dir
getFiles(__dirname, function (file) {
workWithFile(file);
});
imports
const fs = require('fs');
const os = require('os');
const {spawn} = require('child_process');
const path = require('path');
const maxProcess = os.cpus().length - 1;// count of procces - 1 for system needs
let nowPlayed = 0;
method for convert file, and delete when converted.
function chpoc(args) {
console.log(args[1] + " start converting");
spawn(`ddjvu`, ["-format=pdf", args[0], args[1] + ".pdf"]).on('close', (data) => {
console.log(args[1] + ".pdf converted");
fs.unlink(args[0], (err) => {
if (err) throw err;
console.log(args[0] + ' successfully deleted!');
nowPlayed--;
})
});
}
queue for optimize max convertions at one time let queue = [];
function startQueue() {
if (nowPlayed < maxProcess && queue.length) {
nowPlayed++;
queue.pop()();
}
}
setInterval(startQueue, 500)
fill the queue and start it
function workWithFile(filepath) {
const args = filepath.match(/(.*)\.djvu/)
if (args && args.length) {
queue.push(() => {
chpoc(args);
});
}
}
show errors
const eachCallback = function (err) {
err && console.error(err);
}
catalog three and finde the djvus
let filePaths = [];
function getFiles(dirPath, callback) {
fs.readdir(dirPath, function (err, files) {
if (err) return callback(err);
files.forEach((fileName) => {
setTimeout(() => {
let filePath = path.join(dirPath, fileName);
if (filePath) {
fs.stat(filePath, function (err, stat) {
if (err) return eachCallback(err);
if (stat.isDirectory()) {
getFiles(filePath, callback);
} else if (stat.isFile() && /\.djvu$/.test(filePath)) {
filePaths.push(filePath);
callback(filePath)
}
})
}
});
});
});
}
init from started dir
getFiles(__dirname, function (file) {
workWithFile(file);
});

How to read all file with specified ext in a dir?

I am new to nodejs and want this to work.
var fs = require('fs');
fs.readdir("dir", function(err, files) {
if (err) return;
files.forEach(function(f) {
data = []
if f.extension = "rtf"
data = data + f.data
});
});
You can try this :
const fs = require('fs');
const path = require('path');
fs.readdir("dir", (err, files) => {
if (err) return;
files.forEach(f => {
let data = []
const ext = path.extname(file)
if (ext == ".rtf") {
fs.readFile(f, function read(err, content) {
if (err) {
throw err;
}
data.push(content);
});
}
});
});
You will have each content of the files under the array data. But it will be better to put it into an object to know where the content come from like this :
const fs = require('fs');
const path = require('path');
fs.readdir("dir", (err, files) => {
if (err) return;
files.forEach(f => {
let data = {}
const ext = path.extname(file)
if (ext == ".rtf") {
fs.readFile(f, function read(err, content) {
if (err) {
throw err;
}
data[f] = content;
});
}
});
});

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

Resources