convert tree-manager npm module (filetree) for koajs / co compatibility - node.js

How do I get a list of files compatible with co/yield (for koajs) ?
I am trying to convert this module for koa/co: https://www.npmjs.com/package/tree-manager
The original function is
fileModule.prototype.walkDir = function(dir, done) {
var self = this;
var results = [];
fs.readdir(dir, function(err, list) {
if (err) {
return done(err);
}
console.log(pending)
var pending = list.length;
if(!pending) {
return done(null, results);
}
list.forEach(function(file) {
var dfile = path.join(dir, file);
var el = {};
var fid = path.join(dir.replace(self.root, ''), file);
el.text = file;
el.id = fid;
fs.stat(dfile, function(err, stat) {
if(err) {
throw err;
}
if(stat.isDirectory()) {
return self.walkDir(dfile, function(err, res) {
el.children = res;
results.push(el);
!--pending && done(null, results);
});
}
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid};
results.push(el);
!--pending && done(null, results);
});
});
});
}
I can replace require('fs') with co-fs-plus (or extra)
so i can remove all fs callbacks with simple yield fs.xxx
but i don t understand the foreach loop :/
UPDATE (solution):
so ...
add wrap to co and adding dfile to the isDirectory function seems working
fileModule.prototype.walkDir = co.wrap(function*(dir) {
var self = this;
var list = yield fs.readdirAsync(dir);
// yield a list of promises
// created by mapping with an asynchronous function
var results = yield list.map(co.wrap(function*(file) {
var dfile = path.join(dir, file);
var fid = path.join(dir.replace(self.root, ''), file);
var el = {
text: file,
id: fid
};
try {
if (yield fs.isDirectoryAsync(dfile)) {
el.children = yield self.walkDir(dfile);
} else {
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid};
}
return el;
} catch(err) {
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid+' !! FILE UNREADABLE !!'};
return el;
}
}));
return results;
})
Thanks !

That loops runs all the actions in parallel, that's why it's so complicated (with the pending thingy and all). However, doing things in parallel is not really a strength of co, you should look into promises for that.
import fs from 'fs-extra-promise';
fileModule.prototype.walkDir = co(function*(dir) {
var self = this;
var list = yield fs.readdirAsync(dir);
// yield a list of promises
// created by mapping with an asynchronous function
var results = yield list.map(co(function*(file) {
var dfile = path.join(dir, file);
var fid = path.join(dir.replace(self.root, ''), file);
var el = {
text: file,
id: fid
};
if (yield fs.isDirectoryAsync()) {
el.children = yield self.walkDir(dfile);
} else {
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid};
}
return el;
}));
return results;
});

Related

Node.js : Call function using value from callback or async

I have written below .js file to call below defined function.
objectRepositoryLoader.readObjectRepository() returns me a hashmap from where i have to use values in enterUserName(), enterPassword(), clickLoginButton() functions.
var path = require('path');
var elementRepoMap = {}
var LandingPage = function(){
var fileName = path.basename(module.filename, path.extname(module.filename))
objectRepositoryLoader.readObjectRepository(fileName+'.xml' , function(elementRepo){
console.log(elementRepo) //values are being printed here
this.elementRepoMap = elementRepo
});
this.enterUserName = function(value){
console.log(elementRepoMap) //values are not being printed here
//Some Code
};
this.enterPassword = function(value){
//Some Code
};
this.clickLoginButton = function(){
//Some Code
};
};
module.exports = new LandingPage();
The objectRepositoryLoader.readObjectRepository() function defined in another file is as below:
var ObjectRepositoryLoader = function() {
this.readObjectRepository = function(fileName, callback) {
var filePath = './elementRepository/'+fileName;
this.loadedMap = this.objectRepoLoader(filePath, function(loadedMap){
return callback(loadedMap);
});
}
this.objectRepoLoader = function(filePath, callback){
if (filePath.includes(".xml")) {
this.xmlObjectRepositoryLoader(filePath, function(loadedMap){
return callback(loadedMap);
});
}
this.xmlObjectRepositoryLoader = function (xmlPath, callback){
var innerMap = {};
var elementName;
fs.readFile(xmlPath, "utf-8",function(err, data) {
if(err){
console.log('File not found!!')
}
else{
var doc = domparser.parseFromString(data,"text/xml");
var elements = doc.getElementsByTagName("A1");
for(var i =0 ; i< elements.length;i++){
var elm = elements[i];
elementName = elm.getAttribute("name");
var params = elm.getElementsByTagName("AS");
innerMap = {};
for(var j =0 ; j< params.length;j++){
var param = params[j];
var locatorType = param.getAttribute("type");
var locatorValue = param.getAttribute("value");
innerMap[locatorType] = locatorValue;
}
loadedMap[elementName] = innerMap;
innerMap={};
};
}
return callback(loadedMap);
});
};
How can I call enterUserName(), enterPassword(), clickLoginButton() function from spec.js file and is there any way I can avoid using callback and use async.js and call enterUserName(), enterPassword(), clickLoginButton() from spec.js file ?
EDIT
I have modified my file like below:
this.xmlObjectRepositoryLoader = function (xmlPath){
var innerMap = {};
var elementName;
var filePath = xmlPath+'.xml'
var self = this
return new Promise(
function(resolve, reject){
console.log("In xmlObjectRepositoryLoader : "+filePath)
self.readFilePromisified(filePath)
.then(text => {
var doc = domparser.parseFromString(text,"text/xml");
var elements = doc.getElementsByTagName("Element");
for(var i =0 ; i< elements.length;i++){
var elm = elements[i];
elementName = elm.getAttribute("name");
var params = elm.getElementsByTagName("param");
innerMap = {};
for(var j =0 ; j< params.length;j++){
var param = params[j];
var locatorType = param.getAttribute("type");
var locatorValue = param.getAttribute("value");
innerMap[locatorType] = locatorValue;
}
map[elementName] = innerMap;
innerMap={};
}
console.log(map) // prints the map
resolve(text)
})
.catch(error => {
reject(error)
});
});
}
this.readFilePromisified = function(filename) {
console.log("In readFilePromisified : "+filename)
return new Promise(
function (resolve, reject) {
fs.readFile(filename, { encoding: 'utf8' },
(error, data) => {
if (error) {
reject(error);
} else {
resolve(data);
}
})
})
}
I am calling above function from another file as below:
objectRepositoryLoader.readObjectRepository(fileName)
.then(text => {
console.log(text);
})
.catch(error => {
console.log(error);
});
But it gives me error as
.then(text => { ^
TypeError: Cannot read property 'then' of undefined
In this case how can I use promise to call another promise function and then use the returned value in one more promise function and return calculated value to calling function where I can use the value in other functions. I sound a bit confused. Please help
You can use async.waterfall and async.parallel to perform this task
see the reference
I just tried your code to make it working, I explained the way of implementation in comment.
async.waterfall([
function(next){
objectRepositoryLoader.readObjectRepository(fileName+'.xml' ,next)//pass this next as parameter in this function defination and after manipulation return result with callback like this(null,result)
}
],function(err,result){
if(!err){
//Do wahtever you want with result
async.parallel([
function(callback){
this.enterUserName = function(value){
console.log(elementRepoMap)
//Some Code
};
},
function(callback){
this.enterPassword = function(value){
//Some Code
};
},
function(callback){
this.clickLoginButton = function(){
//Some Code
};
}
], function(err, results) {
// optional callback
};
}
})

How to traverse all files, and support pause and continue

I have created a NodeJS (electron) code for read all the files in a specific directory and subdirectories.
I don't want to use too much HD resources, that why I use a delay of 5ms between folders.
Now my question. I want the if my NODE process stop? I want to be able to continue from when it is stopped. How should I do that?
In other words: How to keep index of current state while walking in all files and folder, so I can continue the traversing from when it has stopped.
Thank you
My Code:
var walkAll=function(options){
var x=0
walk(options.dir,function(){})
function walk(dir,callback) {
var files=fs.readdirSync(dir);
var stat;
async.eachSeries(files,function(file,next){
file=dir +'/' + file
if (dir.match(/Recycle/)) return next()
if (dir.match(/.git/)) return next()
if (dir.match(/node_modules/)) return next()
fs.lstat(file,function(err,stat){
if(err) return next()
if(stat.mode==41398) return next()
if (stat.isDirectory()) {
setTimeout(function(file){
walk(file,next)
}.bind(null,file),5)
}
else{
x++
if(false || x % 1000===0) console.log((new Date().valueOf()-start)/1000,x,file)
next()
}
})
},function(){
callback()
})
}
}
walkAll({
dir:'c:/',
delay:1000
});
Keep a list of sub directories to be visited, and update the list every iteration.
The walk function in the following example takes a previous state, and returns files of next sub directory with next state.
You can save the state before stopping the process, then load the saved state to continue the traversal when restarting.
function walk(state, readdir) {
let files = [], next = [];
while (state.length > 0) {
try {
const current = state.shift()
files = readdir(current).map(file => current + '/' + file)
next = state.concat(files)
break
} catch(e) {}
}
return [next, files]
}
function main() {
const {writeFileSync: writeFile, readdirSync: readdir} = require('fs')
const save = './walk.json'
let state
try {
state = require(save)
} catch(e) {}
if (!state || state.length < 1) state = ['.']
const [nextState, files] = walk(state, readdir)
console.log(files)
writeFile(save, JSON.stringify(nextState, null, 2))
}
main()
an alternate idea,
var miss = require('mississippi')
var fs = require("fs")
var through2 = require("through2")
var path = require("path")
function traverseDir(dirPath) {
var stack = [path.resolve(dirPath)];
var filesStack = []
return miss.from.obj(function(size, next) {
if (filesStack.length) {
return next(null, filesStack.shift())
}
var self = this;
try {
while(stack.length) {
readADir(stack.pop()).forEach(function (f) {
if (f.t=="d") {
stack.push(f.p)
}
filesStack.push(f)
})
if (filesStack.length) {
return next(null, filesStack.shift())
}
}
return next(null, null)
}catch(ex) {
return next(ex)
}
})
}
function readADir (dir) {
return fs.readdirSync(dir)
.map(function (f) {return path.join(dir, f)})
.filter(function (f) { return !f.match(/\.git/) })
.filter(function (f) { return !f.match(/Recycle/)})
.filter(function (f) { return !f.match(/node_modules/)})
.map(function (p) {
try {
var stat = fs.lstatSync(p);
if(stat.mode==41398) return null
var t = stat.isDirectory() ? "d":"f"
return { t: t, p: p }
}catch (ex) {}
return null
})
.filter(function (o) {return o!==null})
}
function loadState(base){
base = path.resolve(base)
var state = {base: base, last:null}
if (fs.existsSync("state.json")) {
state = JSON.parse(fs.readFileSync("state.json"))
} else {
saveState(state)
}
return state
}
function saveState(state){
fs.writeFileSync("state.json", JSON.stringify(state))
}
var state = loadState("..")
var sincePath = state.last;
var filesStream = traverseDir(state.base)
.on('end', function () {
console.log("end")
})
.pipe(through2.obj(function (chunk, enc, next) {
if(!sincePath) this.push(chunk)
if(chunk.p===sincePath) {
sincePath=null
}
next()
}))
var tr = through2.obj(function (chunk, enc, next) {
state.last = chunk.p
saveState(state)
console.log("data %v %j", chunk.t, chunk.p)
this.push(chunk)
setTimeout(next, 500)
}).resume()
require('keypress')(process.stdin);
process.stdin.on('keypress', function (ch, key) {
if(!key) return
if (key.name == "c") {
console.log("continue")
filesStream.pipe(tr)
} else if (key.name=="p") {
console.log("pause")
filesStream.unpipe(tr)
}
});
console.log("Press 'c' to start")

Return list from asynchronous function with nodejs

I have the following code :
var fs = require("fs");
function getMediaList(){
var media_copy_list, line_list;
media_copy_list = [];
fs.readFile("input.csv", function(err, data) {
line_list = data.toString('utf-8').trim().split('\n');
return line_list.forEach(function(file_line) {
var output_path, source_path, split_list;
if (file_line.length) {
split_list = file_line.split(';');
console.log(split_list[0]);
if (split_list.length >= 2) {
source_path = split_list[0].toString('utf-8').trim();
output_path = split_list[1].toString('utf-8').trim();
media_copy_list.push({
source: source_path,
destination: output_path
});
}
}
});
});
}
You can see that that I'm filling a list with :
media_copy_list.push({
source: source_path,
destination: output_path
});
What I'd like to do is to return this list once I have finished reading the input.csv file.
I don't have any issues if I read the file synchrnously( just have to call return media_copy_list). But in this case , I don't know.
I heard about async.parallel but really don't know how to apply.
Example of input.csv :
FirstPart;SecondPart
Test/test2;Whatever/example
Just wrap your code inside a promise and resolve it only once you're done. Some suggest callbacks, which does pretty much the same thing, but this pattern is discouraged, now. You should really use a promise.
var fs = require("fs");
function getMediaList(file){
return new Promise(function (resolve, reject) {
fs.readFile(file, 'utf-8', function(err, data) {
if (err) {
return reject(err);
}
resolve(data.split('\n').reduce(function(media_copy_list, file_line) {
var output_path;
var source_path;
var split_list;
file_line = file_line.trim();
if (file_line.length) {
split_list = file_line.split(';');
console.log(split_list[0]);
if (split_list.length >= 2) {
source_path = split_list[0].toString('utf-8').trim();
output_path = split_list[1].toString('utf-8').trim();
media_copy_list.push({
source: source_path,
destination: output_path
});
}
}
return media_copy_list;
}, []));
});
});
}
Then, invoke with
getMediaList('input.csv').then(function (mediaList) {
// ...
}).catch(function (err) {
console.error(err.stack);
});
Note: bluebird, Q, etc. are quite unnecessary since Node 4.2+. Unless you are using an earlier version of Node, try to avoid them. IMO.
The reason why Promises are encouraged is because Node will implement async/await, which will allow you to call this exact same function like :
var mediaList = await getMediaList('input.csv');
As noted in the comments, you don't want to return the list from the function.. what you should do is include a callback as a parameter to getMediaList and call that callback with your results. I would use async.each for looping through the lines in the file. You can read more about async.each here: https://github.com/caolan/async#each. Here is an example:
var fs = require("fs");
function getMediaList(callback){
var media_copy_list, line_list;
media_copy_list = [];
fs.readFile("input.csv", function(err, data) {
if(err) {
return callback(err);
}
line_list = data.toString('utf-8').trim().split('\n');
async.each(line_list, function(file_line, next) {
var output_path, source_path, split_list;
if (file_line.length) {
split_list = file_line.split(';');
console.log(split_list[0]);
if (split_list.length >= 2) {
source_path = split_list[0].toString('utf-8').trim();
output_path = split_list[1].toString('utf-8').trim();
media_copy_list.push({
source: source_path,
destination: output_path
});
}
}
next(err);
}, function (err) {
callback(err, media_copy_list);
}
});
}
Or you can use promises(bluebird in the case below).
var Promise = require('bluebird'),
fs = require("fs"),
media_copy_list, line_list,
media_copy_list = [];
fs.readFile("input.csv", function(err, data) {
line_list = data.toString('utf-8').trim().split('\n');
Promise.map(line_list, function(file_line) {
var output_path, source_path, split_list;
if (file_line.length) {
split_list = file_line.split(';');
if (split_list.length >= 2) {
source_path = split_list[0].toString('utf-8').trim();
output_path = split_list[1].toString('utf-8').trim();
media_copy_list = {
source: source_path,
destination: output_path
};
}
}
return media_copy_list
}).then(function(values){
console.log(values);
})
});

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

node.js glob pattern for excluding multiple files

I'm using the npm module node-glob.
This snippet returns recursively all files in the current working directory.
var glob = require('glob');
glob('**/*', function(err, files) {
console.log(files);
});
sample output:
[ 'index.html', 'js', 'js/app.js', 'js/lib.js' ]
I want to exclude index.html and js/lib.js.
I tried to exclude these files with negative pattern '!' but without luck.
Is there a way to achieve this only by using a pattern?
I suppose it's not actual anymore but I got stuck with the same question and found an answer.
This can be done using only glob module.
We need to use options as a second parameter to glob function
glob('pattern', {options}, cb)
There is an options.ignore pattern for your needs.
var glob = require('glob');
glob("**/*",{"ignore":['index.html', 'js', 'js/app.js', 'js/lib.js']}, function (err, files) {
console.log(files);
})
Check out globby, which is pretty much glob with support for multiple patterns and a Promise API:
const globby = require('globby');
globby(['**/*', '!index.html', '!js/lib.js']).then(paths => {
console.log(paths);
});
You can use node-globule for that:
var globule = require('globule');
var result = globule.find(['**/*', '!index.html', '!js/lib.js']);
console.log(result);
Or without an external dependency:
/**
Walk directory,
list tree without regex excludes
*/
var fs = require('fs');
var path = require('path');
var walk = function (dir, regExcludes, done) {
var results = [];
fs.readdir(dir, function (err, list) {
if (err) return done(err);
var pending = list.length;
if (!pending) return done(null, results);
list.forEach(function (file) {
file = path.join(dir, file);
var excluded = false;
var len = regExcludes.length;
var i = 0;
for (; i < len; i++) {
if (file.match(regExcludes[i])) {
excluded = true;
}
}
// Add if not in regExcludes
if(excluded === false) {
results.push(file);
// Check if its a folder
fs.stat(file, function (err, stat) {
if (stat && stat.isDirectory()) {
// If it is, walk again
walk(file, regExcludes, function (err, res) {
results = results.concat(res);
if (!--pending) { done(null, results); }
});
} else {
if (!--pending) { done(null, results); }
}
});
} else {
if (!--pending) { done(null, results); }
}
});
});
};
var regExcludes = [/index\.html/, /js\/lib\.js/, /node_modules/];
walk('.', regExcludes, function(err, results) {
if (err) {
throw err;
}
console.log(results);
});
Here is what I wrote for my project:
var glob = require('glob');
var minimatch = require("minimatch");
function globArray(patterns, options) {
var i, list = [];
if (!Array.isArray(patterns)) {
patterns = [patterns];
}
patterns.forEach(pattern => {
if (pattern[0] === "!") {
i = list.length-1;
while( i > -1) {
if (!minimatch(list[i], pattern)) {
list.splice(i,1);
}
i--;
}
}
else {
var newList = glob.sync(pattern, options);
newList.forEach(item => {
if (list.indexOf(item)===-1) {
list.push(item);
}
});
}
});
return list;
}
And call it like this (Using an array):
var paths = globArray(["**/*.css","**/*.js","!**/one.js"], {cwd: srcPath});
or this (Using a single string):
var paths = globArray("**/*.js", {cwd: srcPath});
A samples example with gulp:
gulp.task('task_scripts', function(done){
glob("./assets/**/*.js", function (er, files) {
gulp.src(files)
.pipe(gulp.dest('./public/js/'))
.on('end', done);
});
});

Resources