How to transition from async.parallel & series to composable Promises? - node.js

I am trying to transition from async to promises and this is what I have. If the code looks contrived it's because I simplified it from what I'm working on to make it easier to grasp. I'm struggling to get the Promise.all to execute.
I commented out the async code that I want to implement in promises:
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs-extra'));
var path = require('path');
var tar = require('tar-fs');
module.exports = Archive;
function Archive() {
var self = this;
var self.base_dir = '/bar/baz',
var self.file1 = 'foo/file1',
var self.file2 = 'foo/file2',
var self.file3 = 'foo/file3',
var self.file4 = 'foo/file4'
}
Archive.prototype.make = function(done) {
var self = this;
// async.series([
// function(next) {
// self._prepareFilesDir(next);
// },
// function(next) {
// self._copyFiles(next);
// },
// function(next) {
// self._writeArchive(next);
// }
// ], done)
self._prepareFilesDir().bind(self)
.then(self._copyFiles.bind(self))
.then(self._writeArchive.bind(self))
.catch(function(e) {
return done(e);
});
};
// ********************************
// * Private functions
// ********************************
Archive.prototype._prepareFilesDir = function() {
var self = this;
return fs.emptyDirAsync(self.base_dir);
};
Archive.prototype._copyFiles = function() {
var self = this;
var sources = {
file1: path.resolve('baz', 'file1'),
file2: path.resolve('baz', 'file2')
file3: path.resolve('baz', 'file3')
file4: path.resolve('baz', 'file4')
file5: path.resolve('baz', 'file5')
};
var destinations = {
file1: path.resolve(self.base_dir, self.file1),
file2: path.resolve(self.base_dir, self.file2),
file3: path.resolve(self.base_dir, self.file3),
file4: path.resolve(self.base_dir, self.file4),
file5: path.resolve(self.base_dir, self.file5)
};
var filters = {
qux: /^qux/,
bru: /^bru/,
blerg: /blerg$/
};
function copyFile1() {
console.log('hello world');
return fs.copyAsync(sources.file2, destinations.file1, { filter: filters.qux });
};
function copyFile2() {
return fs.copyAsync(sources.file2, destinations.file2);
};
function copyFile3() {
return fs.copyAsync(sources.file3, destinations.file3, { filter: filters.bru });
};
function copyFile4() {
return fs.copyAsync(sources.file4, destinations.file4, { filter: filters.blerg });
};
return Promise.all([
copyFile1,
copyFile2,
copyFile3,
copyFile4
]);
// async.parallel([
// copyFile1(next),
// copyFile2(next),
// copyFile3(next),
// copyFile4(next)
// ], function(err) {
// if (err) return done(err);
// done(null);
// })
};
Archive.prototype._writeArchive = function() {
var self = this;
var archive_dir_path = path.resolve(self.base_dir, '..');
var tarPromise = function() {
return new Promise(function(resolve, reject) {
tar.pack(self.files_path)
.pipe(fs.createWriteStream(archive_dir_path + '.tar'))
.on('error', reject)
.on('finish', resolve)
});
};
fs.ensureDirAsync(archive_dir_path)
.then(tarPromise);
};
I must be doing something wrong because the 'hello world' is never printed. I think the stream is promisified correctly but I'm not so sure either. I based my conversion on the promise-nuggets.github.io snippets.
How do I have to do the Promise.all? I'd like to keep separate functions as I think it helps understanding the code better.
Thanks,

the mistakes that I found:
in make method, done would be called only in case of error, suggestion remove done callback, just return promise
again in make, you are doing _prepareFilesDir().bind(self), for staters bind at that point is redundant, it should have been call/apply at that point.
in _writeArchive, you need to return promise, else it ll return undefined and assume that the async function is finished.
updated code in fiddle

There were several issues which #mido22 fixed. There is one more issue though, in Pormise.all(), the functions shouldn't be passed as references but rather executed.
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs-extra'));
var path = require('path');
var tar = require('tar-fs');
module.exports = Archive;
function Archive() {
var self = this;
var self.base_dir = '/bar/baz',
var self.file1 = 'foo/file1',
var self.file2 = 'foo/file2',
var self.file3 = 'foo/file3',
var self.file4 = 'foo/file4'
}
Archive.prototype.make = function() { // CHANGED
var self = this;
// async.series([
// function(next) {
// self._prepareFilesDir(next);
// },
// function(next) {
// self._copyFiles(next);
// },
// function(next) {
// self._writeArchive(next);
// }
// ], done)
return self._prepareFilesDir() // CHANGED
.then(self._copyFiles.bind(self))
.then(self._writeArchive.bind(self)); // CHANGED
};
// ********************************
// * Private functions
// ********************************
Archive.prototype._prepareFilesDir = function() {
var self = this;
return fs.emptyDirAsync(self.base_dir);
};
Archive.prototype._copyFiles = function() {
var self = this;
var sources = {
file1: path.resolve('baz', 'file1'),
file2: path.resolve('baz', 'file2')
file3: path.resolve('baz', 'file3')
file4: path.resolve('baz', 'file4')
file5: path.resolve('baz', 'file5')
};
var destinations = {
file1: path.resolve(self.base_dir, self.file1),
file2: path.resolve(self.base_dir, self.file2),
file3: path.resolve(self.base_dir, self.file3),
file4: path.resolve(self.base_dir, self.file4),
file5: path.resolve(self.base_dir, self.file5)
};
var filters = {
qux: /^qux/,
bru: /^bru/,
blerg: /blerg$/
};
function copyFile1() {
console.log('hello world');
return fs.copyAsync(sources.file2, destinations.file1, { filter: filters.qux });
};
function copyFile2() {
return fs.copyAsync(sources.file2, destinations.file2);
};
function copyFile3() {
return fs.copyAsync(sources.file3, destinations.file3, { filter: filters.bru });
};
function copyFile4() {
return fs.copyAsync(sources.file4, destinations.file4, { filter: filters.blerg });
};
return Promise.all([
copyFile1(), // execute functions
copyFile2(), // idem
copyFile3(), // idem
copyFile4() // idem
]);
// async.parallel([
// copyFile1(next),
// copyFile2(next),
// copyFile3(next),
// copyFile4(next)
// ], function(err) {
// if (err) return done(err);
// done(null);
// })
};
Archive.prototype._writeArchive = function() {
var self = this;
var archive_dir_path = path.resolve(self.base_dir, '..');
var tarPromise = function() {
return new Promise(function(resolve, reject) {
tar.pack(self.files_path)
.pipe(fs.createWriteStream(archive_dir_path + '.tar'))
.on('error', reject)
.on('finish', resolve)
});
};
return fs.ensureDirAsync(archive_dir_path)
.then(tarPromise); // CHANGED,
};

Related

How to call another prototype async function from another in the same class

I have one async prototype function and that is called from another async prototype function. But, I got the error below. How to resolve?
I use node.js v14.1.0.
UnhandledPromiseRejectionWarning: TypeError: this.saySomething2 is not a function
const Person = function() {
console.log("CALLED PERSON");
};
Person.prototype.saySomething = async function() {
this.saySomething2();
};
Person.prototype.saySomething2 = async function() {
//do something
console.log("hello");
};
(async function() {
var ape = new Person();
await ape.saySomething();
}());
update: adding actual code.
// constructor function for the KingInfo class
function KingInfo(
date,
employeeKey,
workDayTypeName) {
this._date = date;
this._employeeKey = employeeKey;
this._workDayTypeName = workDayTypeName;
}
KingInfo.prototype.sync = async function(){
var isSucceed = false;
let syncType = await this.getSyncType();
if(syncType === "post"){
}
else if(syncType === "patch"){
}
else{
}
return isSucceed;
}
//#return: "post", "patch", "none"
KingInfo.prototype.getSyncType = async function(){
let syncType = "none";
let sql = [];
sql.push("SELECT count(*) as count");
...
let records = await db_module.query(sql.join(""));
let count = records[0].count;
if(count <= 0){
syncType = "post";
}
else{
let isSame = await this.compareToDataInDB();
if(!isSame){
syncType = "patch";
}
}
return syncType;
}
KingInfo.prototype.compareToDataInDB = async function(){
let date = this._date;
let empKey = this._employeeKey;
let sql = [];
sql.push("SELECT workDayTypeName");
...
let records = await db_module.query(sql.join(""));
let record = records[0];
let res = false;
if(workDayTypeName === record.workDayTypeName){
res = true;
}
return res;
}
(async function(){
let date = "2020-08-01";
let employeeKey = "";
let workDayTypeName = "aaaaa";
let kingInfo = new KingInfo(
date,
employeeKey,
workDayTypeName);
kingInfo.sync();
}());
module.exports = {
KingInfo: KingInfo
}
I got this error:
UnhandledPromiseRejectionWarning: TypeError: Cannot read property '_date' of undefined
UnhandledPromiseRejectionWarning: TypeError: this.compareToDataInDB is not a function
update:
adding code for db module.
this module might effect badlly?
db.js
"use strict";
const mysql = require('mysql');
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'something'
});
function query(sql){
return new Promise(function(resolve, reject) {
connection.connect((err) => {
if (err) throw err;
connection.query(sql, function (err, result, fields) {
if (err) throw err;
resolve(result);
});
});
});
}
module.exports = {
query: query
}

Node.js : Call function using value from callback or async

I have written below .js file to call below defined function.
objectRepositoryLoader.readObjectRepository() returns me a hashmap from where i have to use values in enterUserName(), enterPassword(), clickLoginButton() functions.
var path = require('path');
var elementRepoMap = {}
var LandingPage = function(){
var fileName = path.basename(module.filename, path.extname(module.filename))
objectRepositoryLoader.readObjectRepository(fileName+'.xml' , function(elementRepo){
console.log(elementRepo) //values are being printed here
this.elementRepoMap = elementRepo
});
this.enterUserName = function(value){
console.log(elementRepoMap) //values are not being printed here
//Some Code
};
this.enterPassword = function(value){
//Some Code
};
this.clickLoginButton = function(){
//Some Code
};
};
module.exports = new LandingPage();
The objectRepositoryLoader.readObjectRepository() function defined in another file is as below:
var ObjectRepositoryLoader = function() {
this.readObjectRepository = function(fileName, callback) {
var filePath = './elementRepository/'+fileName;
this.loadedMap = this.objectRepoLoader(filePath, function(loadedMap){
return callback(loadedMap);
});
}
this.objectRepoLoader = function(filePath, callback){
if (filePath.includes(".xml")) {
this.xmlObjectRepositoryLoader(filePath, function(loadedMap){
return callback(loadedMap);
});
}
this.xmlObjectRepositoryLoader = function (xmlPath, callback){
var innerMap = {};
var elementName;
fs.readFile(xmlPath, "utf-8",function(err, data) {
if(err){
console.log('File not found!!')
}
else{
var doc = domparser.parseFromString(data,"text/xml");
var elements = doc.getElementsByTagName("A1");
for(var i =0 ; i< elements.length;i++){
var elm = elements[i];
elementName = elm.getAttribute("name");
var params = elm.getElementsByTagName("AS");
innerMap = {};
for(var j =0 ; j< params.length;j++){
var param = params[j];
var locatorType = param.getAttribute("type");
var locatorValue = param.getAttribute("value");
innerMap[locatorType] = locatorValue;
}
loadedMap[elementName] = innerMap;
innerMap={};
};
}
return callback(loadedMap);
});
};
How can I call enterUserName(), enterPassword(), clickLoginButton() function from spec.js file and is there any way I can avoid using callback and use async.js and call enterUserName(), enterPassword(), clickLoginButton() from spec.js file ?
EDIT
I have modified my file like below:
this.xmlObjectRepositoryLoader = function (xmlPath){
var innerMap = {};
var elementName;
var filePath = xmlPath+'.xml'
var self = this
return new Promise(
function(resolve, reject){
console.log("In xmlObjectRepositoryLoader : "+filePath)
self.readFilePromisified(filePath)
.then(text => {
var doc = domparser.parseFromString(text,"text/xml");
var elements = doc.getElementsByTagName("Element");
for(var i =0 ; i< elements.length;i++){
var elm = elements[i];
elementName = elm.getAttribute("name");
var params = elm.getElementsByTagName("param");
innerMap = {};
for(var j =0 ; j< params.length;j++){
var param = params[j];
var locatorType = param.getAttribute("type");
var locatorValue = param.getAttribute("value");
innerMap[locatorType] = locatorValue;
}
map[elementName] = innerMap;
innerMap={};
}
console.log(map) // prints the map
resolve(text)
})
.catch(error => {
reject(error)
});
});
}
this.readFilePromisified = function(filename) {
console.log("In readFilePromisified : "+filename)
return new Promise(
function (resolve, reject) {
fs.readFile(filename, { encoding: 'utf8' },
(error, data) => {
if (error) {
reject(error);
} else {
resolve(data);
}
})
})
}
I am calling above function from another file as below:
objectRepositoryLoader.readObjectRepository(fileName)
.then(text => {
console.log(text);
})
.catch(error => {
console.log(error);
});
But it gives me error as
.then(text => { ^
TypeError: Cannot read property 'then' of undefined
In this case how can I use promise to call another promise function and then use the returned value in one more promise function and return calculated value to calling function where I can use the value in other functions. I sound a bit confused. Please help
You can use async.waterfall and async.parallel to perform this task
see the reference
I just tried your code to make it working, I explained the way of implementation in comment.
async.waterfall([
function(next){
objectRepositoryLoader.readObjectRepository(fileName+'.xml' ,next)//pass this next as parameter in this function defination and after manipulation return result with callback like this(null,result)
}
],function(err,result){
if(!err){
//Do wahtever you want with result
async.parallel([
function(callback){
this.enterUserName = function(value){
console.log(elementRepoMap)
//Some Code
};
},
function(callback){
this.enterPassword = function(value){
//Some Code
};
},
function(callback){
this.clickLoginButton = function(){
//Some Code
};
}
], function(err, results) {
// optional callback
};
}
})

Why this promise.all does not work?

var Promise = require("bluebird");
var MongoDB = Promise.promisifyAll(require('mongodb'));
var MongoClient = MongoDB.MongoClient;
var database = "mongodb://localhost/test";
MongoClient.connect(database)
.then(function(db) {
var c1 = db.collection('c1');
var c2 = db.collection('c2');
return Promise.all([
c1.count().then(function(count) {
if(count==0) {
return c1.insertMany([{a:1},{a:2}]);
}
else { // what should I write here?
} //
}),
c2.count().then(function(count) {
if(count==0) {
return c2.insertMany([{a:1},{a:2}]);
}
})
]);
})
.catch(function(err) {
console.log(err)
});
It just hangs there.
And what should I write in else part?
if(count==0) {
return c1.insertMany([{a:1},{a:2}]);
}
else { // what should I write here?
} //
I guess db.collection() returns a promise as well so you need to write something like this
var Promise = require("bluebird");
var MongoDB = Promise.promisifyAll(require('mongodb'));
var MongoClient = MongoDB.MongoClient;
var database = "mongodb://localhost/test";
var insertIfEmpty = function(collection) {
collection.count().then(function(count) {
if(count==0) {
return collection.insertMany([{a:1},{a:2}]);
}
else {
return Promise.resolve()
});
}
MongoClient.connect(database)
.then(function(db) {
var promisedCollections = [db.collection('c1'), db.collection('c2')]
return Promise.all(promisedCollections).map(insertIfEmpty);
})
.catch(function(err) {
console.log(err)
});
If you need to populate the collections one at a time you can use .each in place of .map

convert tree-manager npm module (filetree) for koajs / co compatibility

How do I get a list of files compatible with co/yield (for koajs) ?
I am trying to convert this module for koa/co: https://www.npmjs.com/package/tree-manager
The original function is
fileModule.prototype.walkDir = function(dir, done) {
var self = this;
var results = [];
fs.readdir(dir, function(err, list) {
if (err) {
return done(err);
}
console.log(pending)
var pending = list.length;
if(!pending) {
return done(null, results);
}
list.forEach(function(file) {
var dfile = path.join(dir, file);
var el = {};
var fid = path.join(dir.replace(self.root, ''), file);
el.text = file;
el.id = fid;
fs.stat(dfile, function(err, stat) {
if(err) {
throw err;
}
if(stat.isDirectory()) {
return self.walkDir(dfile, function(err, res) {
el.children = res;
results.push(el);
!--pending && done(null, results);
});
}
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid};
results.push(el);
!--pending && done(null, results);
});
});
});
}
I can replace require('fs') with co-fs-plus (or extra)
so i can remove all fs callbacks with simple yield fs.xxx
but i don t understand the foreach loop :/
UPDATE (solution):
so ...
add wrap to co and adding dfile to the isDirectory function seems working
fileModule.prototype.walkDir = co.wrap(function*(dir) {
var self = this;
var list = yield fs.readdirAsync(dir);
// yield a list of promises
// created by mapping with an asynchronous function
var results = yield list.map(co.wrap(function*(file) {
var dfile = path.join(dir, file);
var fid = path.join(dir.replace(self.root, ''), file);
var el = {
text: file,
id: fid
};
try {
if (yield fs.isDirectoryAsync(dfile)) {
el.children = yield self.walkDir(dfile);
} else {
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid};
}
return el;
} catch(err) {
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid+' !! FILE UNREADABLE !!'};
return el;
}
}));
return results;
})
Thanks !
That loops runs all the actions in parallel, that's why it's so complicated (with the pending thingy and all). However, doing things in parallel is not really a strength of co, you should look into promises for that.
import fs from 'fs-extra-promise';
fileModule.prototype.walkDir = co(function*(dir) {
var self = this;
var list = yield fs.readdirAsync(dir);
// yield a list of promises
// created by mapping with an asynchronous function
var results = yield list.map(co(function*(file) {
var dfile = path.join(dir, file);
var fid = path.join(dir.replace(self.root, ''), file);
var el = {
text: file,
id: fid
};
if (yield fs.isDirectoryAsync()) {
el.children = yield self.walkDir(dfile);
} else {
el.icon = 'file'; // #TODO - to settings
el.a_attr = {id: fid};
}
return el;
}));
return results;
});

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

Resources