Bluebird Promises with Event Emitter - node.js

I am fairly new with using Bluebird promises. I was trying to use them over an emitter. However, I am stuck on how to handle errors.
I have a stream object which is the emitter. Code is as below -
return new Promise((resolve, reject) => {
var onDocFunc = doc => {
//JSON.parse('*');
// some logic goes in here to construct setStmt
bulk.find(query).upsert().update({$set: setStmt});
count++;
if (count % bulkSize == 0) {
stream.pause();
var execute = Promise.promisify(bulk.execute);
execute().catch(() => {}).then(() => {
stream.resume();
});
}
};
stream.on('data', onDocFunc);
stream.on('end', () => {
JSON.parse('*'); // how to catch errors that happen here??
var boundResolve = resolve.bind(this, {count: count});
if (count % bulkSize != 0) {
Promise.promisify(bulk.execute)().then(boundResolve).catch(boundResolve);
}
else {
boundResolve();
}
});
stream.on('error', err => {
reject(err);
});
})
I want to know what is the recommended way to catch an error which occurs inside the callback of the end event handler? Right now if any error occurs, the NodeJS application crashes with uncaughtException: Unexpected token *

Don't mix application logic into the promisification of the event emitter. Such code (that can throw etc) should always go in then callbacks. In your case:
var execute = Promise.promisify(bulk.execute);
return new Promise((resolve, reject) => {
stream.on('data', onDocFunc); // not sure what this does
stream.on('end', resolve);
stream.on('error', reject);
}).then(() => {
JSON.parse('*'); // exceptions that happen here are caught implicitly!
var result = {count: count};
if (count % bulkSize != 0) {
return execute().catch(()=>{}).return(result);
} else {
return result;
}
});
Regarding your real code, I'd probably try to factor out the batching into a helper function:
function asyncBatch(stream, size, callback) {
var batch = [], count = 0;
stream.on('data', data => {
batch.push(data);
count++;
if (batch.length == size) {
stream.pause();
Promise.resolve(batch).then(callback).then(() => {
batch = [];
stream.resume();
}, e => {
stream.emit('error', e);
});
}
});
return new Promise((resolve, reject) => {
stream.on('end', resolve);
stream.on('error', reject);
}).then(() => batch.length ? callback(batch) : null).then(() => count);
}
Promise.promisifyAll(Bulk);
return asyncBatch(stream, bulkSize, docs => {
const bulk = new Bulk()
for (const doc of docs) {
// JSON.parse('*');
// some logic goes in here to construct setStmt
bulk.find(query).upsert().update({$set: setStmt});
}
return bulk.executeAsync().catch(err => {/* ignore */});
})

You'll have to use a try/catch block:
stream.on('end', () => {
try {
JSON.parse('*')
// ...the rest of your code
} catch (e) {
reject(e)
}
})

Related

Completely lost with promise async/await

I'm trying to develop a relative simple test on NodeJS 11.6.x. I'm not really a developer, but sometimes try to do some coding.
My objective is to create a SQLite database and repeat some steps every time a run the code:
1. Drop a table if it exists
2. Create a table
3. Insert N lines
4. Count how many lines is in the database
5. Close the database
I tried first with a basic approach using callback, but couldn't figure out a way to make the step 3 (insert N lines) and looking for a solution, the promise async/await 'pattern' sounded the way to accomplish everything.
But, after refactoring the code, the step 1 (drop table) isn't running and I still not being able to execute step 3 (insert N lines) and have no idea what is happening. I also tried to use a promise package with no luck.
Could someone please have a look and help on this and if possible, explain and or give some advice?
Thanks in advance
Edited: Well, I'not used to post here at SO and don't know the 'right' way to update things here. I beleave I should had left the first code as reference, buy I don't have anymore.
Now I think I'm almoust there. All steps are executing in order. Just the step 3 (insert N lines) that I'm not able to make it work. Or it inserts and stops not going to the next '.then' or it just insert 1 line and I'm cannot visualize what is happening.
In the code, I commented in two lines with 'BUG 1:' and 'BUG 2:'.
If I both are commented, I get what is happening, it inserts only 1 line and don't continue the promise chain
If I comment BUG 1 and let BUG 2 active, it inserts just one line and continues. I think I understand why
If I comment BUG 2 and let BUG 1 active, it inserts all lines but don't continue and again, I think I understand why
If I uncomment both (the way I think should work. Don't work, and return an aditional error "Segmentation fault"
Bellow the code:
const sqlite3 = require('sqlite3')
let db = new sqlite3.Database('./test.db');
waitTime = 1
process.stdout.write('Starting...\n')
var test = new Promise((resolve, reject) => {
process.stdout.write('Drop Table... ');
db.run(`DROP TABLE IF EXISTS test`, (err) => {
if (err) {
process.stdout.write(`Dropping Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(`Dropped!\n`)
resolve()
}, waitTime)
}
})
})
test.then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Create Table... ')
db.run(`CREATE TABLE IF NOT EXISTS test (data TEXT)`, (err) => {
if (err) {
process.stdout.write(`Creating Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(`Created!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Insert Line... ')
lines = 10
let loop = (async () => {
for (let i = 0; i < lines; i++) {
await new Promise(resolve =>
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
throw (err)
} else {
setTimeout(() => {
// process.stdout.write(`Line ${i} Inserted!\n`)
process.stdout.write(`, ${i+1}`)
resolve() // BUG 1: if this line is commented, comment it, it will insert only 1 line
}, waitTime)
}
})
)
}
})()
process.stdout.write(`, IDone\n`)
resolve() // BUG 2: If this line is commented, the promise chain stops here
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Count Line(s)... ')
db.all(`SELECT COUNT(*) AS totalLines FROM test`, [], (err, rows) => {
if (err) {
process.stdout.write(`Count Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(` ${rows[0].totalLines} Count!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Select Line(s)... ')
db.all('SELECT data FROM test', [], (err, rows) => {
if (err) {
process.stdout.write(`Select Error ${err.message}\n`)
reject()
} else {
rows.forEach((row) => {
console.log(row.data);
})
setTimeout(() => {
process.stdout.write(`${rows[0].totalLines} Select!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
return new Promise((resolve, reject) => {
process.stdout.write('Close DB... ')
db.close((err) => {
if (err) {
process.stdout.write(`Closing Error ${err.message}\n`)
reject()
} else {
setTimeout(() => {
process.stdout.write(`Closed!\n`)
resolve()
}, waitTime)
}
})
})
}).then(() => {
console.log('Finished')
})
After the great explanation from #CertainPerformance (Thanks a lot), I was able to get it running. I believe it is now "the right" way to do it. May be there are some better ways, but for now, it is ok for me, bellow the final code:
const sqlite3 = require('sqlite3')
let db = new sqlite3.Database('./test.db');
lines = 10
process.stdout.write('Starting... ')
var test = new Promise((resolve, reject) => { process.stdout.write(`Promise Created...!\n`)
resolve()
})
test.then(() => { process.stdout.write('Drop Table... ')
return new Promise((resolve, reject) => {
db.run(`DROP TABLE IF EXISTS test`, (err) => {
if (err) {
reject(err)
} else { process.stdout.write(`Dropped!\n`)
resolve() }
})
})
}).then(() => { process.stdout.write('Create Table... ')
return new Promise((resolve, reject) => {
db.run(`CREATE TABLE IF NOT EXISTS test (data TEXT)`, (err) => {
if (err) {
reject(err)
} else {
process.stdout.write(`Created!\n`)
resolve() }
})
})
}).then(() => { process.stdout.write('Insert Line... ')
let insertLoop = (async () => {
for (let i = 0; i < lines; i++) {
await new Promise(resolve =>
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
reject(err)
} else { ( i == 0 ) ? process.stdout.write(`${i + 1}`) : process.stdout.write(`, ${i + 1}`)
resolve() }
})
)
}
process.stdout.write(`, Inserted!\n`)
})()
return insertLoop
}).then(() => { process.stdout.write('Count Line(s)... ')
return new Promise((resolve, reject) => {
db.all(`SELECT COUNT(*) AS totalLines FROM test`, [], (err, rows) => {
if (err) {
reject(err)
} else { process.stdout.write(` ${rows[0].totalLines} Counted!\n`)
resolve()
}
})
})
}).then(() => { process.stdout.write('Close DB... ')
return new Promise((resolve, reject) => {
db.close((err) => {
if (err) {
reject(err)
} else { process.stdout.write(`Closed!\n`)
resolve()
}
})
})
}).then(() => {
console.log('Finished')
}).catch((err) => {
process.stdout.write(`The process did not finish successfully: ${err}`)
})
There are two main issues. First, in the second .then, you declare loop as an async function that is immediately invoked: this means that loop will resolve to a Promise. The trimmed code looks like:
}).then(() => {
return new Promise((resolve, reject) => {
let loop = (async () => {
// do some asynchronus stuff
})()
resolve() // BUG 2
})
}).then(() => {
Declaring a Promise alone will not cause the current thread to wait for it. The above code doesn't work as expected for the same reason that this code prints after immediately:
console.log('start');
const prom = new Promise((resolve) => {
setTimeout(resolve, 500);
});
console.log('after');
You must call .then on a Promise (or await the Promise) in order to schedule additional operations after the Promise completes. Or, if you're currently inside a .then, you can return the Promise, which will mean that the next .then will run as soon as the returned Promise resolves:
}).then(() => {
let loop = (async () => {
// do some asynchronus stuff
})();
return loop;
}).then(() => {
// this block will run once `loop` resolves
Note the lack of a new Promise((resolve... constructor above - inside a .then, just returning the next Promise is often the preferred way to go, since it means a lot less code and avoids an antipattern.
The other issue with the current code is that errors will not be caught. For example, if your
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
throw (err)
// else call resolve()
throws an error, the Promise that's currently being awaited at that point will never resolve, nor will it reject - it will remain pending and unfulfilled forever. You should pass reject as the second argument to the Promise constructor, and call it when there's an error (instead of throw), for example:
await new Promise((resolve, reject) => {
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
reject(err)
} else {
// ...
This way, the awaited Promise will get rejected, which means that the whole loop will reject, and if the loop is returned, it'll allow a .catch to catch the error, for example:
var test = new Promise((resolve, reject) => {
// ...
});
test.then(() => {
return new Promise(...
// ...
})
.then(() => {
return new Promise(...
// ..
})
.then(() => {
return new Promise(...
// ..
})
.catch((err) => {
process.stdout.write(`The process did not finish successfully:`, err)
// handle errors
});
Note that, unless each db. function call needs to execute serially, it would be better to make all the requests at once, and resolve once each request has finished - this can significantly reduce the time required for the script to run. Create an array of Promises for each asynchronous call, then call Promise.all on that array to get a Promise that resolves when all of those Promises are done (or, rejects as soon as one of those Promises rejects). For example, for the second .then:
}).then(() => {
process.stdout.write('Insert Line... ')
const proms = Array.from(
{ length: lines },
(_, i) => new Promise((resolve, reject) => {
db.run(`INSERT INTO test (data) VALUES ('a')`, (err) => {
if (err) {
process.stdout.write(`Inserting Error ${err.message}\n`)
reject(err)
} else {
setTimeout(() => {
// process.stdout.write(`Line ${i} Inserted!\n`)
process.stdout.write(`, ${i+1}`)
resolve()
}, waitTime);
}
});
})
);
return Promise.all(proms);
}).then(() => {
Nothing else in your code looks to deal with asynchronous loops, luckily.
You may also consider a utility function like Promisify which will turn callback-based functions to Promises without all the extra new Promise(... boilerplate every time there's an asynchronous call.
Further improvements can be made by promisifying db into reusable functions, and leveraging the full power of async/await instead of mixing it with then:
const sqlite3 = require('sqlite3')
let db = new sqlite3.Database('./test.db');
function runDbAsync(sql) {
return new Promise((resolve, reject) => {
db.run(sql, (err) => {
if (err) reject(err);
else resolve();
});
});
}
function getDbAsync(sql, val) {
return new Promise((resolve, reject) => {
db.all(`SELECT COUNT(*) AS totalLines FROM test`, [], (err, rows) => {
if (err) reject(err);
else resolve(rows);
});
});
}
function closeDbAsync() {
return new Promise((resolve, reject) => {
db.close((err) => {
if (err) reject(err);
else resolve();
});
});
}
function write(text) {
process.stdout.write(text);
}
function writeLn(text) {
write(text + "\n");
}
async function main() {
const lines = 10
writeLn('Starting... ')
write('Drop Table... ');
await runDbAsync(`DROP TABLE IF EXISTS test`);
writeLn(`Dropped!`);
write('Create Table... ');
await runDbAsync(`CREATE TABLE IF NOT EXISTS test (data TEXT)`);
writeLn(`Created!`);
write('Insert Line... ');
for (let i = 0; i < lines; i++) {
await runDbAsync(`INSERT INTO test (data) VALUES ('a')`);
write( i == 0 `${i + 1}` : `, ${i + 1}`);
}
writeLn(`, Inserted!`);
write('Count Line(s)... ')
const rows = getDbAsync(`SELECT COUNT(*) AS totalLines FROM test`, []);
writeLn(` ${rows[0].totalLines} Counted!`)
write('Close DB... ');
await closeDbAsync();
writeLn(`Closed!`);
}
main().then(() => {
console.log('Finished')
}, err => {
writeLn(`The process did not finish successfully: ${err}`)
});

Stream Promise results to client as they resolve in Promise.all

In Node, I am using bluebird's Promise.all to execute Promises asynchronously. I don't want to wait for all Promises to resolve before I send results back to the client; rather, I'd like to stream the result of each Promise as soon as it resolves. Is this possible? My research indicates that it is not, but I thought it worth reaching out to the community.
Here is the code (with no attempt to implement streaming):
async bulkExecution(req, res) {
try {
const { assets } = req.body;
let thePromises = _.map(assets, (asset) => {
return onePromise(asset);
});
// I want to stream each Promise result to client as it resolves
let results = await Promise.all(thePromises);
return res.status(200).send(results);
} catch (err) {
return res.status(500).send(err);
}
}
I think you'd have to iterate over all the promises and do a .then() on each promise and write a partial response then close the response after all the promises complete. Something like this should work. The order of the responses may be different than the order of the promise array. For instance, my example below will return the second promise first.
let res = {
write: (data) => {console.log(data)},
end: () => {}
};
let promise1 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve('foo');
}, 500);
});
let promise2 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve('foo');
}, 300);
});
let promises = [promise1, promise2];
let count = 0;
promises.forEach((promise) => {
promise.then((data) => {
res.write(data);
count++;
if (count >= promises.length){
res.end();
}
})
})
Modified to write in JSON format.
let res = {
write: (data) => {
console.log(data)
},
end: () => {}
};
let promise1 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve({foo:"bar"});
}, 500);
});
let promise2 = new Promise(function(resolve, reject) {
setTimeout(function() {
resolve({bar:"foo"});
}, 300);
});
let promises = [promise1, promise2];
let count = 0;
res.write("[");
promises.forEach((promise) => {
promise.then((data) => {
res.write(JSON.stringify(data));
count++;
if (count >= promises.length) {
res.write("]");
res.end();
} else {
res.write(",");
}
})
})
Streaming out JSON like this isn't common, but could be used to reduce the amount of memory the server uses. That's because you don't need to create the full response in memory before sending the response. The client will still need enough memory to read the entire object into memory.
Try doing .then( data => data.json) on the promise like so
let thePromises = _.map(assets, (asset) => {
return onePromise(asset).then(data => data.json);
});

Node js stop long process task when Promise reject it

I create a promise function to processing a long-time query task. Some time the task will block for hours. I want set a time out to stop the task. Below is the code.
It can return error message correctly, but it still running connection.execute() for long time before stop. So how can stop it immediately when it return reject message?
Thanks!
function executeQuery(connection, query) {
return new Promise((resolve, reject) => {
"use strict";
//long time query
connection.execute(query, function (err, results) {
if (err) reject('Error when fetch data');
else resolve(results);
clearTimeout(t);
});
let t = setTimeout(function () {
reject('Time Out');
}, 10);
})
(async () => {
"use strict";
oracle.outFormat = oracle.OBJECT;
try {
let query = fs.readFileSync("query.sql").toString();
let results = await executeQuery(connection, query);
console.log(results.rows);
} catch (e) {
console.log(`error:${e}`);
}
So how can stop it immediately when it return reject message?
According to the docs, you can use connection.break:
return new Promise((resolve, reject) => {
connection.execute(query, (err, results) => {
if (err) reject(err);
else resolve(results);
clearTimeout(t);
});
const t = setTimeout(() => {
connection.break(reject); // is supposed to call the execute callback with an error
}, 10);
})
Make sure to also release the connection in a finally block.
Try this (using bluebird promises):
var execute = Promise.promisify(connection.execute);
function executeQuery(connection, query) {
return execute.call(connection, query)
.timeout(10000)
.then(function (results) {
// handle results here
})
.catch(Promise.TimeoutError, function (err) {
// handle timeout error here
});
.catch(function (err) {
// handle other errors here
});
};
If this still blocks, there's a possibility that the database driver you are using is actually synchronous rather than asynchronous. In that case, that driver would be incompatible with the node event loop and you may want to look into another one.
As Bergi mentioned, you'll need to use the connection.break method.
Given the following function:
create or replace function wait_for_seconds(
p_seconds in number
)
return number
is
begin
dbms_lock.sleep(p_seconds);
return 1;
end;
Here's an example of its use:
const oracledb = require('oracledb');
const config = require('./dbConfig.js');
let conn;
let err;
let timeout;
oracledb.getConnection(config)
.then((c) => {
conn = c;
timeout = setTimeout(() => {
console.log('Timeout expired, invoking break');
conn.break((err) => {
console.log('Break finished', err);
});
}, 5000);
return conn.execute(
`select wait_for_seconds(10)
from dual`,
[],
{
outFormat: oracledb.OBJECT
}
);
})
.then(result => {
console.log(result.rows);
clearTimeout(timeout);
})
.catch(err => {
console.log('Error in processing', err);
if (/^Error: ORA-01013/.test(err)) {
console.log('The error was related to the timeout');
}
})
.then(() => {
if (conn) { // conn assignment worked, need to close
return conn.close();
}
})
.catch(err => {
console.log('Error during close', err)
});
Keep in mind that the setTimeout call is just before the execute (because of the return statement). That timeout will start counting down immediately. However, the execute call isn't guaranteed to start immediately as it uses a thread from the thread pool and it may have to wait till one is available. Just something to keep in mind...

Data aggregation in NodeJS with promisified functions

I'd like to aggregate data from MongoDB in NodeJS with promisified functions.
My script with dump are here https://github.com/network-spy/lego
Little description: there are 2 collections in database: "zip" and "restaurants". "zip" contains zip codes of locations and "restaurants" contains information about restaurants with zip codes. So script should create new collection "stat" and fill it with documents like:
{"zip_code" : "01002", "restaurants" : [ list of restaurants ] }
The problem is that in "zip" collection there are 29353 documents, but after script processing I get "stat" collection with 29026 documents(sometimes count of documents can change).
I guess it's because of broken synchronization somewhere in my JS code. Could you look please at my code and advice anything how to repair it?
const MongoClient = require('mongodb').MongoClient;
const mongoDbUrl = 'mongodb://127.0.0.1:27017/world';
MongoClient.connect(mongoDbUrl, function(err, db) {
if (err) {
console.log(err);
return;
}
console.log("Connected to server.");
clearStat(db).then(
result => {
console.log(result);
processZips(db).then(
result => {
console.log(result);
closeMongoDBConnection(db);
},
error => {
console.log(error);
closeMongoDBConnection(db);
}
);
},
error => {
console.log(error);
closeMongoDBConnection(db);
}
);
});
let closeMongoDBConnection = (db) => {
db.close();
console.log("Disconnected from server.");
};
let clearStat = (db) => {
return new Promise((resolve, reject) => {
db.collection('stat').deleteMany({}, function(err, results) {
if (err) {
reject(err);
}
resolve('Stat data cleared');
});
});
};
let processZips = (db) => {
return new Promise((resolve, reject) => {
db.collection('zip').find({}, {"_id":1}).each((err, zipCode) => {
if (zipCode == null) {
resolve('Zips precessed');
} else if (err) {
reject(err);
} else {
findRestaurantsByZip(db, zipCode._id).then(
result => {
insertToStat(db, zipCode._id, result).then(
result => {
console.log('Inserted: ');
console.dir(result);
},
error => {
reject(error);
}
);
},
error => {
reject(error);
}
);
}
});
});
};
let findRestaurantsByZip = (db, zipCode) => {
return new Promise((resolve, reject) => {
db.collection('restaurant').find({"address.zipcode": zipCode}).toArray((err, restaurants) => {
if (err) {
reject(err);
}
resolve(restaurants);
});
});
};
let insertToStat = (db, zip, restaurants) => {
return new Promise((resolve, reject) => {
let statDocument = {};
statDocument.zip_code = zip;
statDocument.restaurants = restaurants;
db.collection('stat').insertOne(statDocument).then(
result => {
resolve(statDocument);
},
error => {
reject(error);
}
);
});
};
Firstly, a simplification of your processZips function. This is functionally identical to your code but uses Promise chaining rather than nested Promises
let processZips = (db) => new Promise((resolve, reject) =>
db.collection('zip').find({}, {"_id":1}).each((err, zipCode) => {
if (zipCode == null) {
resolve('Zips precessed');
} else if (err) {
reject(err);
} else {
findRestaurantsByZip(db, zipCode._id)
.then(result => insertToStat(db, zipCode._id, result))
.then(result => console.log('Inserted: ', result))
.catch(error => reject(error));
}
})
);
The problem may be (I can't test anything) that you resolve the processZips promise at the end of the .each processing. This "triggers" the .then that closes the database. However, due to the asynchronous find/insert code it may well be that some of that is "in progress" at the time. I don't profess to know mongodb well, so I don't know what closing the db while processing is still active would do - seems likely that's the reason why you're output data is "short"
So, there's two ways to approach this
1 - process each zipCode in series, i.e. each find/insert waits for the previous to complete, and then resolve when last zipCode is done
let processZips = (db) => {
// set p to a resolved Promise so the first find/insert will kick off
let p = Promise.resolve();
return new Promise((resolve, reject) =>
db.collection('zip').find({}, {"_id":1}).each((err, zipCode) => {
if (zipCode == null) {
// wait for last insert to complete before resolving the Promise
resolve(p.then(() => resolve('Zips precessed'))); // see note 1, 2
} else if (err) {
reject(err);
} else {
// wait for previous insert to complete before starting new find/insert
p = p
.then(() => findRestaurantsByZip(db, zipCode._id))
.then(result => insertToStat(db, zipCode._id, result))
.then(result => console.log('Inserted: ', result)); // see note 1
}
})
);
};
With this code, as soon as a find/insert rejects, no more find/insert will actually be performed
2 - process each code in "parallel", i.e. kick off all the find/insert and then resolve when all zipCode are done
let processZips = (db) => {
// create an array for all the find/insert Promises
let p = [];
return new Promise((resolve, reject) =>
db.collection('zip').find({}, {"_id":1}).each((err, zipCode) => {
if (zipCode == null) {
// wait for all find/insert to complete before resolving this Promise
resolve(Promise.all(p).then(() => 'Zips precessed')); // see note 1, 2
} else if (err) {
reject(err);
} else {
p.push(findRestaurantsByZip(db, zipCode._id)
.then(result => insertToStat(db, zipCode._id, result))
.then(result => console.log('Inserted: ', result))
); // see note 1
}
})
);
};
The one caveat with the second method is, like in your original code, if one of the find/insert fails that wont stop subsequent find/insert from processing.
You'll notice that there seems to be a lack of error handling compared to your original code. This code uses the 2 "features" of promises.
rejections will "flow through" the promise chain,
if you resolve a promise with a rejected promise, it is identical to rejecting the promise.

Deferred.notify() inside async.whilst() doesn't trigger progress handler until callback

I have a function that returns a promise (using Q), and the notifications don't seem to happen at the right time. The onFulfilled and onRejected callback work as intended, but the progress callback doesn't fire until after async.whilst() finishes running, and fires everything at once.
This is the function
function generateSentences(data, num, options) {
var deferred = Q.defer();
const markov = new Markov(data, options);
markov.buildCorpus()
.then(() => {
var count = 0;
async.whilst(
function () { return count < num; },
function (callback) {
markov.generateSentence()
.then(result => {
console.log("Count: " + count);
deferred.notify(count / num); //update progress
count++;
callback(null);
}, (err) => {
deferred.reject(err.toString());
count++;
});
},
function (err, n) {
//PROGRESS EVENTS DON'T HAPPEN UNTIL HERE
deferred.resolve(generatedSentences); //finish
}
);
}, (err) => console.log(err));
return deferred.promise;
}
and this is using the promise
function generateScript() {
fs.readdir(parser.videoBasePath, function (err, files) {
parseFiles(files, parser.parse).then((a) => {
console.log("Total Lines: " + fullScript.length + "\n");
fullScript = _.shuffle(fullScript);
markov.generateSentences(fullScript, 20).then((data) => {
console.log(data);
}, (err) => {
console.log(err);
}, (progress) => {
console.log(progress);
});
});
});
}
I've read some threads like this saying I need to wrap a setTimeout around the notify(), but it doesn't seem to affect anything.
I've read that Promises + async.js don't mix (but can't find anything to say as much!!), and I can't really see why that should be an issue in this case to be honest
Having said that, the code you've shown seems to be possible without async, so try this to see if the progress works any better
function generateSentences(data, num, options) {
var deferred = Q.defer();
const markov = new Markov(data, options);
const genSentence = count => markov.generateSentence()
.then(result => {
console.log("Count: " + count);
deferred.notify(count / num); //update progress
if (count < num) {
return genSentence(count + 1);
}
});
markov.buildCorpus()
.then(() => genSentence(0))
.then(() => deferred.resolve(generatedSentences)) //finish
.catch(err => deferred.reject(err.toString()));
return deferred.promise;
}
At the very least, the code is (in my opinion) a little cleaner anyway

Resources