Wait for nodeJs async/promise process to end before sending response to client - node.js

I have the following code that I cannot figure out what I need to do to make this entire process finish before sending a response back to the client. Basically I am looking for the client to receive a response once everything in the method completes or errors out, not before.
I have tried several things, moving things around and adding some .then() and .catch() blocks that may not even be needed. I'm still fairly new to NodeJs so while I understand how things kind of work, I still have not got used to asynchronous coding.
let markStepsComplete = async function() {
let stepsToProcess = new Multimap();//multimap JS npm
let results = await dbUtils.getCompletedSteps();
results.forEach(result => {
stepsToProcess.set(result.ticket_id, result.step_id);
});
return new Promise((resolve,reject)=>{
stepsToProcess.forEachEntry(async function(entry, key) {
let payload = {
ticketId: key,
stepIds: entry
}
let response = await updateStep(payload)//returns promise
if (response.statusCode === 200) {
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsCompleted(stepId, payload.ticketId);
} catch(err) {
reject(err);
}
}));
}
else {
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
}));
}
});
resolve('Steps Marked Complete');
});//promise end
}
This is the path that i am testing right now that's making the call to the above method.
The updateStep() method is the method that actually calls a HTTP request to an outside REST API. That seems to be working returning a promise.
app.use('/api/posts',async (req,res,next)=>{
let data = await markStepsComplete.markStepsComplete()
.then((data)=>{
res.status(200).json({message: data})
})
.catch((e)=>{
console.log('error from catch:',e);
})
})
The code above runs and does run some stuff in our database as part of this process, but I get back the "Steps Marked Complete" resolve message before the process finishes.
Since the resolve is hit before the process ends, I can never get any of the reject() errors to come back to the client since resolve is called already.
Thanks in advance.

I noticed this snippet and wanted to call it out because it wont work:
payload.stepIds.forEach(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
});
when looping through an array of items that require an async fetch, you can do something like:
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
}));

Related

Struggling with calling a function that uses promises in node.js

I am struggling with some code... The 2 examples below I would think would work the same but the second example throws an error? I am also struggling to figure out the error, it's not bubbling up? Admittedly I am not a seasoned node developer so any guidance would be much appreciated! If it's relevant the create method in the module is calling the sequelize create.
This works
var p1 = deliverabiltyConfigs.create2(cfgObject);
return Promise.all([p1]).then(function([res1]) {
res.json({result: res1})
});
This does not
deliverabiltyConfigs.create2(cfgObject).then(res1 =>{
res.json({result: res1})
})
Here is the function that I am calling in a controller module
exports.create2 = (dConfig) => {
DeliverabilityConfig.create(dConfig)
.then(data => {
return data
})
.catch(err => {
return {
message:
err.message || "Some error occurred while createing this config."
};
});
};
The create2 function always returns null, so neither invocation will work. Promise.all([p1]) hides the problem, returning a promise to perform an array of no promises.
create2(cfgObject).then(res1 =>{ attempts to invoke then() on null, generating a more obvious error. But neither way works.
Fix by deciding which promise syntax you want, using each as follows:
Using original promise syntax....
exports.create2 = dConfig => {
// note the return
return DeliverabilityConfig.create(dConfig)
.catch(err => {
const message = err.message || "Some error occurred while createing this config.";
return { message };
});
};
// caller
deliverabiltyConfigs.create2(cfgObject).then(result =>{
res.json(result);
})
With recent syntactic sugar...
exports.create2 = async (dConfig) => {
try {
// its fine to not await here, since the caller will await
// but just to illustrate how you might perform more async work here...
return await DeliverabilityConfig.create(dConfig);
} catch (err) {
const message = err.message || "Some error occurred while createing this config."
return { message }
}
}
// caller
var result = await deliverabiltyConfigs.create2(cfgObject);
res.json(result);
Use Promise.all() to run >1 promise concurrently. You've only got one promise in the OP, so no reason for it here.

Problem getting puppeteer-cluster waiting on page event before closing

I'm currently setting up a CI environment to automate e2e tests our team runs in a test harness. I am setting this up on Gitlab and currently using Puppeteer. I have an event that fires from our test harness that designates when the test is complete. Now I am trying to "pool" the execution so I don't use up all resources or run out of listeners. I decided to try out "puppeteer-cluster" for this task. I am close to having things working, however I can't seem to get it to wait for the event on page before closing the browser. Prior to using puppeteer-cluster, I was passing in a callback to my function and when the custom event was fired (injected via exposeFunction), I would go about calling it. That callback function is now being passed in data though now and therefore not waiting. I can't seem to find a way to get the execution to wait and was hoping someone might have an idea here. If anyone has any recommendations, I'd love to hear them.
test('Should launch the browser and run e2e tests', async (done) => {
try {
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_CONTEXT,
maxConcurrency: 10,
monitor: false,
timeout: 1200000,
puppeteerOptions: browserConfig
});
// Print errors to console
cluster.on("taskerror", (err, data) => {
console.log(`Error crawling ${data}: ${err.message}`);
});
//Setup our task to be run
await cluster.task( async ({page, data: {testUrl, isLastIndex, cb}, worker}) => {
console.log(`Test starting at url: ${testUrl} - isLastIndex: ${isLastIndex}`);
await page.goto(testUrl);
await page.waitForSelector('#testHarness');
await page.exposeFunction('onCustomEvent', async (e) => {
if (isLastIndex === true){ ;
//Make a call to our callback, finalizing tests are complete
cb();
}
console.log(`Completed test at url: ${testUrl}`);
});
await page.evaluate(() => {
document.addEventListener('TEST_COMPLETE', (e) => {
window.onCustomEvent('TEST_COMPLETE');
console.log("TEST COMPLETE");
});
});
});
//Perform the assignment of all of our xml tests to an array
let arrOfTests = await buildTestArray();
const arrOfTestsLen = arrOfTests.length;
for( let i=0; i < arrOfTestsLen; ++i){
//push our tests on task queue
await cluster.queue( {testUrl: arrOfTests[i], isLastIndex: (i === arrOfTestsLen - 1), cb: done });
};
await cluster.idle();
await cluster.close();
} catch (error) {
console.log('ERROR:',error);
done();
throw error;
}
});
So I got something working, but it really feels hacky to me and I'm not really sure it is the right approach. So should anyone have the proper way of doing this or a more recommended way, don't hesitate to respond. I am posting here shoudl anyone else deal with something similar. I was able to get this working with a bool and setInterval. I have pasted working result below.
await cluster.task( async ({page, data: {testUrl, isLastIndex, cb}, worker}) => {
let complete = false;
console.log(`Test starting at url: ${testUrl} - isLastIndex: ${isLastIndex}`);
await page.goto(testUrl)
await page.waitForSelector('#testHarness');
await page.focus('#testHarness');
await page.exposeFunction('onCustomEvent', async (e) => {
console.log("Custom event fired");
if (isLastIndex === true){ ;
//Make a call to our callback, finalizing tests are complete
cb();
complete = true;
//console.log(`VAL IS ${complete}`);
}
console.log(`Completed test at url: ${testUrl}`);
});
//This will run on the actual page itself. So setup an event listener for
//the TEST_COMPLETE event sent from the test harness itself
await page.evaluate(() => {
document.addEventListener('TEST_COMPLETE', (e) => {
window.onCustomEvent('TEST_COMPLETE');
});
});
await new Promise(resolve => {
try {
let timerId = setInterval(()=>{
if (complete === true){
resolve();
clearInterval(timerId);
}
}, 1000);
} catch (e) {
console.log('ERROR ', e);
}
});
});

Multiple and fast external POST requests

I have a node.js server that making POST requests to an external API, each time I have to make ~10k requests (don't worry I'm not abusing the API) and I need that it will take around 2-3 minutes.
I'm using request-promise library in order to make the requests along with Promise.all() to wait for all the requests to resolve.
My problem is that the requests seems stuck and not running in parallel, I know that the promise executes as soon it's created but it seems that the resolve event can only listen to about 10 events at one time.
I tried updating the maxListeners and also using es6-promise-pool (with pool of 500) but no luck.
My next solution will probably be to use child-process with fork, will this solution seems the best for my problem?
Thanks!
code:
async function send_msg(msg) {
return new Promise(function (resolve, reject) {
request.post(options, function (err, res, body) {
if (err) {
logger.error('error sending msg ' + err);
resolve(null);
} else {
resolve(body);
}
})
});
}
}
async function send_msgs() {
let msgs = await OutgoingMessage.findAll();
for (let i = 0; i < msgs.length; i++) {
promises.push(send_msg(msgs[i]).then(async (result) => {
if (result != null) {
try {
let sid = result['MessageSid'];
let status = result['Status'];
msgs[i].update({sid: sid, status: status});
} catch (e) {
logger.error(e + JSON.stringify(result));
msgs[i].update({status: 'failed'});
}
}
}));
}
return Promise.all(promises);
}

How can i set a timeout on google cloud datastore .get?

I'm just start with some google cloud services, and I'm trying to get a entity from datastore.
If the client have internet connection, everything its going well.
But i want to put a try catch statement for the cases were the client have no access to datastore, due any reason (like internet).
Here's my code:
try{
let search = datastore.key(['Client', Client_id])
datastore.get(search, /*{timeout: 1000},*/ function (err, entity) {
console.log('limit >>>', entity.limit)
evt.emit('comparedate', res, entity.limit)
});
}
catch(error){
console.log('Error >>>', error)
}
My problem is: there is no time limit for connection attempt. When the client have no access to the internet the request keep "pending" forever, and don't go to the catch condition.
I tried some parameters like: Global#CallOptions, but with no success.
Thanks for any help!
EDIT >>>> I know that's not the most trustworthy way. But for now I resolved with this code:
evt.on('isonline', (res) => {
try{
require('dns').lookup('google.com',function(err) {
if (err && err.code == "ENOTFOUND") {
console.log('NO INTERNET')
evt.emit('readofflinedata', res)
} else {
console.log('WITH INTERNET')
evt.emit('readonlinedata', res)
}
})
}
catch(error){
res.status(200).send({ error: true, message: error.message })
}
})
The Datastore client uses a library internally called google-gax. You can configure timeouts/etc. by passing in gax options.
datastore.get(key, {
gaxOptions: {timeout: 1000}
}, (err, entity) => {
// ...
});
I didn't found any parameter to add a timeout in the get function of datastore. However you can use a Promise and set a timer, if the execution of the function takes too long it will stop it.
var Promise = require("bluebird");
var elt = new Promise((resolve, reject) => {
fun(param, (err) => {
if (err) reject(err);
doSomething(); // <- datastore.get() funtion
resolve();
});
elt.timeout(1000).then(() => console.log('done'))
.catch(Promise.TimeoutError, (e) => console.log("timed out"))

NodeJS fs.appendFile not working within promise in mocha

I want to keep a log during my integration test suite. I'm testing that every 'item' is being compiled and logging how much time it took. I'm using node 4.3.
First of all I create the log file:
before(function() {
if (!fs.existsSync("./log.csv")) {
fs.writeFile("./log.csv", "Name; Time");
}
});
Then within each it block I would do this:
for (const item of items) {
it('compiles', function() {
return item.testCompile();
});
}
And item class has these methods:
testCompile() {
return this
.buildItem()
.then(result => {
// whatever testing stuff
});
}
buildItem() {
return this
.internalLogicForCompiling()
.then(result => {
// This is not appending anything
fs.appendFile("./log.csv", `${item.name}; ${result.compileTime}`);
return result;
});
}
So far the file is created but never updated... Any clue what I'm doing wrong?
PS: I assume if the file doesn't exists, fs should throw an error, however it doesn't.
Your code is generally ignoring the fact that your fs calls are asynchronous. Promises are not magic. If you use code that is asynchronous but does not use promises, you need to do more than plop that code in a promise can call it done.
The easiest way to deal with the issue would be to use fs.writeFileSync and fs.appendFileSync instead of the calls you make. Otherwise, you should write your before like this:
before(function(done) {
if (!fs.existsSync("./log.csv")) {
fs.writeFile("./log.csv", "Name; Time", done);
}
});
I've just added the done callback.
And buildItem could be something like this so that the promise it returns won't resolve before appendFile is done doing its work:
buildItem() {
return this
.internalLogicForCompiling()
.then(result => {
return new Promise((resolve, reject) => {
fs.appendFile("./log.csv", `${item.name}; ${result.compileTime}`, (err) => {
if (err) {
reject(err);
return;
}
resolve(result);
});
});
});
}

Resources