I'm struggling trying to chain together three requests that require synchrony in node.js. Here is my attempt at using promises, but i am getting an error saying that db.run isn't a function. The first action should insert into my sqlite db. The most important thing i need is the
this.lastID variable, which lists the id of the last enetered item. Before attempting to use promises, I was having trouble with scoping. This is important because i need to take this value and use use it in my JSON object under the callback key. Lastly, Im using the requests npm package to send the request.
I am using the bluebird promises library, sqlite3 npm package, nodejs, express.
Any help with this would be awesome because I'm lost.
function db() {
return new Promise(function(resolve, reject) {
db.run(`INSERT INTO scan_requests(name, date) VALUES(?,?);`, [name,date], function(err) {
if (err) {
console.log(err)
}
let q = this.lastID
resolve(q)
})
})
}
db()
.then(function(q) {
let options = {
url: 'API',
body: {
name: req.name,
scan_callback: `http://localhost:80/${q}`
},
json: true
}
resolve(options)
}).then(function(options) {
console.log(options)
})
1st rule of "promises" ... always return your "promises". Except when you create a new one.
Try this ...
app.post('/route', function (req,res) {
new Promise(function(resolve, reject) {
db.run(`INSERT INTO scan_requests(req.name,req.date) VALUES(?,?);`, [name,date]).then(function(result) {
let options = {
url: 'http://API',
body: {
name: req.name,
date: req.date
callback: `http://localhost:80/${this.lastID}`,
},
json: true
}
// this resolves this promise ... it is now passed on
resolve(options);
}).then(function(options) {
// options is now the result from the promise
console.log(options)
request
.post(options)
.on('error', function(err) {
console.log(err)
})
.pipe(res)
});
});
});
UPDATE (question modified)
You're using resolve(options) but resolve is not in scope there (it doesn't exist). Remember the first rule of promises ...
db()
.then(function(q) {
let options = {
url: 'API',
body: {
name: req.name,
scan_callback: `http://localhost:80/${q}`
},
json: true
}
// *** change the following line ***
// --- you must return your data ---
return options;
}).then(function(options) {
console.log(options)
// -------------------------
// --- contrived example ---
// -------------------------
return { success: true };
}).then(status => {
console.log(`Success ${status.success}`);
});
The example includes a rather useless but illustrative example of how to continue passing data down the "promise chain".
Related
I have a route that uses the below shown method, in a node app that uses express.
I create a transaction but don't use it in the update method. Sequelize is configured to not use managed transactions and auto commit is set to false.
When this route is called multiple times/under load around 7 - 10 times per second (the number of calls differ), I end up with 5 or so dangling transactions even though commit is called for the transaction at the end of the method call.
(Because of these dangling transactions, subsequent calls and my node app is not able to make anymore db calls)
But if I pass the transaction in the params object this behaviour doesnt occur. And I don't get dangling transactions.
What can be the reason this is happening?
updateItem = (obj) => {
this.logDebug(`Updating item - `, obj);
return new Promise(async (resolve, reject) => {
let transaction;
try {
transaction = await this.getTransaction();
} catch(error) { return reject(error);
const params = {
where: {
id: obj.id
},
returning: true,
plain: true
};
return models[modelName].
update(obj, params).then(result => {
if (!result) { return result; }
result = JSON.parse(JSON.stringify(result[1]));
return result;
}).
then(async (result) => {
await transaction.commit();
return resolve(result);
}).
catch(async error => {
this.logError(`Failed to update - `, error);
await transaction.rollback();
return reject(error);
});
});
};
Causes dangling transaction.
const params = {
where: {
id: obj.id
},
returning: true,
plain: true
};
No dangling transactions occur.
const params = {
where: {
id: obj.id
},
returning: true,
plain: true,
transaction
};
Using Nodejs 12.4, Sequelize 5.21.9, postgres 9.x
Just using the created transaction in the params object somehow doesn't cause dangling transaction.
While NOT using the transaction in the params causes this issue of dangling transactions.
Wanted to know the cause of this behaviour? Is it a bug with my code? Or bug with Sequelize?
There are a few points here;
1- there is a missing curly bracket at the end of the first line (a simple type i guess)
2- you dont't have to use the returns other than the first one
3- it is not required to use double "then" fucntion at the end
4- if json parse fails your transaction probably leak
With all these the code may be updated like this;
updateItem = (obj) => {
this.logDebug(`Updating item - `, obj);
return new Promise(async (resolve, reject) => {
let transaction;
try {
transaction = await this.getTransaction();
} catch (error) { return reject(error); }
const params = {
where: {
id: obj.id
},
returning: true,
plain: true
};
return models[modelName].
update(obj, params).then(async (result) => {
try {
resolve(JSON.parse(JSON.stringify(result[1])));
await transaction.commit();
} catch (err) {
reject(err);
}
}).catch(async error => {
this.logError(`Failed to update - `, error);
await transaction.rollback(); // an extra try catch wont hurt here...
reject(error);
});
});
};
But things may get complicated if you want to make multiple updates in the future. I may suggest you to check Multiple Transaction Manager's Sequelize context for a clean way to arrange your transactions.
I have a set of functions in Node.js that I would like to load in a certain order. I will provide some mockup code abstracted and simplified:
function updateMyApp() {
loadDataToServer()
.then(() => useData())
.then(() => saveData())
.then(() => { console.log("updateMyApp done") })
}
function loadDataToServer() {
return new Promise( (resolve, reject) {
...preparing data and save file to cloud...
resolve()})
}
function handleDataItem(item) {
// Function that fetches data item from database and updates each data item
console.log("Name", item.name)
}
function saveData() {
// Saves the altered data to some place
}
useData is a bit more complex. In it I would like to, in order:
console.log('Starting alterData()')
Load data, as json, from the cloud data source
Iterate through every item in the json file and do handleDataItem(item) on it.
When #2 is done -> console.log('alterData() done')
Return a resolved promise back to updateMyApp
Go on with saveData() with all data altered.
I want the logs to show:
Starting useData()
Name: Adam
Name: Ben
Name: Casey
useData() done
my take on this is the following:
function useData() {
console.log('Starting useData()')
return new Promise( function(resolve, reject) {
readFromCloudFileserver()
.then(jsonListFromCloud) => {
jsonListFromCloud.forEach((item) => {
handleDataItem(item)
}
})
.then(() => {
resolve() // I put resolve here because it is not until everything is finished above that this function is finished
console.log('useData() done')
}).catch((error) => { console.error(error.message) })
})
}
which seems to work but, as far as I understand this is not how one is supposed to do it. Also, this seems to do the handleDataItem outside of this chain so the logs look like this:
Starting useData()
useData() done
Name: Adam
Name: Ben
Name: Casey
In other words. It doesn't seem like the handleDataItem() calls are finished when the chain has moved on to the next step (.then()). In other words, I can not be sure all items have been updated when it goes on to the saveData() function?
If this is not a good way to handle it, then how should these functions be written? How do I chain the functions properly to make sure everything is done in the right order (as well as making the log events appear in order)?
Edit: As per request, this is handleDataItem less abstracted.
function handleDataItem(data) {
return new Promise( async function (resolve) {
data['member'] = true
if (data['twitter']) {
const cleanedUsername = twitterApi.cleanUsername(data['twitter']).toLowerCase()
if (!data['twitter_numeric']) {
var twitterId = await twitterApi.getTwitterIdFromUsername(cleanedUsername)
if (twitterId) {
data['twitter_numeric'] = twitterId
}
}
if (data['twitter_numeric']) {
if (data['twitter_protected'] != undefined) {
var twitterInfo = await twitterApi.getTwitterGeneralInfoToDb(data['twitter_numeric'])
data['twitter_description'] = twitterInfo.description
data['twitter_protected'] = twitterInfo.protected
data['twitter_profile_pic'] = twitterInfo.profile_image_url.replace("_normal", '_bigger')
data['twitter_status'] = 2
console.log("Tweeter: ", data)
}
} else {
data['twitter_status'] = 1
}
}
resolve(data)
}).then( (data) => {
db.collection('people').doc(data.marker).set(data)
db.collection('people').doc(data.marker).collection('positions').doc(data['report_at']).set(
{
"lat":data['lat'],
"lon":data['lon'],
}
)
}).catch( (error) => { console.log(error) })
}
The twitterAPI functions called:
cleanUsername: function (givenUsername) {
return givenUsername.split('/').pop().replace('#', '').replace('#', '').split(" ").join("").split("?")[0].trim().toLowerCase()
},
getTwitterGeneralInfoToDb: async function (twitter_id) {
var endpointURL = "https://api.twitter.com/2/users/" + twitter_id
var params = {
"user.fields": "name,description,profile_image_url,protected"
}
// this is the HTTP header that adds bearer token authentication
return new Promise( (resolve,reject) => {
needle('get', endpointURL, params, {
headers: {
"User-Agent": "v2UserLookupJS",
"authorization": `Bearer ${TWITTER_TOKEN}`
}
}).then( (res) => {
console.log("result.body", res.body);
if (res.body['errors']) {
if (res.body['errors'][0]['title'] == undefined) {
reject("Twitter API returns undefined error for :'", cleanUsername, "'")
} else {
reject("Twitter API returns error:", res.body['errors'][0]['title'], res.body['errors'][0]['detail'])
}
} else {
resolve(res.body.data)
}
}).catch( (error) => { console.error(error.message) })
})
},
// Get unique id from Twitter user
// Twitter API
getTwitterIdFromUsername: async function (cleanUsername) {
const endpointURL = "https://api.twitter.com/2/users/by?usernames="
const params = {
usernames: cleanUsername, // Edit usernames to look up
}
// this is the HTTP header that adds bearer token authentication
const res = await needle('get', endpointURL, params, {
headers: {
"User-Agent": "v2UserLookupJS",
"authorization": `Bearer ${TWITTER_TOKEN}`
}
})
if (res.body['errors']) {
if (res.body['errors'][0]) {
if (res.body['errors'][0]['title'] == undefined) {
console.error("Twitter API returns undefined error for :'", cleanUsername, "'")
} else {
console.error("Twitter API returns error:", res.body['errors'][0]['title'], res.body['errors'][0]['detail'])
}
} else {
console.error("Twitter API special error:", res.body)
}
} else {
if (res.body['data']) {
return res.body['data'][0].id
} else {
//console.log("??? Could not return ID, despite no error. See: ", res.body)
}
}
},
You have 3 options to deal with your main issue of async methods in a loop.
Instead of forEach, use map and return promises. Then use Promise.all on the returned promises to wait for them to all complete.
Use a for/of loop in combination with async/await.
Use a for await loop.
It sounds like there's a problem in the implementation of handleDataItem() and the promise that it returns. To help you with that, we need to see the code for that function.
You also need to clean up useData() so that it properly returns a promise that propagates both completion and errors.
And, if handleDataItem() returns a promise that is accurate, then you need to change how you do that in a loop here also.
Change from this:
function useData() {
console.log('Starting useData()')
return new Promise( function(resolve, reject) {
readFromCloudFileserver()
.then(jsonListFromCloud) => {
jsonListFromCloud.forEach((item) => {
handleDataItem(item)
}
})
.then(() => {
resolve() // I put resolve here because it is not until everything is finished above that this function is finished
console.log('useData() done')
}).catch((error) => { console.error(error.message) })
})
}
to this:
async function useData() {
try {
console.log('Starting useData()')
const jsonListFromCloud = await readFromCloudFileserver();
for (let item of jsonListFromCloud) {
await handleDataItem(item);
}
console.log('useData() done');
} catch (error) {
// log error and rethrow so caller gets the error
console.error(error.message)
throw error;
}
}
The structural changes here are:
Switch to use async/await to more easily handle the asynchronous items in a loop
Remove the promise anti-pattern that wraps new Promise() around an existing promise - no need for that AND you weren't capturing or propagating rejections from readFromCloudFileServer() which is a common mistake when using that anti-pattern.
rethrow the error inside your catch after logging the error so the error gets propagated back to the caller
I'm working on a Slackbot that compares a repo branch file with the same file in Master, which requires making two API calls to bitbucket's API. The first grabs all the most recent branches in our workspace, which includes a URL that I can then use to call the API for the difference between the two files:
Snippet from NodeJS Controller File
let diffs = await axios.get('API-Call-To-BB-For-Recent-Branches', {
headers: {
'Authorization': `${AUTH}`
}
})
let filteredByUser = diffs.data.values.filter(element => {
if (element.target.author.raw.includes(username)) {
if (element.target.repository.full_name.includes("nameOfMasterBranch")) {
axios.get(element.target.links.diff.href, {
headers: {
'Authorization': `${AUTH}`
}
}).then(response => {
let clippedBranch = {
branch: element.name,
author: element.target.author.user.display_name,
diff: response.data
}
// Console Logging here shows the data I'm looking for
console.log(clippedBranch)
return clippedBranch
}).catch(error => {
console.log(error)
})
}
}
})
// Console logging Here returns an empty array.
console.log(filteredByUser)
// Console logging the returned value on the main server file returns a Promise<Pending>
return filteredByUser
What I've Tried
I've tried using a Promise.resolve and Promise.All to fix the issue.
Making the second API call inside of a for Of statement and a forEach statement
I've tried nesting the array processing and second API call inside of a .then on the first API call to BitBucket.
Whats preventing the data from being resolved in time to be returned?
Thanks in advance for your time!
Your problem is in the .filter loop. You are calling axios.get but not doing anything with the Promise it returns. The filter function should return true if the object is to be kept and false if not. But it returns nothing at all (which is equivalent to false. You might think it returns clippedBranch but it doesn't.
I suspect you want filteredByUser to be an array of clippedBranch objects, but it will end up being an empty array.
The following code will get you further:
let diffs = await axios.get("API-Call-To-BB-For-Recent-Branches", {
headers: {
Authorization: `${AUTH}`
}
});
const filteredByUser = [];
// use forEach instead of filter
diffs.data.values.forEach(async element => { // <--- use async to allow await
if (element.target.author.raw.includes(username)) {
if (element.target.repository.full_name.includes("nameOfMasterBranch")) {
const clippedBranch = await axios.get( // <--- wait for the result!
element.target.links.diff.href, {
headers: {
Authorization: `${AUTH}`
}
})
.then(response => {
let clippedBranch = {
branch: element.name,
author: element.target.author.user.display_name,
diff: response.data
};
// Console Logging here shows the data I'm looking for
console.log(clippedBranch);
return clippedBranch;
})
.catch(error => {
console.log(error);
});
if (clippedBranch) {
filteredByUser.push(clippedBranch); // <-- add to array if exists
}
}
}
});
Your original code didn't do anything with the value that was finally resolved by the axios.get(href). In fact those calls would complete long after your function ended since you didn't wait on them at all.
I used forEach instead of filter because we want to process each entry but not simply keep or discard it (which is what filter is for). We want to build a new array, so we just push the things we want onto it if found while looping through each entry.
You might want to chain calls to both .filter and .map separately. Not only will this "flatten" your code but it should read a bit easier as well. Use filter first to remove elements that you don't need. Then map through the resulting list and return a list of promises which can be passed to Promise.all
Here is what I believe you're attempting to do:
async function getBranchesByUsername(username) {
try {
const diffs = await axios.get("API-Call-To-BB-For-Recent-Branches", {
headers: {
Authorization: `${AUTH}`,
},
});
const requests = diffs.data.values
.filter(element => {
return element.target.author.raw.includes(username) &&
element.target.repository.full_name.includes("nameOfMasterBranch")
})
.map(element => {
return axios
.get(element.target.links.diff.href, {
headers: {
Authorization: `${AUTH}`,
},
})
.then((response) => {
const clippedBranch = {
branch: element.name,
author: element.target.author.user.display_name,
diff: response.data,
};
console.log(clippedBranch);
return clippedBranch;
});
});
return await Promise.all(requests)
} catch (error) {
console.log(error)
throw error
}
}
I have not been able to test this but the concept still applies
I have a mongoose schema like the following:
const User: Schema = new Schema({
// some other fields
email: {type: String, unique: true, require: true, validate: [myValidator, 'invalid email provided'],
// some more fields
)};
My myValidator uses validatorJS isEmail function to check if a valid email address has been entered and looks like this:
function myValidator(email: String) {
return isEmail(email: String, {require_tld: true, domain_specific_validation: true});
So far all of this works as expected. But now I have the following problem, which might be linked to my limited understanding of Typescript/JaveScript.
I would like to extend myValidator to check the entered email address against an external API.
So far I've tried something like this:
function myValidator(email: String) {
let isValid = false;
request('https://disposable.debounce.io/?email=' + email, { json: true }, async (err, res, body) => {
if (err) { return console.log(err); }
isValid = body.disposable;
});
if (isValid && !isEmail(email, {require_tld: true, domain_specific_validation: true})) {
return false;
}
But this obviously fails as request is an async operation so isValid will always be false
So I 'attempted' to make this function an async/await function.
function myValidator(email: String) {
const remoteCheck = function() {
return new Promise(function() {
request('https://disposable.debounce.io/?email=' + email, { json: true }, async (err, res, body) => {
if (err) { return console.log(err); }
return body.disposable;
});
});
};
async function f() {
const isValid = await remoteCheck();
if (isValid === true) {
return false;
} else {
isEmail(email, {require_tld: true, domain_specific_validation: true});
}
}
if(f()) {
return true;
} else {
return false;
}
But this gives me an error, because this function is not assignable to validate.
Please note that this is my first attempt on async/await functions. Could somebody point out what I'm missing here?
UPDATE:
So after I've spend a whole day now to familiarize myself with the concepts of callbacks and promises I came up with a working solution. I'll provide it as an answer below
I checked SO and I couldn't find a solution to this. So if anybody comes across this issue again, I'll provide a working solution for this.
If you want to validate a field against an external data-source, try the following. This validator will check the email-address provided against an API (you can of course use any other API as well) that checks if the address is a DEA and also use ValidatorJS isEmail to check for syntax:
const YourSchema = new Schema({
// your fields
})
SchemaName.path('fieldNameToCheck').validate({
validator: function(value) {
return new Promise(function (resolve, reject) {
request.get('https://disposable.debounce.io/?email=' + value, function (error, response, data) {
if (error) reject(error);
const content = JSON.parse(data);
resolve((content.disposable === 'false' && isEmail(value)))
}
});
}
});
The mistake I made was that I tried to resolve my Promise outside of the callback_function of my request. But the data I want to use is obviously only available inside of it on time.
EDIT:
for the purpose of understanding the API call. The API returns a JSON in the form of {disposable: boolean} whether the entered email-address belongs to a DEA (disposable Email Address) service
With the request library, is there a way to use promises to simplify this callback?
var context = {};
request.get({
url: someURL,
}, function(err, response, body) {
context.one = JSON.parse(body);
request.get({
url: anotherURL,
}, function(err, response, body) {
context.two = JSON.parse(body);
// render page
res.render('pages/myPage');
});
});
Here's a solution using the Bluebird promises library. This serializes the two requests and accumulates the results in the context object and rolls up error handling all to one place:
var Promise = require("bluebird");
var request = Promise.promisifyAll(require("request"), {multiArgs: true});
var context = {};
request.getAsync(someURL).spread(function(response, body) {
context.one = JSON.parse(body);
return request.getAsync(anotherURL);
}).spread(response, body)
context.two = JSON.parse(body);
// render page
res.render('pages/myPage');
}).catch(function(err) {
// error here
});
And, if you have multiple URLs, you can use some of Bluebirds other features like Promise.map() to iterate an array of URLs:
var Promise = require("bluebird");
var request = Promise.promisifyAll(require("request"), {multiArgs: true});
var urlList = ["url1", "url2", "url3"];
Promise.map(urlList, function(url) {
return request.getAsync(url).spread(function(response,body) {
return [JSON.parse(body),url];
});
}).then(function(results) {
// results is an array of all the parsed bodies in order
}).catch(function(err) {
// handle error here
});
Or, you could create a helper function to do this for you:
// pass an array of URLs
function getBodies(array) {
return Promise.map(urlList, function(url) {
return request.getAsync(url).spread(function(response.body) {
return JSON.parse(body);
});
});
});
// sample usage of helper function
getBodies(["url1", "url2", "url3"]).then(function(results) {
// process results array here
}).catch(function(err) {
// process error here
});
Here is how I would implement chained Promises.
var request = require("request");
var someURL = 'http://ip.jsontest.com/';
var anotherURL = 'http://ip.jsontest.com/';
function combinePromises(context){
return Promise.all(
[someURL, anotherURL].map((url, i)=> {
return new Promise(function(resolve, reject){
try{
request.get({
url: url,
}, function(err, response, body) {
if(err){
reject(err);
}else{
context[i+1] = JSON.parse(body);
resolve(1); //you can send back anything you want here
}
});
}catch(error){
reject(error);
}
});
})
);
}
var context = {"1": "", "2": ""};
combinePromises(context)
.then(function(response){
console.log(context);
//render page
res.render('pages/myPage');
}, function(error){
//do something with error here
});
Doing this with native Promises. It's good to understand the guts.
This here is known as the "Promise Constructor Antipattern" as pointed out by #Bergi in the comments. Don't do this. Check out the better method below.
var contextA = new Promise(function(resolve, reject) {
request('http://someurl.com', function(err, response, body) {
if(err) reject(err);
else {
resolve(body.toJSON());
}
});
});
var contextB = new Promise(function(resolve, reject) {
request('http://contextB.com', function(err, response, contextB) {
if(err) reject(err);
else {
contextA.then(function(contextA) {
res.render('page', contextA, contextB);
});
}
});
});
The nifty trick here, and I think by using raw promises you come to appreciate this, is that contextA resolves once and then we have access to it's resolved result. This is, we never make the above request to someurl.com, but still have access to contextA's JSON.
So I can conceivable create a contextC and reuse the JSON without having to make another request. Promises always only resolve once. You would have to take that anonymous executor function out and put it in a new Promise to refresh that data.
Bonus note:
This executes contextA and contextB in parallel, but will do the final computation that needs both contexts when both A & B are resolved.
Here's my new stab at this.
The main problem with the above solution is none of the promises are reusable and they are not chained which is a key feature of Promises.
However, I still recommend promisifying your request library yourself and abstaining from adding another dependency to your project. Another benefit of promisifying yourself is you can write your own rejection logic. This is important if you're working with a particular API that sends error messages in the body. Let's take a look:
//Function that returns a new Promise. Beats out constructor anti-pattern.
const asyncReq = function(options) {
return new Promise(function (resolve, reject) {
request(options, function(err, response, body) {
//Rejected promises can be dealt with in a `catch` block.
if(err) {
return reject(err);
}
//custom error handling logic for your application.
else if (hasError(body)) {
return reject(toError(body));
}
// typically I just `resolve` `res` since it contains `body`.
return resolve(res);
}
});
};
asyncReq(urlA)
.then(function(resA) {
//Promise.all is the preferred method for managing nested context.
return Promise.all([resA, asyncReq(urlB)]);
})
.then(function(resAB) {
return render('page', resAB[0], resAB[1]);
})
.catch(function(e) {
console.err(e);
});
You can use the request-promise library to do this. In your case, you could have something like this, where you chain your requests.
request
.get({ url: someURL })
.then(body => {
context.one = JSON.parse(body);
// Resolves the promise
return request.get({ url: anotherURL });
})
.then(body => {
context.two = JSON.parse(body);
res.render('pages/myPage');
})
.catch(e => {
//Catch errors
console.log('Error:', e);
});
By far the easiest is to use request-promise library. You can also use use a promise library like bluebird and use its promisify functions to convert the request callback API to a promise API, though you may need to write your own promisify function as request does not use the standard callback semantics. Lastly, you can just make your own promise wrapper, using either native promises or bluebird.
If you're starting fresh, just use request-promise. If you're refactoring existing code, I would just write a simple wrapper for request using bluebird's spread function.