I want a suggestion on how to perform nodejs-DB operations
Here are my options
case one : When you have to run more than one query and former one is dependent on the latter
first way
connection.query(Q1, function(error, results, fields) {
if (error) {
throw error;
}
console.log(results[0]);
//call the second query here
connection.query(Q2, function(error, results, fields) {
if (error) {
throw error;
}
console.log(results[0]);
});
});
or
define first query in a function return the result via promise and await function
call the second query in the same way
use both results
like
var res1 = await funcName1(params);
var res2 = await funcName2(params);// params deducted from res1
which one is better?
case two : when we want to call select query more than 1 times
and capture the results in an array
the last one is pretty preplexing, do i have to call the mysql async function in a loop? is it a correct way?
Can you guys redirect me in the correct way?
I would suggest using the second option, or rather a similar approach to the second option. I believe the code is much easier to read and simpler to maintain. You'll also end up with a fraction of the number of lines of code.
It's also worth nothing that many database modules such as mysql2 and mongodb return promises directly, so there's no need to create wrapper functions, that is you won't need to maintain a load of funcName1, funcName2.. etc.
for example using mysql2:
const mysql = require('mysql2/promise');
const connection = await mysql.createConnection({
host: 'localhost',
user: 'some_user',
password: '***',
database: 'test_db'
});
let [users] = await connection.query('select * from users');
let [accounts] = await connection.query('select * from accounts');
console.log( { users, accounts })
If you wish to create promisified wrapper functions for a generic db access proc, have a look at Util.promisify, this allows you to cut down on boilerplate code.
e.g.
const dbQuery = util.promisify(db.query);
Then you can use like
let result = await dbQuery(param1, param2...);
As long as db.query looks something like
db.query(query, callback(err, result))
Also, querying within loops should work very well, for example:
async function testLoopQuery() {
for(let organization of organizationList) {
let [users] = await connection.query('select * from users where org_id = ?', [organization.id]);
}
}
Related
I'm scanning all items from a DynamoDB table - within a Lambda function - with DocumentClient. I'm then looping through each item and extracting the payload that I need. I'll use that item from the payload as a parameter with ExpressionAttributeValues in a new query.
Everything works dandy independently. The issue is with the use of the asynchronous function queryItems when nested within an array forEach() method. I getting a parsing error with the function queryItems. I can query the table when I call the function outside of the loop but how else am I going to query each item independently?
I'm not sure how to handle this.
'use strict';
const aws = require('aws-sdk');
const docClient = new aws.DynamoDB.DocumentClient();
var paramsAll = {
TableName: 'MyTable',
Select: "ALL_ATTRIBUTES"
};
exports.handler = async (event, context) => {
try {
let arr = [];
let sequence = '';
//scan all items in table
docClient.scan(paramsAll, function(err, data) {
if (err) {
//handle error
}
else {
//Loop through each item in the table:
let items = (data.Items);
items.forEach(function(Item) {
let p = (Item.payload);
//Extract sequence from the payload
sequence = (p.seq);
arr.push(sequence);
//perform other function with this array (not listed for brevity)
});
//Here is where I'm having the issue:
arr.forEach(function(Item) {
//Pass these items as a paramater within queryItems function but getting Parsing Error: unexpected token queryItems
const results = await queryItems(Item);
//do something with the results...
})
}
});
}
catch (err) {
return { error: err };
}
};
async function queryItems(p) {
try {
var params = {
TableName: 'MyTable',
KeyConditionExpression: '#seq = :value',
ExpressionAttributeValues: { ':value': p },
ExpressionAttributeNames: { '#seq': 'seq' }
};
const data = await docClient.query(params).promise();
return data;
}
catch (err) {
return err;
}
}
I've definitely run into a similar issue. What I believe is happening is just a Javascript syntax issue, where awaiting queryItems inside the synchronous function provided to forEach will produce an error. (Although, when running the code, I do get the specific error "SyntaxError: await is only valid in async functions and the top level bodies of modules", so there might be something else going on.)
I see nothing wrong with the DynamoDB queries, but hoangdv's suggestions are spot on. Specifically, I'd also suggest using the promise style for scan, and while a for...loop will definitely work, using Promise.all and map will be a lot quicker to complete all the queries. Here's how I'd modify the code:
'use strict';
const aws = require('aws-sdk');
const docClient = new aws.DynamoDB.DocumentClient();
// avoid var unless you specifically require it's hoisting behavior.
const paramsAll = {
TableName: 'MyTable',
Select: "ALL_ATTRIBUTES" // most likely not needed, I'd review this section of the docs: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html#DDB-Scan-request-Select
};
exports.handler = async (event, context) => {
try {
// unless you need to assign a new array to this variable, it is better practice to use const instead.
const arr = [];
// let sequence = ''; // see line 24 for why I commented this out.
// scan all items in table.
// Destructure Items out of the response.
// You may also need to continue scanning with the LastEvaluatedKey depending on the size of your table, and/or your use case.
// You'd continue scanning in a while loop, for example.
const { Items, LastEvaluatedKey } = await docClient.scan(paramsAll).promise();
// push the sequence to the arr.
// There is most likely a reason you omitted for brevity to have sequence defined above,
// but since this example doesn't need it above, I've omitted it entirely
Items.forEach(Item => {
const p = Item.payload;
arr.push(p.seq);
});
// use a for loop or map here instead. forEach will return undefined, which cannot be await'ed.
// instead, map will return a new array of Promises (since the callback is async).
// Then, you can use Promise.all to await until each Promise in the array is resolved.
// Keep in mind, depending on how many items you are iterating through, you may run into DynamoDB's ThrottlingException.
// You would have to batch the queries (in other words, split the arr into pieces, and iterate over each piece), which would have to be done before using map. Then, sleep for a few milliseconds before starting on the next piece.
// I doubt the queries will be quick enough to cause this when using a for loop, though.
await Promise.all(arr.map(async Item => {
const results = await queryItems(Item);
// do something with the results...
}));
}
catch (err) {
// Again, not sure what the use case is, but just FYI this is not a valid return value if this lambda function is intended for use with using API Gateway.
// See here :) https://docs.aws.amazon.com/lambda/latest/dg/services-apigateway.html#apigateway-types-transforms
return { error: err };
}
};
// Presumably, MyTable has a partitionKey of seq, otherwise this KeyConditionExpression is invalid.
async function queryItems(p) {
try {
var params = {
TableName: 'MyTable',
KeyConditionExpression: '#seq = :value',
ExpressionAttributeValues: { ':value': p },
ExpressionAttributeNames: { '#seq': 'seq' }
};
const data = await docClient.query(params).promise();
return data;
}
catch (err) {
return err;
}
}
Your issue is how you await on the for loop, its best to use Promise.all() with a map to await inside of a loop:
await Promise.all(arr.map(async Item => {
const results = await queryItems(Item);
// do something with the results...
}));
However, I cannot seem to understand your logic really well.
You Scan a table called MyTable, but you do not paginate, meaning you are only getting up to 1MB worth of data.
With the results, you strip out the seq value and then once again read every item from MyTable this time using a Query and seq as the key?
How to use multi query like find , update , insert , delete in one service on mongodb
I can query this below
router.get('/userlist', function(req, res) {
User.find({},function (err, docs) {
res.json(docs);
});
});
but i want to query like this is an error
router.get('/userlist', function(req, res) {
var data = array();
User.find({},function (err, docs) {
data['data1'] = docs
})
Content.find({},function (err, docs) {
data['data2'] = docs
})
res.json(docs)
});
Can anyone help me or It is possible to use query like this?
Thank you
You can use async await to run multiple mongo queries inside a function like below:
router.get('/userlist', async function(req, res) {
var data = array();
try{
//first query
let user_data = await User.find({}).exec();
data['data1'] = user_data;
//second query
let content_data = await Content.find({}).exec();
data['data2'] = content_data;
return res.status(200).json(data)
}
catch(err){
return res.status(400).json({err})
}
});
My question is what is stopping you? Yes, you can do this. I did those kinds of works.
There remain two points.
If you need to perform multiple queries, based on the response of the previous query, you can use async/await or promises. If you don't wanna try those, there is still a way. You can check the response of the previous query, make conditions, and if right, execute the seconde one. And thus you can make a chain of queries.
And if you don't need to rely on the responses, do whatever you want. There is nothing wrong...
Happy Coding!
You have to wait for your database call response. So, You can use promise for this. MDN: Promise Make promise of database call and resolve that after you got data from database. Promise.all will wait until your both the promises resolved. MDN: Promise.all()
router.get('/userlist', function(req, res) {
var data = [];
let promise1 = new Promise((resolve, reject) => {
User.find({},function (err, docs) {
data['data1'] = docs;
resolve();
})
});
let promise2 = new Promise((resolve, reject) => {
Content.find({},function (err, docs) {
data['data2'] = docs
})
});
Promise.all([promise1, promise2]).then(result => res.json(docs))
});
It's my code:
const mariadb = require('mariadb');
var test = async function(){
var ret = "";
await mariadb.createConnection({
host: "localhost",
user: "dave",
connectionLimit: 5,
password: "!##",
database: "db",
rowsAsArray: false
}).then((data)=>{
ret = "test";
});
return ret;
}
console.log(test());
How can I get a return from then using await?
I know that similar questions have been raised in many cases, but I could not find any using mariadb.
Already I can see a few pitfalls, which as beginners you will come across in this code:
avait is waiting for results to be returned, as he will get Promisses to solve it
then solves promises asynchronously
function defined as async always returns promises
After a moment of delight, 'async / await' now I try to avoid him. Mainly because every function in which I use the asynchronous function must also be asynchronous until you use then, eg make small test:
let test = async function() {
let x = await 11
return 1;
}
console.log(test()) //returns: Promise { <pending> }
There is nothing asynchronous there, but adding async / await has caused a mess.
Now fixes for your code
const mariadb = require('mariadb');
// use local scope `let` instead global `var`
let test = async function(){
let conn = await mariadb.createConnection({
host: "localhost",
user: "dave",
connectionLimit: 5,
password: "!##",
database: "db",
rowsAsArray: false
});
return conn.query("SELECT 1 as val") // no sense using `avait` because `test()` returns `promise`
}
test().then(function(rows) { console.log(rows)});
and without asnc. then can return promise and this can be resolbed by next then
mariadb.createConnection(...).then(conn => { // create connection
return conn.query("SELECT 1 as val") // make query
}).then(rows => { //get result
console.log(rows)
}).catch(err => console.error(err)) // get errors
By the way: get interested in query builder, for example knex.js. It allow write code independent from database engine.
Update
Let's start with the fact that Node is based on events.
Let's take an example of receiving data from the database. In PHP / C ++, you make an query, wait, receive a result. And this behavior simulates by the await. (Await appeared somewhere near version 8)
Normally, the code in the Node works so that you execute the query, and Node creates a new thread. Old run next instruction, in new you will get the results. (OK, I'm lying, but it's easier to explain).
So, you have to handle the event of receiving the data. And more specifically the promise of providing data. By await,.then (), orcallback (hell)
first code explanation:
You try return ret, but this code first make return ret and after make assigment.
await "returns" data, so you should use let var_name = await asyncFunction()
I'm guess, that you want this:
let getSomeDataFromDB = function(){
return mariadb.createConnection([skip]).then(conn => {
return conn.query("SELECT 1 as val")
})
}
let rows = await getSomeDataFromDB()
In this function you return promise, who return promise. And by await this promise chain is resolved.
But here is a "small" error in the code. Because you are connecting and you are not ending your connection anywhere. Therefore, it is better to have a global connection object, or use something like this:
let getSomeDataFromDB = function(){
return new Promise(function(resolve, reject){
mariadb.createConnection([skip]).then(conn => {
conn.query("SELECT 1 as val")
.then(rows=>resolve(rows))
.catch(e=>reject(e))
.then(()=>conn.close())
}).catch(e => reject(e))
})
}
let rows = await getSomeDataFromDB()
And here you discover another important thing: resolve does not interrupt the execution of the code. Despite the data being returned to the user, you can still do something.
Or the same with await
let getSomeDataFromDB = async function(){
let conn = await reateConnection([skip])
let rows = await conn.query("SELECT 1 as val")
await conn.close();
return rows;
}
let rows = await getSomeDataFromDB()
I am developing an Express nodeJS web app and am trying to increase my knowledge of asynchronous programming and how to access the data that I need properly.
I have two callback functions which query an SQL database. My functions are declared like so:
function CustomerList(callback) {
const db = require('../db');
db.query('SELECT * FROM users WHERE role = (?)', ['User'], (error, results, fields) => {
if (error) throw error;
callback(results);
});
}
function AdminList(callback) {
const db = require('../db');
db.query('SELECT * FROM admins WHERE role = (?)', ['Moderator'], (error, results, fields) => {
if (error) throw error;
callback(results);
});
}
I use the data retrieved from both callbacks and send it to my view engine template to be displayed in the browser like so:
CustomerList((userData) => {
AdminList((adminData) => {
res.render('adminhome', {
title: 'Admin - Home',
results: 'test',
customers: userData,
admins: adminData
});
});
This works as expected and I can manipulate the data however I please on my view template. But... This seems... 'clunky', and I feel like this method I am using is going to cause me trouble in the future if I create additional functions to retrieve more data, and then keep nesting additional callbacks to pass the data to my view template.
Essentially, I am struggling to find a suitable approach to retrieve the UserData results and AdminData results and pass them to my res.render API call.
What are better alternatives to my current approach?
Yes, it'll create callback hell if you call multiple callbacks inside the callback, use promises(async/await)
const util = require('util');
const db = require('../db');
const query = util.promisify(db.query);
const getData = async () => {
try {
const CustomerList = await query('SELECT * FROM users WHERE role = (?)', ['User']);
const AdminList = await query('SELECT * FROM admins WHERE role = (?)', ['Moderator']);
// OR Promise.all
const [CustomerList, AdminList] = await Promise.all([
query('SELECT * FROM users WHERE role = (?)', ['User']),
query('SELECT * FROM admins WHERE role = (?)', ['Moderator'])
]);
} catch (error) {
console.log(error);
}
}
Complete NodeJS testing noob here. Trying to individually test functions that are called through my API (meaning, rather than make an http request to a specific endpoint, which usually invokes several functions, which in turn make requests to different third party APIs, I want to test the functions themselves separately). The way they're called is I've built a class for each data source (data source = third party API), each class contains the same functions with the same exact signatures - getData and convertData, and return a callback with the results.
I've also created a module that creates many user mocks, since each user context returns different data (meaning, a user object is fed into getData, which uses certain user properties in order to determine what data should be returned).
The way I wanted to test this was to create numerous mocks, then run the functions for each. This is what I've got so far:
// Data sources to iterate over. Each is a class instance acquired through "require".
var dataSources = [
source1,
source2,
source3,
source4
];
describe('getData', function() {
this.timeout(10000);
describe('per data source,', function() {
context('standard call', function() {
// Associative array to hold the data returned, a key for each data source.
var finalResults = {};
// Iterate over all data sources
_.forEach(dataSources, function(dataSource) {
// Generate user mocks
var users = userMocks(10);
// Iterate over all users.
_.forEach(users, function (user) {
// Call each data source with each of the users.
// Numbers of calls to make - (users * data-sources), so in this case - 10*4.
dataSource.getData(user, function (err, data) {
if (err) return done(err);
// Convert the data returned to my format
dataSource.convertData(data, function (err, processedData) {
if (err) return done(err);
// Populate finalResults with converted data from each source
if (finalResults[dataSource.sourceName]) {
finalResults[dataSource.sourceName] = finalResults[dataSource.sourceName].concat(processedData);
} else {
finalResults[dataSource.sourceName] = processedData;
}
});
});
});
});
it('should return something', function(done) {
_.forEach(finalResults.keys, function(key) {
expect(finalResults[key]).to.not.be.empty;
expect(finalResults[key].length).to.be.greaterThan(0);
});
setTimeout(function() {
done();
}, 10000);
})
});
});
});
});`
This works (or at least the test passes when the query is valid, which is what I wanted), but it's cumbersome and (so very) far from elegant or effective, specifically the usage of timeout rather than using promises, async of some sort, or maybe a different alternative I'm not yet familiar with.
Since most of the resources I found (http://alanhollis.com/node-js-testing-a-node-js-api-with-mocha-async-and-should/, https://developmentnow.com/2015/02/05/make-your-node-js-api-bulletproof-how-to-test-with-mocha-chai-and-supertest/, https://justinbellamy.com/testing-async-code-with-mocha/, just to name a few) discuss direct API testing rather than specific async functions, I would love to get some input/best practices tips from more experienced Noders.
You need to know when bunch of asynchronous operations complete. Elegant way to test that is to use promises and promise aggregation:
Promise.all([ promise1, promise2, promise3 ]).then(function(results) {
// all my promises are fulfilled here, and results is an array of results
});
Wrap your dataSources into a promises using bluebird. You don't need to modify tested code self, bluebird provides convenience method:
var Promise = require('bluebird')
var dataSources = [
source1,
source2,
source3,
source4
].map(Promise.promisifyAll);
Use newly promisified functions to create promise for each call:
context('standard call', function() {
var finalResults = {};
var promiseOfResults = datasources.map(function(dataSource) {
var users = userMocks(10);
// Promise.all will take an array of promises and return a promise that is fulfilled then all of promises are
return Promise.all( users.map(function(user) {
// *Async functions are generated by bluebird, via Promise.promisifyAll
return dataSource.getDataAsync(user)
.then(dataSource.convertDataAsync)
.then(function(processedData) {
if (finalResults[dataSource.sourceName]) {
finalResults[dataSource.sourceName] = finalResults[dataSource.sourceName].concat(processedData);
} else {
finalResults[dataSource.sourceName] = processedData;
}
});
});
});
// promiseOfResults consists now of array of agregated promises
it('should return something', function(done) {
// Promise.all agregates all od your 'datasource' promises and is fulfilled when all of them are
// You don't need the promise result here, since you agegated finalResults yourself
return Promise.all( promiseOfResults ).then(function() {
_.forEach(finalResults.keys, function(key) {
expect(finalResults[key]).to.not.be.empty;
expect(finalResults[key].length).to.be.greaterThan(0);
});
done();
});
});
Rest of your test should use same Promise.all( promiseOfResults ), unless you need new set of results.