I am in the process of converting a large system over from a standard sequential programming which means I am planning on converting 1,500+ functions. These are simple functions that (for the most part) do basic select/insert/update/delete on tables.
Because I am about to be duplicating SO MANY functions, I want to ensure I am making the best decisions w/ syntax and structure.
My plan so far was to create a functions file for each table in the database and to create simple promise-based functions for these CRUD operations. I export these functions and can require the correct files (by table-name) for any route that needs to use a function on a particular table or tables.
As an example, these is my telephone.js file, which is meant to handle basic CRUD operations for the TELEPHONE table.
const sql = require('mssql');
const { poolPromise } = require('../functions/db');
module.exports = {
async get (cus_id) {
const pool = await poolPromise;
let query;
return new Promise(function(resolve, reject) {
try {
query = ("SELECT TEL_NUMBER, TEL_TYPE, TEL_TYPE_GENERAL FROM TEL WHERE TEL_STATUS = 'A' AND TEL_PRIMARY = 'Y' AND TEL_CUS_ID = #CUS_ID ORDER BY TEL_RECEIVED_SMS_FROM DESC, CASE WHEN (TEL_TYPE_GENERAL = 'mobile') THEN 'Y' ELSE 'N' END, TEL_PRIMARY DESC");
pool.request()
.input('CUS_ID', sql.Int, cus_id)
.query(query, (err, results) => {
if (err) {
reject('Query failed in get(): ' + query);
} else {
if (results.rowsAffected > 0) {
resolve(results.recordsets[0]);
} else {
resolve(null);
}
}
});
} catch (err) {
reject('get() failed: ' + query);
}
});
}
};
This code works fine. I am able to get the records I need through the applicable route file, format the records and send them into a response like so:
const telephone = require('../functions/telephone');
...
telephone.get(cus_id).then(function(tel_recs) {
//do stuff w/ tel_recs
}).catch (function(err) {
console.log(err.message);
res.status(500);
res.send(err.message)
});
I have this working...BUT is this the best way?
I don't know what I don't know so I don't know everything to ask. I guess I am attempting to solicit general advice before spending the next 3 months replicating this code 1,500 times:
Any pitfalls this code is introducing.
Should try {} catch {} be used this way inside of a promise?
Is creating a separate functions file for each table the "best" way of handling a store of CRUD operations?
Related
I need some advice on how to structure this function as at the moment it is not happening in the correct order due to node being asynchronous.
This is the flow I want to achieve; I don't need help with the code itself but with the order to achieve the end results and any suggestions on how to make it efficient
Node routes a GET request to my controller.
Controller reads a .csv file on local system and opens a read stream using fs module
Then use csv-parse module to convert that to an array line by line (many 100,000's of lines)
Start a try/catch block
With the current row from the csv, take a value and try to find it in a MongoDB
If found, take the ID and store the line from the CSV and this id as a foreign ID in a separate database
If not found, create an entry into the DB and take the new ID and then do 6.
Print out to terminal the row number being worked on (ideally at some point I would like to be able to send this value to the page and have it update like a progress bar as the rows are completed)
Here is a small part of the code structure that I am currently using;
const fs = require('fs');
const parse = require('csv-parse');
function addDataOne(req, id) {
const modelOneInstance = new InstanceOne({ ...code });
const resultOne = modelOneInstance.save();
return resultOne;
}
function addDataTwo(req, id) {
const modelTwoInstance = new InstanceTwo({ ...code });
const resultTwo = modelTwoInstance.save();
return resultTwo;
}
exports.add_data = (req, res) => {
const fileSys = 'public/data/';
const parsedData = [];
let i = 0;
fs.createReadStream(`${fileSys}${req.query.file}`)
.pipe(parse({}))
.on('data', (dataRow) => {
let RowObj = {
one: dataRow[0],
two: dataRow[1],
three: dataRow[2],
etc,
etc
};
try {
ModelOne.find(
{ propertyone: RowObj.one, propertytwo: RowObj.two },
'_id, foreign_id'
).exec((err, searchProp) => {
if (err) {
console.log(err);
} else {
if (searchProp.length > 1) {
console.log('too many returned from find function');
}
if (searchProp.length === 1) {
addDataOne(RowObj, searchProp[0]).then((result) => {
searchProp[0].foreign_id.push(result._id);
searchProp[0].save();
});
}
if (searchProp.length === 0) {
let resultAddProp = null;
addDataTwo(RowObj).then((result) => {
resultAddProp = result;
addDataOne(req, resultAddProp._id).then((result) => {
resultAddProp.foreign_id.push(result._id);
resultAddProp.save();
});
});
}
}
});
} catch (error) {
console.log(error);
}
i++;
let iString = i.toString();
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(iString);
})
.on('end', () => {
res.send('added');
});
};
I have tried to make the functions use async/await but it seems to conflict with the fs.openReadStream or csv parse functionality, probably due to my inexperience and lack of correct use of code...
I appreciate that this is a long question about the fundamentals of the code but just some tips/advice/pointers on how to get this going would be appreciated. I had it working when the data was sent one at a time via a post request from postman but can't implement the next stage which is to read from the csv file which contains many records
First of all you can make the following checks into one query:
if (searchProp.length === 1) {
if (searchProp.length === 0) {
Use upsert option in mongodb findOneAndUpdate query to update or upsert.
Secondly don't do this in main thread. Use a queue mechanism it will be much more efficient.
Queue which I personally use is Bull Queue.
https://github.com/OptimalBits/bull#basic-usage
This also provides the functionality you need of showing progress.
Also regarding using Async Await with ReadStream, a lot of example can be found on net such as : https://humanwhocodes.com/snippets/2019/05/nodejs-read-stream-promise/
I'm new to Node and am having problems reading from Oracle.
I have the basic examples all set up and can issue basic queries, and process the results etc..
The problem I'm having is that I need to;
Execute one query (Q1)
For each item in the results of Q1 I need to execute a second query (Q2)
I need to combine the results of Q1 and Q2s into an array to return as a promise
I am struggling to find an example where I can perform #2 - call the same query multiple times for each item returned from Q1, using the same connection which was used for Q1.
My code is below - I first perform a read, then iterate through the results storing connection.execute objects which I then run via the Promise.all line - the result of which I just output as I want to get this working before I code the logic to combine the results of Q1 and Q2.
When I run this via mocha, the results of don't contain any data - I see the column headings but no data.
So what am I missing here?
// placeholder for the connection
let conn;
// return case list array
var caseList = [];
var queryList = [];
return new Promise((resolve, reject) => {
// retrieve connection
oracledb.getConnection({
user: dbconfig.user,
password: dbconfig.password,
connectString: dbconfig.connectString
}) // the connection is returned as a promise
.then(connection => {
console.log('Connected to the DB!');
// assign connection
conn = connection;
// execute statement
return connection.execute(
`select caseid, casereference, startdate from caseheader inner join orgobjectlink on caseheader.ownerorgobjectlinkid = orgobjectlink.orgobjectlinkid where orgobjectlink.username = :username`,
[params.username], {
outFormat: oracledb.OBJECT // set the output format to be object
}
);
})
.then(result => {
// iterate around rows
result.rows.forEach(row => {
var caseObj = {
caseID: row.CASEID,
reference: row.CASEREFERENCE,
dateAssigned: moment(row.STARTDATE).format('YYYY-MM-DD'),
username: params.username,
}
caseList.push(caseObj);
console.log(caseObj.caseID)
queryList.push(conn.execute(`select concernroleid, concernrolename from concernrole inner join caseparticipantrole on concernrole.concernroleid = caseparticipantrole.participantroleid where caseparticipantrole.caseid = :caseID and (caseparticipantrole.typecode = 'PRI' or caseparticipantrole.typecode = 'MEM')`,
[caseObj.caseID], {
outFormat: oracledb.OBJECT
}));
});
// build up queries
return Promise.all(queryList).then(results => {
console.log(results);
Promise.resolve(results);
}, err => {
console.log(err);
});
}).then({
if(conn){
console.log("Closing DB connection");
conn.close();
}
}).catch(err => {
console.log('Error', err);
});
});
Promise.all will not work for you as you want to use a single connection and as mentioned previously a connection will only do one thing at a time anyway. To solve this problem using promises, you'd have to build up and unwind a promise chain. I can show you an example, but it's nasty - probably better to just forget I mentioned it.
A better option would be to go into a simple for loop using async/await. I can show you can example of that too but again, I think this is the wrong move. We call this row by row fetching (a.k.a slow by slow).
It's likely the best solution for you will be to take the results from the first query and build up an array. Then execute the second query using one of these options to process the array. https://oracle.github.io/node-oracledb/doc/api.html#sqlwherein
You'll need to include the caseid column in the select clause and perhaps even order by that column so that post-processing of the result set is simplified in Node.js.
This solution has the potential to greatly improve performance and resource utilization, but that has to be balanced against the amount of data you have, the resources, etc. I could probably show you an example of this too, but it will take a bit longer and I'd want to get some more info from you to ensure we're on the right path.
One problem is the Promise.all().then... function doesn't return anything (and doesn't need the additional resolve()). The way to get this sorted is build small, testable, promise returning functions, and test them individually.
Starting simply, write a mocha test to connect to the database...
function connect() {
return oracledb.getConnection({
user: dbconfig.user,
password: dbconfig.password,
connectString: dbconfig.connectString
});
}
Here's one that can run a command on the db. Test this with a simple query that you know will return some results.
function executeCmd(connection, cmd, params) {
return connection.execute(cmd, params, { outFormat: oracledb.OBJECT });
}
With just these two (and one more) we can outline a simple function that does the job: connect to the database, run a select, process each result asynchronously, then disconnect.
function connectAndQuery(username) {
let connection;
return connect().then(result => {
connection = result;
let cmd = `select caseid, casereference, startdate from caseheader inner join orgobjectlink on caseheader.ownerorgobjectlinkid = orgobjectlink.orgobjectlinkid where orgobjectlink.username = :username`;
return executeCmd(connection, cmd, [username]);
}).then(result => {
let promises = result.rows.map(row => processCaseRow(connection, row, username));
return Promise.all(promises);
}).then(result => {
// result should be an array of caseObj's
return connection.close().then(() => result);
});
}
The last thing to build and test is a promise-returning function which processes a row from the main function above.
I had to take some liberty with this, but I think the objective is -- given a row representing a "case" -- build a case object, including a collection of "concernedRoles" that can be queried with the caseID. (that last bit was my idea, but you can build a separate collection if you like)
// return a promise that resolves to an object with the following properties...
// caseID, reference, dateAssigned, username, concernedRoles
// get concernedRoles by querying the db
function processCaseRow(connection, row, username) {
var caseObj = {
caseID: row.CASEID,
reference: row.CASEREFERENCE,
dateAssigned: moment(row.STARTDATE).format('YYYY-MM-DD'),
username: username
}
let cmd = `select concernroleid, concernrolename from concernrole inner join caseparticipantrole on concernrole.concernroleid = caseparticipantrole.participantroleid where caseparticipantrole.caseid = :caseID and (caseparticipantrole.typecode = 'PRI' or caseparticipantrole.typecode = 'MEM')`;
return executeCmd(connection, cmd, row.CASEID).then(result => {
caseObj.concernedRole = result
return caseObj
})
}
I am adding user validation an data modification page on a node.js application.
In a synchronous universe, in a single function I would:
Lookup the original record in the database
Lookup the user in LDAP to see if they are the owner or admin
Do the logic and write the record.
In an asynchronous universe that won't work. To solve it I've built a series of hand-off functions:
router.post('/writeRecord', jsonParser, function(req, res) {
post = req.post;
var smdb = new AWS.DynamoDB.DocumentClient();
var params = { ... }
smdb.query(params, function(err,data){
if( err == null ) writeRecordStep2(post,data);
}
});
function writeRecord2( ru, post, data ){
var conn = new LDAP();
conn.search(
'ou=groups,o=amazon.com',
{ ... },
function(err,resp){
if( err == null ){
writeRecordStep3( ru, post, data, ldap1 )
}
}
}
function writeRecord3( ru, post, data ){
var conn = new LDAP();
conn.search(
'ou=groups,o=amazon.com',
{ ... },
function(err,resp){
if( err == null ){
writeRecordStep4( ru, post, data, ldap1, ldap2 )
}
}
}
function writeRecordStep4( ru, post, data, ldap1, ldap2 ){
// Do stuff with collected data
}
Additionally, because the LDAP and Dynamo logic are in their own source documents, these functions are scattered tragically around the code.
This strikes me as inefficient, as well as inelegant. I'm eager to find a more natural asynchronous pattern to achieve the same result.
Any promise library should sort your issue out. My preferred choice is bluebird. In summary they help you in performing blocking operations.
If you haven't heard about bluebird then just use it. It converts all function of a module and return promise which is then-able. Simply put, it promisifies all functions.
Here is the mechanism:
Module1.someFunction() \\do your job and finally pass the return object to next call
.then() \\Use that object which is return from the first call, do your job and return the updated value
.then() \\same goes on
.catch() \\do your job when any error occurs.
Hope you understand. Here is an example:
var readFile = Promise.promisify(require("fs").readFile);
readFile("myfile.js",
"utf8").then(function(contents) {
return eval(contents);
}).then(function(result) {
console.log("The result of evaluating
myfile.js", result);
}).catch(SyntaxError, function(e) {
console.log("File had syntax error", e);
//Catch any other error
}).catch(function(e) {
console.log("Error reading file", e);
});
I could not tell from your pseudo-code exactly which async operations depend upon results from with other ones and knowing that is key to the most efficient way to code a series of asynchronous operations. If two operations do not depend upon one another, they can run in parallel which generally gets to an end result faster. I also can't tell exactly what data needs to be passed on to later parts of the async requests (too much pseudo-code and not enough real code to show us what you're really attempting to do).
So, without that level of detail, I'll show you two ways to approach this. The first runs each operation sequentially. Run the first async operation, when it's done, run the next one and accumulates all the results into an object that is passed along to the next link in the chain. This is general purpose since all async operations have access to all the prior results.
This makes use of promises built into the AWS.DynamboDB interface and makes our own promise for conn.search() (though if I knew more about that interface, it may already have a promise interface).
Here's the sequential version:
// promisify the search method
const util = require('util');
LDAP.prototype.searchAsync = util.promisify(LDAP.prototype.search);
// utility function that does a search and adds the result to the object passed in
// returns a promise that resolves to the object
function ldapSearch(data, key) {
var conn = new LDAP();
return conn.searchAsync('ou=groups,o=amazon.com', { ... }).then(results => {
// put our results onto the passed in object
data[key] = results;
// resolve with the original object (so we can collect data here in a promise chain)
return data;
});
}
router.post('/writeRecord', jsonParser, function(req, res) {
let post = req.post;
let smdb = new AWS.DynamoDB.DocumentClient();
let params = { ... }
// The latest AWS interface gets a promise with the .promise() method
smdb.query(params).promise().then(dbresult => {
return ldapSearch({post, dbresult}, "ldap1");
}).then(result => {
// result.dbresult
// result.ldap1
return ldapSearch(result, "ldap2")
}).then(result => {
// result.dbresult
// result.ldap1
// result.ldap2
// doSomething with all the collected data here
}).catch(err => {
console.log(err);
res.status(500).send("Internal Error");
});
});
And, here's a parallel version that runs all three async operations at once and then waits for all three of the to be done and then has all the results at once:
// if the three async operations you show can be done in parallel
// first promisify things
const util = require('util');
LDAP.prototype.searchAsync = util.promisify(LDAP.prototype.search);
function ldapSearch(params) {
var conn = new LDAP();
return conn.searchAsync('ou=groups,o=amazon.com', { ... });
}
router.post('/writeRecord', jsonParser, function(req, res) {
let post = req.post;
let smdb = new AWS.DynamoDB.DocumentClient();
let params = { ... }
Promise.all([
ldapSearch(...),
ldapSearch(...),
smdb.query(params).promise()
]).then(([ldap1Result, ldap2Result, queryResult]) => {
// process ldap1Result, ldap2Result and queryResult here
}).catch(err => {
console.log(err);
res.status(500).send("Internal Error");
});
});
Keep in mind that due to the pseudo-code nature of the code in your question, this is also pseudo-code where implementation details (exactly what parameters you're searching for, what response you're sending, etc...) have to be filled in. This should be illustrative of promise chaining to serialize operations and the use of Promise.all() for parallelizing operations and promisifying a method that didn't have promises built in.
My removeObjects function has me stummped.The function is suppose to syncronoulsy get a list of objects in an S3 bucket then asyncronously removes the objects. Repeat if the list was truncated, until the there are no more objects to remove. (AWS doesn't provide the total count of objects in the bucket and listObjects pages the results.)
What am I doing wrong / why doesn't my function work? The solution should exploit single thread and async nature of JS. For the bounty I am hoping for an answer specific to the module. The git repo is public if you want to see the entire module.
export function removeObjects(params: IS3NukeRequest): Promise<S3.Types.DeleteObjectsOutput> {
const requests: Array<Promise<S3.Types.DeleteObjectsOutput>> = [];
let isMore;
do {
listObjectsSync(params)
.then((objectList: S3.Types.ListObjectsV2Output) => {
isMore = objectList.ContinuationToken = objectList.IsTruncated ? objectList.NextContinuationToken : null;
requests.push(params.Client.deleteObjects(listObjectsV2Output2deleteObjectsRequest(objectList)).promise());
})
.catch((err: Error) => { Promise.reject(err); });
} while (isMore);
return Promise.all(requests);
}
export async function listObjectsSync(params: IS3NukeRequest): Promise<S3.Types.ListObjectsV2Output> {
try {
return await params.Client.listObjectsV2(s3nukeRequest2listObjectsRequest(params)).promise();
} catch (err) {
return Promise.reject(err);
}
}
Thanks.
The thing is that listObjectsSync function returns a Promise, so you need to treat it as an async function and can't just use a loop with it. What you need to do is to create a chain of promises while your isMore is true, I've done it using a recursive approach (I'm not pro in TS, so please check the code before using it). I also haven't tried the code live, but logically it should work :)
const requests: Array<Promise<S3.Types.DeleteObjectsOutput>> = [];
function recursive(recursiveParams) {
return listObjectsSync(recursiveParams).then((objectList: S3.Types.ListObjectsV2Output) => {
let isMore = objectList.ContinuationToken = objectList.IsTruncated ? objectList.NextContinuationToken : null;
requests.push(params.Client.deleteObjects(listObjectsV2Output2deleteObjectsRequest(objectList)).promise());
if (isMore) {
//do we need to change params here?
return recursive(recursiveParams)
}
//this is not necessary, just to indicate that we get out of the loop
return true;
});
}
return recursive(params).then(() => {
//we will have all requests here
return Promise.all(requests);
});
I have an Express app which requires very low response rate ~<200ms. Right now we can only get this number but that's a separate topic.
We're planning to fetch a piece of data from the database, if found in Redis return the data if not then fire the request and save that to redis so the next requests can get it from Redis.
I'm running some testing and was wondering if there's a way to reduce the amount of database fetching requests?
For example, currently our application has 300req/s per box. We have six boxes running on AWS. If for the first time that piece of data is not available in Redis, there might be around ~500 requests trying to fetch the data from DB and cache that in Redis. We're trying to reduce that number down. Not sure if there's a way in Node.js or Redis to handle that.
Here's the code that I'm testing.
client.getAsync('key').then(function (data) {
if(data) {
console.log(data); // Return this data if found
res.send(data);
} else {
// I'm trying to reduce the number of calls for concurrent requests in this block.
console.log('not found');
var dataFromDb = // fetch data from DB
client.set('key', dataFromDb); // Fire and forget
res.send('not found'); // Return not found right away
}
});
And I test the call by using ab
ab -n 20 -c 10 http://localhost:8081/redis
This is the results I got
not found
not found
not found
not found
not found
not found
something
not found
something
something
something
something
something
something
something
something
something
something
In this example, there's 7 requests trying to fetch database with the same data and save to Redis.
My question is, is there anyway I can reduce the number of requests down? Because fetching DB is quite slow as of now ~900ms (We're trying to optimize that)
Yes there is. I did same thing. I will describe only logic here. Method to fetchCache should return a promise. Also you keep array of { cacheKey, promise }. Each time you send a request - you add key to this array. When next time you need to fetch cache - you check array first and if key there grabbing this promise. Else calling fetchCache.
Here is my code. It works, but probably hard to read. Should give you a basic understanding.
class DictTranslatableRepo {
constructor(model) {
var self = this;
self.title = model + "s Repo";
self.model = models[model];
self.running = {};
self.curItems = {};
}
*start() {
var self = this;
var curItems = yield self.model.findAll();
_.forEach(curItems, function(row) {
self.curItems[row.key] = row.value;
});
};
*map(from) {
var self = this;
if (from == "") return "";
if (!_.isUndefined(self.curItems[from])) return self.curItems[from];
if (_.isUndefined(self.running[from])) {
self.running[from] = [];
return new Promise(function(resolve, reject) {
self.running[from].push(resolve);
self.job(from, function(err, to) { // Main job
var callbackArr = self.running[from];
delete self.running[from];
_.forEach(callbackArr, function(callback) {
callback(to);
});
});
});
} else {
return new Promise(function(resolve, reject) {
self.running[from].push(resolve);
});
}
};
job(from, callback) {
var self = this;
var to = "as shown";
co(function*() {
try {
to = yield translator.translate(from);
yield self.model.add({key: from, value: to});
self.curItems[from] = to;
callback(null, to);
} catch (err) {
callback(err);
//logger.error("Cant translate entity: " + from);
}
}).catch(function(err) {
// Unhandled Error
callback(new Error(err));
});
};
}
My map method is your fetchCache method.