Javascript promise to iterate/include dynamic number of Arguments - node.js

I'm using the mssql npm module (with Tedious driver) to read/write to Azure Sql database from my node Server. https://www.npmjs.com/package/mssql
All the examples I've found provide an hardcoded example of the query whether to read or write records, like this:
var insertRecordIntoTable = function (callback) {
sql.connect(dbConfig).then(pool => {
return pool.request()
.input('ID', sql.Int, 210)
.input('Name', sql.NVarChar, "John Doe")
.input('EmailAddress', sql.NVarChar, "test#test.com")
.query("INSERT INTO Accounts (ID, Name, EmailAddress) VALUES (#ID, #Name, #EmailAddress)")
}).then(result => {
console.dir(result)
callback(result);
}).catch(err => {
// ... error checks
console.log("Error occured: " + err);
callback(err);
});
}
Obviously, I'd like to write one standard method to write records to any table in the database.
Now I can fetch structure of each table and use that to find how what datatype each field should be from the key in jsonRecord and write something like this:
var insertRecordIntoTable = function (jsonRecord, tableName, callback) {
let arrKeys = jsonRecord.allKeys();
let columnNames = getCommaSeparatedColumnNames(arrKeys);
let valuePlaceholders = getValuePlaceholdersForSql(arrKeys);
sql.connect(dbConfig).then(pool => {
return pool.request()
// how do I write something like this so that dynamic number of fields and values get populated in the query inside this promise.
// I'm open to methods without promise as well.
for(let x=0; x < arrKeys.length; x++){
let key = arrKeys[x];
// .input('ID', sql.Int, 210)
.input(key, getTypeForKey(key, tableName), jsonRecord[ key ] )
}
.query("INSERT INTO " + tableName + " (" + columnNames + ") VALUES (" + valuePlaceholders + ")")
}).then(result => {
console.dir(result)
callback(result);
}).catch(err => {
// ... error checks
console.log("Error occured: " + err);
callback(err);
});
}
function getTypeForKey(key){. // looks up table schema and returns keyType }
function getCommaSeparatedColumnNames(arrKeys){ return arrKeys.join(", "); }
function getValuePlaceholdersForSql(arrKeys){ // write code to append '#' before every key and then join using comma's and return that string }
I'm sure node.js writing to SQL is a fairly common functionality and there may be better ways to achieve what I'm trying to do here. Please feel free to go a different route.
P.S. - Although I should say that I prefer mssql over Tedious package. It just seems better in functionality after going through the documentation in the last several hours.

If you want to interact with your database without creating all the queries by yourself, you can use a query builder like knex to manage the data as objects:
knex('Accounts').insert({ID: 210, Name: "John Doe", EmailAddress: "test#test.com"})
Would be similar to:
insert into `Accounts` (`EmailAddress`, `ID`, `Name`) values ('test#test.com', 210, 'John Doe')
Also I see you are checking types. If you need validation, maybe a complete ORM (I like Objection.js) would be a good choice.

Related

Dynamo DB Query Filter Node.js

Running a Node.js serverless backend through AWS.
Main objective: to filter and list all LOCAL jobs (table items) that included the available services and zip codes provided to the filter.
Im passing in multiple zip codes, and multiple available services.
data.radius would be an array of zip codes = to something like this:[ '93901', '93902', '93905', '93906', '93907', '93912', '93933', '93942', '93944', '93950', '95377', '95378', '95385', '95387', '95391' ]
data.availableServices would also be an array = to something like this ['Snow removal', 'Ice Removal', 'Salting', 'Same Day Response']
I am trying to make an API call that returns only items that have a matching zipCode from the array of zip codes provided by data.radius, and the packageSelected has a match of the array data.availableServices provided.
API CALL
import * as dynamoDbLib from "./libs/dynamodb-lib";
import { success, failure } from "./libs/response-lib";
export async function main(event, context) {
const data = JSON.parse(event.body);
const params = {
TableName: "jobs",
FilterExpression: "zipCode = :radius, packageSelected = :availableServices",
ExpressionAttributeValues: {
":radius": data.radius,
":availableServices": data.availableServices
}
};
try {
const result = await dynamoDbLib.call("query", params);
// Return the matching list of items in response body
return success(result.Items);
} catch (e) {
return failure({ status: false });
}
Do I need to map the array of zip codes and available services first for this to work?
Should I be using comparison operators?
https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LegacyConditionalParameters.QueryFilter.html
Is a sort key value or partition key required to query and filter? (the table has a sort key and partition key but i would like to avoid using them in this call)
Im not 100% sure on how to go about this so if anyone could point me in the right direction that would be wonderful and greatly appreciated!!
I'm not sure what your dynamodb-lib refers to but here's an example of how you can scan for attribute1 in a given set of values and attribute2 in a different set of values. This uses the standard AWS JavaScript SDK, and specifically the high-level document client.
Note that you cannot use an equality (==) test here, you have to use an inclusion (IN) test. And you cannot use query, but must use scan.
const AWS = require('aws-sdk');
let dc = new AWS.DynamoDB.DocumentClient({'region': 'us-east-1'});
const data = {
radius: [ '93901', '93902', '93905', '93906', '93907', '93912', '93933', '93942', '93944', '93950', '95377', '95378', '95385', '95387', '95391' ],
availableServices: ['Snow removal', 'Ice Removal', 'Salting', 'Same Day Response'],
};
// These hold ExpressionAttributeValues
const zipcodes = {};
const services = {};
data.radius.forEach((zipcode, i) => {
zipcodes[`:zipcode${i}`] = zipcode;
})
data.availableServices.forEach((service, i) => {
services[`:services${i}`] = service;
})
// These hold FilterExpression attribute aliases
const zipcodex = Object.keys(zipcodes).toString();
const servicex = Object.keys(services).toString();
const params = {
TableName: "jobs",
FilterExpression: `zipCode IN (${zipcodex}) AND packageSelected IN (${servicex})`,
ExpressionAttributeValues : {...zipcodes, ...services},
};
dc.scan(params, (err, data) => {
if (err) {
console.log('Error', err);
} else {
for (const item of data.Items) {
console.log('item:', item);
}
}
});

Orient multiple traverse

I have this graph:
And I'm trying to write a query with the "SQL Syntax" of OrientDB v3.0 that start from a Client and follow the read path (the X means: and don't have Have relation to the Client).
I can get the Segments, but I don't find how to walk to the Contact.
The docs have many examples but only walking one path.
I have tried without success these queries:
SELECT FROM (TRAVERSE out("Access").out("Contain") FROM (SELECT #rid FROM Client where myId = 30543) MAXDEPTH 1)
SELECT FROM (
TRAVERSE out("Contain") FROM
(TRAVERSE out("Access") FROM (SELECT #rid FROM Client where myId = 30543) MAXDEPTH 1)
MAXDEPTH 1
)
SELECT out('Access').out("Contain") FROM Client WHERE myId = 30543
Do you have any info to accomplish this traverse?
I'm using the Node.js API:
const pool = await orient.getPool();
const session = await pool.acquire();
logger.info('Running query...');
session.command(`SELECT out('Access').out("Contain") FROM Client WHERE myId = 30543`)
.on('data', (data) => {
if (data.out_Contain && data.out_Contain.delegate) {
logger.info('Segment %s contains %o Contact', data['#rid'].toString(), data.out_Contain.delegate.size);
} else if (data['#rid']) {
logger.info('Segment %s contains %o Contact', data['#rid'].toString(), data);
} else {
logger.info('Data %o', data);
}
})
.on('error', (err) => {
logger.error(err);
})
.on('end', () => {
console.timeEnd('query');
logger.info('End of the stream');
process.emit('SIGINT');
});
logger.debug('Registering SIGINT');
process.once('SIGINT', async () => {
await session.close();
await pool.close();
await orient.stop();
});
Please try using this code:
"select out_{edgeclass}.in from (select expand(out_{edgeclass}.in) from {Vetex} where {condition})"
The match is better for this kind of tasks.
With SELECT:
In the first version, the query is more readable, but it does not use indexes, so it is less optimal in terms of execution time. The second and third use indexes if they exist, (on Person.name or City.name, both in the sub-query), but they're harder to read. Which index they use depends only on the way you write the query.
But match:
the query executor optimizes the query for you, choosing indexes where they exist. Moreover, the query becomes more readable, especially in complex cases, such as multiple nested SELECT queries.
Here the right query:
SELECT EXPAND(contatti)
FROM (
match { class: Client, as: user, where : ( myId = 30543)}
.out('Access')
.out('Contain'){ class:Contact, as:contatti, where: (gender= 'M')},
NOT {as:user} -Have-> {as:contatti}
RETURN DISTINCT contatti LIMIT 1000
)

How to pass data between nested callbacks?

I'm a newbie at Node JS, and I'm using NodeJS (v. 8.7.0), sqlite3 and Express.
I have two tables in a SQLite database:
releases (id, title, image)
links (id, url)
Each "release" has one or more "links" associated with it.
I can get all the releases using:
dbh.all("SELECT * FROM releases ORDER BY id DESC", (err, rows) => { ... })
And I can get all the links for a given release using:
dbh.all("SELECT * FROM links WHERE id = ?", (err, rows) => { ... })
But I can't figure out how to add a "links" property to each "release", which contains their corresponding links, so that I can feed the resulting object to Mustache, and generate a HTML page.
I know that storing hierarchical data inside of a relational database is not the best idea, and I could easily do this using PHP, but I really want to learn how to use NodeJS.
This is what I've come up so far:
var sqlite3 = require("sqlite3")
function main() {
db = new sqlite3.Database("releases.sqlite3")
all = []
db.each(
"SELECT * FROM releases ORDER BY id DESC",
(err, release) => {
release.links = []
db.all("SELECT url FROM links WHERE id = ?", [release.id], (err, links) => {
links = links.map((e) => { return e.url })
release.links = links
// line above: tried
// links.forEach((e) => { release.links.push(e.url) })
// too, but that didn't work either.
})
all.push(release)
},
(complete) => { console.log(all) }
)
}
main()
Though, when I run it, it inevitably shows:
links: []
Every time. How can I fix this?
Thank you in advance.
Edit 1:
This SQL snippet generates the database, and populates it with some data.
CREATE TABLE `links` ( `id` TEXT, `url` TEXT );
CREATE TABLE `releases` ( `id` TEXT, `title` TEXT, `image` TEXT );
INSERT INTO links VALUES
('rel-001', 'https://example.com/mirror1'),
('rel-001', 'https://example.com/mirror2');
INSERT INTO releases VALUES
('rel-001', 'Release 001', 'https://example.com/image.jpg');
The goal is to have something like this:
{
releases:[
{
id:'rel-001',
title:'Release 001',
image:'https://example.com/image.jpg',
links:[
'https://example.com/mirror1',
'https://example.com/mirror2'
]
}
]
}
try to see if both queries are being executed by adding console.log in the callbacks, moreover you should push the links only within the second callback since before the callback is fired the value is not existing, thus you are trying to push an empty value, also you don't need to initialize release.links = [], all will be only filled after all queries are executed, so therefore we need to execute console.log(all); in the last child callback:
function main() {
all = []
var parentComplete = false;
db.each("SELECT * FROM releases ORDER BY id DESC", (err, release) => {
db.all("SELECT url FROM links WHERE id = ?", [release.id], (err, links) => {
release.links = links.map(e => e.url);
all.push(release);
if (parentComplete){
console.log(all);
}
})
},
(complete) => {
parentComplete = true;
}
)
}
main();
p.s. in order to get the result you want you will need to initialize all as an object all = {releases:[]}
function main() {
all = {releases:[]};
var parentComplete = false;
db.each("SELECT * FROM releases ORDER BY id DESC", (err, release) => {
db.all("SELECT url FROM links WHERE id = ?", [release.id], (err, links) => {
release.links = links.map(e => e.url);
all.releases.push(release);
if (parentComplete){
console.log(all);
}
})
},
(complete) => {
parentComplete = true;
}
)
}
main();

NodeJS and pg-promise, insert dynamically from JSON-object

I'm running NodeJS and pg-promise, and are trying to accomplish somethings like:
db.none('INSERT INTO my-table (JSON-object-keys) VALUES ($1)', [JSON-object-values])
.catch(function(err) {
console.log('Error on insert into my-table: ' + err);
});
I have JSON-objects which can look like:
{"column1":"value1", "column2":"value2", "column3":"value3"}
{"column2":"value2", "column3":"value3"}
{"column1":"value1", "column3":"value3"}
I would like to have the INSERTS automatically generated corresponding to what the JSON-object contains.
Is that possible in an elegant way?
Explained a bit more, in the 3 examples of JSON the following should be generated:
db.none('INSERT INTO my-table (column1, column2, column3) VALUES ($1, $2, $3)', [value1, value2, value3])
.catch(function(err) {
console.log('Error on insert into my-table: ' + err);
});
db.none('INSERT INTO my-table (column2, column3) VALUES ($1, $2)', [value2, value3])
.catch(function(err) {
console.log('Error on insert into my-table: ' + err);
});
db.none('INSERT INTO my-table (column1, column3) VALUES ($1, $2)', [value1, value3])
.catch(function(err) {
console.log('Error on insert into my-table: ' + err);
});
Your pgp object + the input object with all the properties:
var pgp = require('pg-promise')({
capSQL: true // capitalize all generated SQL
});
var inputObj = {
/* all your optional properties */
};
Define the raw-text type, using Custom Type Formatting:
var rawText = text => ({_rawType: true, toPostgres: () => text});
Create a generic default column, according to class Column:
var defCol = name => ({name, def: rawText('DEFAULT')});
// which is the same as:
var defCol = name => new pgp.helpers.Column({name, def: rawText('DEFAULT')});
Generate the list of default-able columns:
var cols = Object.keys(inputObj).map(defCol);
Create a ColumnSet with those columns:
var cs = new pgp.helpers.ColumnSet(cols, {table: 'my-table'});
When it is time to generate an insert query, you can do:
var insert = pgp.helpers.insert(inputObj, cs);
Recommended Approach
If you know the columns in advance, then you should just do the following:
var cs = new pgp.helpers.ColumnSet(
[defCol('column1'), defCol('column2'), defCol('column3')],
{table: 'my-table'});
A static cs object will always provide a much better performance.
This approach is also safer, because you do not need to verify whether there is at least one property in the object, 'cos if there isn't, you'll get an error saying that it is impossible to generate an insert when there are no columns.
And this approach also works with multi-row inserts, which is very important. See also: Multi-row insert with pg-promise.

Nested transactions with pg-promise

I am using NodeJS, PostgreSQL and the amazing pg-promise library. In my case, I want to execute three main queries:
Insert one tweet in the table 'tweets'.
In case there is hashtags in the tweet, insert them into another table 'hashtags'
Them link both tweet and hashtag in a third table 'hashtagmap' (many to many relational table)
Here is a sample of the request's body (JSON):
{
"id":"12344444",
"created_at":"1999-01-08 04:05:06 -8:00",
"userid":"#postman",
"tweet":"This is the first test from postman!",
"coordinates":"",
"favorite_count":"0",
"retweet_count":"2",
"hashtags":{
"0":{
"name":"test",
"relevancetraffic":"f",
"relevancedisaster":"f"
},
"1":{
"name":"postman",
"relevancetraffic":"f",
"relevancedisaster":"f"
},
"2":{
"name":"bestApp",
"relevancetraffic":"f",
"relevancedisaster":"f"
}
}
All the fields above should be included in the table "tweets" besides hashtags, that in turn should be included in the table "hashtags".
Here is the code I am using based on Nested transactions from pg-promise docs inside a NodeJS module. I guess I need nested transactions because I need to know both tweet_id and hashtag_id in order to link them in the hashtagmap table.
// Columns
var tweetCols = ['id','created_at','userid','tweet','coordinates','favorite_count','retweet_count'];
var hashtagCols = ['name','relevancetraffic','relevancedisaster'];
//pgp Column Sets
var cs_tweets = new pgp.helpers.ColumnSet(tweetCols, {table: 'tweets'});
var cs_hashtags = new pgp.helpers.ColumnSet(hashtagCols, {table:'hashtags'});
return{
// Transactions
add: body =>
rep.tx(t => {
return t.one(pgp.helpers.insert(body,cs_tweets)+" ON CONFLICT(id) DO UPDATE SET coordinates = "+body.coordinates+" RETURNING id")
.then(tweet => {
var queries = [];
for(var i = 0; i < body.hashtags.length; i++){
queries.push(
t.tx(t1 => {
return t1.one(pgp.helpers.insert(body.hashtags[i],cs_hashtags) + "ON CONFLICT(name) DO UPDATE SET fool ='f' RETURNING id")
.then(hash =>{
t1.tx(t2 =>{
return t2.none("INSERT INTO hashtagmap(tweetid,hashtagid) VALUES("+tweet.id+","+hash.id+") ON CONFLICT DO NOTHING");
});
});
}));
}
return t.batch(queries);
});
})
}
The problem is with this code I am being able to successfully insert the tweet but nothing happens then. I cannot insert the hashtags nor link the hashtag to the tweets.
Sorry but I am new to coding so I guess I didn't understood how to properly return from the transaction and how to perform this simple task. Hope you can help me.
Thank you in advance.
Jean
Improving on Jean Phelippe's own answer:
// Columns
var tweetCols = ['id', 'created_at', 'userid', 'tweet', 'coordinates', 'favorite_count', 'retweet_count'];
var hashtagCols = ['name', 'relevancetraffic', 'relevancedisaster'];
//pgp Column Sets
var cs_tweets = new pgp.helpers.ColumnSet(tweetCols, {table: 'tweets'});
var cs_hashtags = new pgp.helpers.ColumnSet(hashtagCols, {table: 'hashtags'});
return {
/* Tweets */
// Add a new tweet and update the corresponding hash tags
add: body =>
db.tx(t => {
return t.one(pgp.helpers.insert(body, cs_tweets) + ' ON CONFLICT(id) DO UPDATE SET coordinates = ' + body.coordinates + ' RETURNING id')
.then(tweet => {
var queries = Object.keys(body.hashtags).map((_, idx) => {
return t.one(pgp.helpers.insert(body.hashtags[i], cs_hashtags) + 'ON CONFLICT(name) DO UPDATE SET fool = $1 RETURNING id', 'f')
.then(hash => {
return t.none('INSERT INTO hashtagmap(tweetid, hashtagid) VALUES($1, $2) ON CONFLICT DO NOTHING', [+tweet.id, +hash.id]);
});
});
return t.batch(queries);
});
})
.then(data => {
// transaction was committed;
// data = [null, null,...] as per t.none('INSERT INTO hashtagmap...
})
.catch(error => {
// transaction rolled back
})
},
NOTES:
As per my notes earlier, you must chain all queries, or else you will end up with loose promises
Stay away from nested transactions, unless you understand exactly how they work in PostgreSQL (read this, and specifically the Limitations section).
Avoid manual query formatting, it is not safe, always rely on the library's query formatting.
Unless you are passing the result of transaction somewhere else, you should at least provide the .catch handler.
P.S. For the syntax like +tweet.id, it is the same as parseInt(tweet.id), just shorter, in case those are strings ;)
For those who will face similar problem, I will post the answer.
Firstly, my mistakes:
In the for loop : body.hashtag.length doesn't exist because I am dealing with an object (very basic mistake here). Changed to Object.keys(body.hashtags).length
Why using so many transactions? Following the answer by vitaly-t in: Interdependent Transactions with pg-promise I removed the extra transactions. It's not yet clear for me how you can open one transaction and use the result of one query into another in the same transaction.
Here is the final code:
// Columns
var tweetCols = ['id','created_at','userid','tweet','coordinates','favorite_count','retweet_count'];
var hashtagCols = ['name','relevancetraffic','relevancedisaster'];
//pgp Column Sets
var cs_tweets = new pgp.helpers.ColumnSet(tweetCols, {table: 'tweets'});
var cs_hashtags = new pgp.helpers.ColumnSet(hashtagCols, {table:'hashtags'});
return {
/* Tweets */
// Add a new tweet and update the corresponding hashtags
add: body =>
rep.tx(t => {
return t.one(pgp.helpers.insert(body,cs_tweets)+" ON CONFLICT(id) DO UPDATE SET coordinates = "+body.coordinates+" RETURNING id")
.then(tweet => {
var queries = [];
for(var i = 0; i < Object.keys(body.hashtags).length; i++){
queries.push(
t.one(pgp.helpers.insert(body.hashtags[i],cs_hashtags) + "ON CONFLICT(name) DO UPDATE SET fool ='f' RETURNING id")
.then(hash =>{
t.none("INSERT INTO hashtagmap(tweetid,hashtagid) VALUES("+tweet.id+","+hash.id+") ON CONFLICT DO NOTHING");
})
);
}
return t.batch(queries);
});
}),

Resources