Would like to do multiple updates via my API using mssql transactions.
Example:
Shipping Table
Listing Table
User_Notes Table
Customer_Login Table
Push_Notification Table
Which is the right way of doing it?
I was thinking at first of doing it with raw queries.
BEGIN TRANSACTION
CREATE IN SHIPPING
UPDATE IN LISTING
CREATE IN USER_NOTES
UPDATE IN CUSTOMER_LOGIN
CREATE IN PUSH_NOTIFICATION
COMMIT
But want to avoid writing a big raw query like this.
Also can I use mssql Transactions and Queries with (request.query).
const transaction = new sql.Transaction(/* [pool] */)
transaction.begin(err => {
// ... error checks
const request = new sql.Request(transaction)
request.query('create in shipping table', (err, result) => {
// ... error checks
transaction.commit(err => {
// ... error checks
console.log("Transaction committed.")
})
})
request.query('Update in Listing Table', (err, result) => {
// ... error checks
transaction.commit(err => {
// ... error checks
console.log("Transaction committed.")
})
})
and so on...
.
.
.
})
Related
I have a isolated sync server that pulls a tab limited text file from a external ftp server and updates(saves) to mongodb after processing.
My code looks like this
//this function pulls file from external ftp server
async function upsteamFile() {
try {
let pythonProcess = spawn('python3', [configVar.ftpInbound, '/outbound/Items.txt', configVar.dataFiles.items], {encoding: 'utf8'});
logger.info('FTP SERVER LOGS...' + '\n' + pythonProcess.stdout);
await readItemFile();
logger.info('The process of file is done');
process.exit();
} catch (upstreamError) {
logger.error(upstreamError);
process.exit();
}
}
//this function connects to db and calls processing function for each row in the text file.
async function readItemFile(){
try{
logger.info('Reading Items File');
let dataArray = fs.readFileSync(configVar.dataFiles.items, 'utf8').toString().split('\n');
logger.info('No of Rows Read', dataArray.length);
await dbConnect.connectToDB(configVar.db);
logger.info('Connected to Database', configVar.db);
while (dataArray.length) {
await Promise.all( dataArray.splice(0, 5000).map(async (f) => {
splitValues = f.split('|');
await processItemsFile(splitValues)
})
)
logger.info("Current batch finished processing")
}
logger.info("ALL batch finished processing")
}
catch(PromiseError){
logger.error(PromiseError)
}
}
async function processItemsFile(splitValues) {
try {
// Processing of the file is done here and I am using 'save' in moongoose to write to db
// data is cleaned and assigned to respective fields
if(!exists){
let processedValues = new Products(assignedValues);
let productDetails = await processedValues.save();
}
return;
}
catch (error) {
throw error
}
}
upstream()
So this takes about 3 hours to process 100,000 thousand rows and update it in the database.
Is there any way that I can speed this up. I am very much limited from the hardware. I am using a ec2 instance based linux server with 2 core and 4 gb ram.
Should I use worker threads like microjob to run multi-threads . if yes , then how would I go about doing it
Or is this the maximum performance?
Note : I cant do bulk update in mongodb as there is mongoose pre hooks are getting triggered on save
You can always try a bulk update with the use of updateOne method.
I would consider also using the readFileStream instead of readFileSync.
With the event-driven architecture you could push, let's say every 100k updates into array chunks and bulk update on them simultaneously.
You can trigger a pre updateOne() (instead of save()) hook during this operation.
I have solved a similar problem (updating 100k CSV rows) with the following solution:
Create a readFileStream (thanks to that, your application won't consume much heap memory in case of the huge files)
I'm using CSV-parser npm library to deconstruct a CSV file into separate rows of data:
let updates = [];
fs.createReadStream('/filePath').pipe(csv())
.on('data', row => {
// ...do anything with the data
updates.push({
updateOne: {
filter: { /* here put the query */ },
update: [ /* any data you want to update */ ],
upsert: true /* in my case I want to create record if it does not exist */
}
})
})
.on('end', async () => {
await MyCollection.bulkWrite(data)
.catch(err => {
logger.error(err);
})
updates = []; // I just clean up the huge array
})
I have one big Casandra collection (1-million docs), and I want to query the whole user table data its around 1 million records. When I run following query its only return around 10K records.
Could you please let me know what is the effective method to query whole documents from Casandra collection
I am using https://www.npmjs.com/package/cassandra-driver npm as casandra driver
Tried
const query = 'select * from users';
db.users.executeQuery(query)
.then(function (result2) {
})
.catch (function (error) {
reject(error);
});
Why you cannot retrieve all the data at once is because there is a limit on the number of item you can read at once which is understandable.
Looking at the documentation you sould use of the stream or eachRow methods. Which allow you to get treat the entries of the collection in multiple iterations.
client.stream(query, parameters, options)
.on('readable', function () {
// readable is emitted as soon a row is received and parsed
let row;
while (row = this.read()) {
// process row
}
})
.on('end', function () {
// emitted when all rows have been retrieved and read
});
Or
client.eachRow(query, parameters, { prepare: true, autoPage : true }, function(n, row) {
// Invoked per each row in all the pages
}, callback);
I'm new to Node and am having problems reading from Oracle.
I have the basic examples all set up and can issue basic queries, and process the results etc..
The problem I'm having is that I need to;
Execute one query (Q1)
For each item in the results of Q1 I need to execute a second query (Q2)
I need to combine the results of Q1 and Q2s into an array to return as a promise
I am struggling to find an example where I can perform #2 - call the same query multiple times for each item returned from Q1, using the same connection which was used for Q1.
My code is below - I first perform a read, then iterate through the results storing connection.execute objects which I then run via the Promise.all line - the result of which I just output as I want to get this working before I code the logic to combine the results of Q1 and Q2.
When I run this via mocha, the results of don't contain any data - I see the column headings but no data.
So what am I missing here?
// placeholder for the connection
let conn;
// return case list array
var caseList = [];
var queryList = [];
return new Promise((resolve, reject) => {
// retrieve connection
oracledb.getConnection({
user: dbconfig.user,
password: dbconfig.password,
connectString: dbconfig.connectString
}) // the connection is returned as a promise
.then(connection => {
console.log('Connected to the DB!');
// assign connection
conn = connection;
// execute statement
return connection.execute(
`select caseid, casereference, startdate from caseheader inner join orgobjectlink on caseheader.ownerorgobjectlinkid = orgobjectlink.orgobjectlinkid where orgobjectlink.username = :username`,
[params.username], {
outFormat: oracledb.OBJECT // set the output format to be object
}
);
})
.then(result => {
// iterate around rows
result.rows.forEach(row => {
var caseObj = {
caseID: row.CASEID,
reference: row.CASEREFERENCE,
dateAssigned: moment(row.STARTDATE).format('YYYY-MM-DD'),
username: params.username,
}
caseList.push(caseObj);
console.log(caseObj.caseID)
queryList.push(conn.execute(`select concernroleid, concernrolename from concernrole inner join caseparticipantrole on concernrole.concernroleid = caseparticipantrole.participantroleid where caseparticipantrole.caseid = :caseID and (caseparticipantrole.typecode = 'PRI' or caseparticipantrole.typecode = 'MEM')`,
[caseObj.caseID], {
outFormat: oracledb.OBJECT
}));
});
// build up queries
return Promise.all(queryList).then(results => {
console.log(results);
Promise.resolve(results);
}, err => {
console.log(err);
});
}).then({
if(conn){
console.log("Closing DB connection");
conn.close();
}
}).catch(err => {
console.log('Error', err);
});
});
Promise.all will not work for you as you want to use a single connection and as mentioned previously a connection will only do one thing at a time anyway. To solve this problem using promises, you'd have to build up and unwind a promise chain. I can show you an example, but it's nasty - probably better to just forget I mentioned it.
A better option would be to go into a simple for loop using async/await. I can show you can example of that too but again, I think this is the wrong move. We call this row by row fetching (a.k.a slow by slow).
It's likely the best solution for you will be to take the results from the first query and build up an array. Then execute the second query using one of these options to process the array. https://oracle.github.io/node-oracledb/doc/api.html#sqlwherein
You'll need to include the caseid column in the select clause and perhaps even order by that column so that post-processing of the result set is simplified in Node.js.
This solution has the potential to greatly improve performance and resource utilization, but that has to be balanced against the amount of data you have, the resources, etc. I could probably show you an example of this too, but it will take a bit longer and I'd want to get some more info from you to ensure we're on the right path.
One problem is the Promise.all().then... function doesn't return anything (and doesn't need the additional resolve()). The way to get this sorted is build small, testable, promise returning functions, and test them individually.
Starting simply, write a mocha test to connect to the database...
function connect() {
return oracledb.getConnection({
user: dbconfig.user,
password: dbconfig.password,
connectString: dbconfig.connectString
});
}
Here's one that can run a command on the db. Test this with a simple query that you know will return some results.
function executeCmd(connection, cmd, params) {
return connection.execute(cmd, params, { outFormat: oracledb.OBJECT });
}
With just these two (and one more) we can outline a simple function that does the job: connect to the database, run a select, process each result asynchronously, then disconnect.
function connectAndQuery(username) {
let connection;
return connect().then(result => {
connection = result;
let cmd = `select caseid, casereference, startdate from caseheader inner join orgobjectlink on caseheader.ownerorgobjectlinkid = orgobjectlink.orgobjectlinkid where orgobjectlink.username = :username`;
return executeCmd(connection, cmd, [username]);
}).then(result => {
let promises = result.rows.map(row => processCaseRow(connection, row, username));
return Promise.all(promises);
}).then(result => {
// result should be an array of caseObj's
return connection.close().then(() => result);
});
}
The last thing to build and test is a promise-returning function which processes a row from the main function above.
I had to take some liberty with this, but I think the objective is -- given a row representing a "case" -- build a case object, including a collection of "concernedRoles" that can be queried with the caseID. (that last bit was my idea, but you can build a separate collection if you like)
// return a promise that resolves to an object with the following properties...
// caseID, reference, dateAssigned, username, concernedRoles
// get concernedRoles by querying the db
function processCaseRow(connection, row, username) {
var caseObj = {
caseID: row.CASEID,
reference: row.CASEREFERENCE,
dateAssigned: moment(row.STARTDATE).format('YYYY-MM-DD'),
username: username
}
let cmd = `select concernroleid, concernrolename from concernrole inner join caseparticipantrole on concernrole.concernroleid = caseparticipantrole.participantroleid where caseparticipantrole.caseid = :caseID and (caseparticipantrole.typecode = 'PRI' or caseparticipantrole.typecode = 'MEM')`;
return executeCmd(connection, cmd, row.CASEID).then(result => {
caseObj.concernedRole = result
return caseObj
})
}
UPDATE: SOLUTION FOUND. ARANGODB CLUSTER DOES NOT SUPPORT TRANSACTIONS. IT IS ONLY SUPPORTED ON SINGLE INSTANCES.
I am trying to use the transactions function using arangoJS library. The function that I will present is just a dummy function that inserts two records, and then tries to get a document that doesn't exist. Getting the nonexistent document generates an error, and the transaction must rollback. Indeed, the error gets generated after trying to get the document that doesn't exist. However, the database does not rollback, and the two inserted documents remain inserted in the database. Does anyone know how to solve it?
"updateCustomer" : function (options, cb) {
const action = String(function (params) {
// This code will be executed inside ArangoDB!
const db = require('#arangodb').db;
const aql = require('#arangodb').aql;
const customer = db._collection('customer');
try{
//insert two documents
db._query(aql`INSERT ${params.user} INTO ${customer} Return NEW`);
db._query(aql`INSERT ${params.customer} INTO ${customer} Return NEW`);
//Get a document that doesn't exist
customer.document('does-not-exist');
}catch(e){
throw new Error("Everything is bad");
}
});
let opts = {
collections : {
read : ["customer"],
write : ["customer"]
},
action : action,
params : {user: options, customer: options},
lockTimeout : 5
};
Arango.transaction(opts,(err, result) => {
console.log("err: " + err);
console.log("result: " + JSON.stringify(result));
return cb(err, result);
});
}
"transaction" : function (options, cb) {
utils.dbConnect().transaction(options.collections, options.action, options.params, options.lockTimeout, cb);
}
UPDATE: I tried this transaction on a single instance ArangoDB and it worked. However, it did not work on a cluster. Is there no support for transactions on ArangoDB clusters?
Single document operations are atomic in arangodb clusters. Multi-document are not as of now. We are currently working on ACID for multi-document operations.
Hi Can anyone give an example of how use insert statement in nodejs. I am able to use select query. But for insert query i am getting the result as []. no error can be seen but the values are not added to the original table. I am using db2, ibm_db,express,nodejs and angularjs.
I wrote a blog entry on using DB2 and node.js on Bluemix a while ago. It includes code for an INSERT statement.
As part of the insert
first prepare the statement,
then bind the values to be inserted and
finally execute the statement.
Here is the relevant code snippet, the full context is in the blog:
exports.insertIP = function(ibmdb,connString,ipinfo) {
console.log("insertIP called",ipinfo);
ibmdb.open(connString, function(err, conn) {
if (err ) {
res.send("error occurred " + err.message);
}
else {
// prepare the SQL statement
conn.prepare("INSERT INTO IP.VISITORS(vtime,ip,country_code,country,region_code,region,city,zip,latitude,longitude,metro,area) VALUES (current timestamp,?,?,?,?,?,?,?,?,?,?,?)", function(err, stmt) {
if (err) {
//could not prepare for some reason
console.log(err);
return conn.closeSync();
}
//Bind and Execute the statment asynchronously
stmt.execute([ipinfo["ip"],ipinfo["country_code"],ipinfo["country_name"],ipinfo["region_code"],ipinfo["region_name"],ipinfo["city"],ipinfo["zipcode"], ipinfo["latitude"], ipinfo["longitude"],ipinfo["metro_code"],ipinfo["area_code"]], function (err, result) {
console.log(err);
// Close the connection to the database
conn.close(function(){
console.log("Connection Closed");
});
});
});
}
})};
I would suggest and recommend (as one of the members of node-ibm_db) to follow the node-ibm_db github repository (https://github.com/ibmdb/node-ibm_db) , we have updated the README document as well as the list of APIs to do particular tasks.
For your above query you can use ".prepare(sql, callback)" or ".prepareSync(sql)" API (as per your requirements Async/sync call), below is the attached code snippet and URL link for particular API documentation.
var ibmdb = require("ibm_db"),
cn ="DATABASE=dbname;HOSTNAME=hostname;PORT=port;PROTOCOL=TCPIP;UID=dbuser;PWD=xxx";
ibmdb.open(cn,function(err,conn){
conn.prepare("insert into hits (col1, col2) VALUES (?, ?)",
function (err, stmt) {
if (err) {
//could not prepare for some reason
console.log(err);
return conn.closeSync();
}
//Bind and Execute the statment asynchronously
stmt.execute(['something', 42], function (err, result) {
if( err ) console.log(err);
else result.closeSync();
//Close the connection
conn.close(function(err){});
});
});
});
API documentation(Github URL) : https://github.com/ibmdb/node-ibm_db#-8-preparesql-callback
Try to install jt400 by using the below command
npm install node-jt400 --save
use the below code to insert the data to table name foo.
Follow link https://www.npmjs.com/package/node-jt400 for for details
pool
.insertAndGetId('INSERT INTO foo (bar, baz) VALUES(?,?)',[2,'b'])
.then(id => {
console.log('Inserted new row with id ' + id);
});