in sqlite3 (node package), how to run multiple queries by sequence? - node.js

I am using this pakcage: sqlite3 to manipulate database queries, code like this:
single query is quit simple:
var sqlite3 = require('sqlite3').verbose();
let db = new sqlite3.Database('database.sqlite3');
let entity_1_name = ''
db.serialize(function(){
db.get('select * from table1 where id = 1', function(error, row){
entity_1_name = row.name
})
})
db.close()
But how to execute multiple queries and in sequence?
(e.g. I want to run query1, then query2, then query3 )
just like:
var sqlite3 = require('sqlite3').verbose();
let db = new sqlite3.Database('database.sqlite3');
let entity_1_name = ''
let entity_2_name = ''
db.serialize(function(){
// this is query1
db.get('select * from table1 where id = 1', [], function(error, row){
entity_1_name = row.name
})
// this is query2
db.get('select * from table2 where dependency_column_name = ' + entity_1_name, function(error,row){
// this sql returns null value, because when this query was executed, the variable "entity_1_name" was not determined by query1.
})
})
db.close()

https://stackoverflow.com/a/40903302/16140221 May help you out
Their solution was to place each query within a function, then simply calling them.
(This is directly the code taken from the answer, it is not mine.)
function db_query1(your_param,...., callback){
  // database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
function db_query2(your_param,...., callback){
  // database operation
db.run( sql , [param,...] , function(err,rows){
if(err) // return
else{
// get rows with callback
callback(null, rows);
}
});
}
Then calling the functions
db_query1(....,function(err,result1){
if(err) ...// return
// do the things with result1
// And then call query2
db_query2(....,function(err,result2){
if(err) ...// return
// do the things with result1
});
});

OK, at last, I used promise + async + await to make it done.
Step1. define a function return new Promise()
do_query_1(material_name){
// 1. return this new Promise()
return new Promise( (resolve, reject) => {
let db = this.$database_tool.get_db()
let that = this
db.serialize(function(){
db.get(`select * from table1 where id = 1`, [], function(error, row){
// 2. this is the point, put anything you want to result to resolve()
resolve(row.id)
})
})
db.close();
})
},
Step2. define a async function, to call the method defined in Step1.
async do_query_2(){
let that = this
// other lines was omitted.
// HERE , use await to get the result from STEP1.
let entity_1_name = await that.do_query_1(row.material_name)
db.serialize(function(){
let sql = `'select * from table2 where dependency_column_name = ' + entity_1_name`
db.run(sql)
})
db.close();
},
Step3. call do_query_2() in normal way:
do_query_2()
That's it!

Related

how to promises in a loop?

I am trying write a cron function in nodejs which fetches user_ids of all the users from the db and then I want to parse through each user_id.
Here is my code :
cron.schedule('43 11 * * *', function(){
var now = moment()
var formatted = now.format('YYYY-MM-DD HH:mm:ss')
console.log('Starting the cron boss!');
var dbSelectPromise = function(db, sql1) {
return new Promise(function(resolve, reject) {
db.select(sql1, function(err, data) {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
var users =[]
var sql = "select distinct(user_id) from user_level_task"
dbSelectPromise(db,sql).then(function(secondResult){
for(i=0;i<secondResult.length;i++){
var sql1 = "select max(level_id) as level from user_level_task where user_id ="+secondResult[i].user_id
dbSelectPromise(db,sql1).then(function(thirdResult){
console.log(thirdResult)
console.log(current)
var sql2 = "select task_id form user_level_task where user_id = '"+secondResult[i].user_id+"' and level_id = '"+thirdResult[0].level+"' "
dbSelectPromise(db,sql2).then(function(fourthResult){
var leng = fourthResult.length
for(i=0;i<leng;i++){
console.log(fourthResult[i])
}
})
})
}
})
});
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
I think what's happening is that i is no longer the same when you create those new promises because the for loop is still running. It appears that what you really need is the user_id and level_id. I suggest you restructure your code a bit to reduce nesting and pass on the values you need for future promises.
Perhaps something similar to this:
dbSelectPromise(db, sql)
.then(secondResult => {
const levelPromises = [];
secondResult.forEach(res => {
levelPromises.push(getLevelByUserId(res.user_id, db));
});
return Promise.all(levelPromises); // Promise.all only if you want to handle all success cases
})
.then(result => {
result.forEach( level => {
{ userId, queryResult } = level;
// ...
})
//...
})
.catch(err => {
console.log(err);
});
function getLevelByUserId(userId, db) {
const query = `select max(level_id) as level from user_level_task where user_id = ${userId}`;
return dbselectPromise(db, query).then(result => { userId, result });
}
It creates an array of all the get level queries as promises and then passes it along to the next step using Promise.all() which will only resolve if all queries were successful. At that point, you will have access to the userId again of each result because we returned it in our new function for your next set of queries.
I think you should abstract your queries a bit further instead of using a generic dbSelectPromise and don't forget to catch() at the end otherwise you won't know what's happening.
Note: It assumes your db variable instantiated properly and your original db.select doesn't need to be returned based on whatever library you're using. There's also some new syntax there.
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
This is because you're using reinitializing i without using let. When the loop is in process, the value will be different than what you expect.
each promise is dependant on the other and need to run synchronously
For this to work, You need to chain promises. Also, you can make use of Promise.all() to execute a bunches of promises at once. Remember, Promise.all() is all or nothing.
Making those changes to your code, I get the following structure.
'use strict';
let _ = require('lodash');
function dbSelectPromise(db, sql1) {
return new Promise((resolve, reject) => {
return db.select(sql1, (err, data) => {
if (err) {
return reject(err);
}
return resolve(data);
});
});
}
function process(secondResult) {
let sql1 = "select max(level_id) as level from user_level_task where user_id =" + secondResult[i].user_id;
return dbSelectPromise(db, sql1).then(function (thirdResult) {
console.log(thirdResult);
let sql2 = "select task_id form user_level_task where user_id = '" + secondResult[i].user_id + "' and level_id = '" + thirdResult[0].level + "' ";
return dbSelectPromise(db, sql2);
});
}
function getUsers() {
let sql = "select distinct(user_id) from user_level_task";
return dbSelectPromise(db, sql).then((users) => {
return users;
}).catch(() => {
return [];
});
}
cron.schedule('43 11 * * *', function () {
var now = moment();
var formatted = now.format('YYYY-MM-DD HH:mm:ss');
getUsers().then((users) => {
let batches = _.map(users, function (user) {
return process(user);
});
return Promise.all(batches);
}).then((fourthResult) => {
console.log('Your fourthResult [[],..]', fourthResult);
}).catch(() => {
console.log('err while processing', err);
});
});

Error: Missing query string MariaDB NodeJS

I have this error every time I try to run a query to DB:
Error: Missing query string
I don't understand what could be wrong with my code, the query is correct I have tested it:
function getChannelCTSize(octopusMac, channelID, adcTicks, callback){
var queryString = "SELECT Channels.CT_size FROM Channels INNER JOIN Octopus ON Octopus.Id=Channels.Octopus_Id WHERE Octopus.Mac = ? AND Channels.Channel_Id = ?";
var filter = [octopusMac, channelID];
var query = mariaDB.query({
sql: queryString
}, filter );
query.on('error', function(err) {
if (err) {
console.log(err.code);
return
}
})
.on('result', function(row) {
callback(result[0].CT_size, channelID, octopusMac, adcTicks);
})
.on('end', function() {
mariaDB.release();
});
}
The query function of mariasql should be invoked with a string as first argument (the query) instead of an object)
It should look like:
var queryString = "SELECT Channels.CT_size FROM Channels INNER JOIN Octopus ON Octopus.Id=Channels.Octopus_Id WHERE Octopus.Mac = :octopusMac AND Channels.Channel_Id = :channelID";
var query = mariaDB.query(queryString, { octopusMac, channelID }, function(err, rows) {
if (err)
console.log(err.code);
callback(rows[0].CT_size, channelID, octopusMac, adcTicks);
})

nodejs get sqlite3 query result using promise or wait

This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})

How to properly return a result from mysql with Node?

In the code
var stuff_i_want = '';
stuff_i_want = get_info(parm);
And the function get_info:
get_info(data){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
console.log(stuff_i_want); // Yep. Value assigned..
}
in the larger scope
stuff_i_want = null
What am i missing regarding returning mysql data and assigning it to a variable?
============ New code per Alex suggestion
var parent_id = '';
get_info(data, cb){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
return cb(results[0].objid); // Scope is larger than function
}
==== New Code in Use
get_data(parent_recording, function(result){
parent_id = result;
console.log("Parent ID: " + parent_id); // Data is delivered
});
However
console.log("Parent ID: " + parent_id);
In the scope outside the function parent_id is null
You're going to need to get your head around asynchronous calls and callbacks with javascript, this isn't C#, PHP, etc...
Here's an example using your code:
function get_info(data, callback){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
return callback(results[0].objid);
})
}
//usage
var stuff_i_want = '';
get_info(parm, function(result){
stuff_i_want = result;
//rest of your code goes in here
});
When you call get_info this, in turn, calls connection.query, which takes a callback (that's what function(err, results) is
The scope is then passed to this callback, and so on.
Welcome to javascript callback hell...
It's easy when you get the hang of it, just takes a bit of getting used to, coming from something like C#
I guess what you really want to do here is returning a Promise object with the results. This way you can deal with the async operation of retrieving data from the DBMS: when you have the results, you make use of the Promise resolve function to somehow "return the value" / "resolve the promise".
Here's an example:
getEmployeeNames = function(){
return new Promise(function(resolve, reject){
connection.query(
"SELECT Name, Surname FROM Employee",
function(err, rows){
if(rows === undefined){
reject(new Error("Error rows is undefined"));
}else{
resolve(rows);
}
}
)}
)}
On the caller side, you use the then function to manage fulfillment, and the catch function to manage rejection.
Here's an example that makes use of the code above:
getEmployeeNames()
.then(function(results){
render(results)
})
.catch(function(err){
console.log("Promise rejection error: "+err);
})
At this point you can set up the view for your results (which are indeed returned as an array of objects):
render = function(results){ for (var i in results) console.log(results[i].Name) }
Edit
I'm adding a basic example on how to return HTML content with the results, which is a more typical scenario for Node. Just use the then function of the promise to set the HTTP response, and open your browser at http://localhost:3001
require('http').createServer( function(req, res){
if(req.method == 'GET'){
if(req.url == '/'){
res.setHeader('Content-type', 'text/html');
getEmployeeNames()
.then(function(results){
html = "<h2>"+results.length+" employees found</h2>"
html += "<ul>"
for (var i in results) html += "<li>" + results[i].Name + " " +results[i].Surname + "</li>";
html += "</ul>"
res.end(html);
})
.catch(function(err){
console.log("Promise rejection error: "+err);
res.end("<h1>ERROR</h1>")
})
}
}
}).listen(3001)
Five years later, I understand asynchronous operations much better.
Also with the new syntax of async/await in ES6 I refactored this particular piece of code:
const mysql = require('mysql2') // built-in promise functionality
const DB = process.env.DATABASE
const conn = mysql.createConnection(DB)
async function getInfo(data){
var sql = "SELECT a from b where info = data"
const results = await conn.promise().query(sql)
return results[0]
}
module.exports = {
getInfo
}
Then, where ever I need this data, I would wrap it in an async function, invoke getInfo(data) and use the results as needed.
This was a situation where I was inserting new records to a child table and needed the prent record key, based only on a name.
This was a good example of understanding the asynchronous nature of node.
I needed to wrap the all the code affecting the child records inside the call to find the parent record id.
I was approaching this from a sequential (PHP, JAVA) perspective, which was all wrong.
Easier if you send in a promise to be resolved
e.g
function get_info(data, promise){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
promise.resolve(results[0].objid);
}
}
This way Node.js will stay fast because it's busy doing other things while your promise is waiting to be resolved
I've been working on this goal since few weeks, without any result, and I finally found a way to assign in a variable the result of any mysql query using await/async and promises.
You don't need to understand promises in order to use it, eh, I don't know how to use promises neither anyway
I'm doing it using a Model class for my database like this :
class DB {
constructor(db) {
this.db = db;
}
async getUsers() {
let query = "SELECT * FROM asimov_users";
return this.doQuery(query)
}
async getUserById(array) {
let query = "SELECT * FROM asimov_users WHERE id = ?";
return this.doQueryParams(query, array);
}
// CORE FUNCTIONS DON'T TOUCH
async doQuery(queryToDo) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
async doQueryParams(queryToDo, array) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, array, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
}
Then, you need to instantiate your class by passing in parameter to constructor the connection variable given by mysql. After this, all you need to do is calling one of your class methods with an await before. With this, you can chain queries without worrying of scopes.
Example :
connection.connect(function(err) {
if (err) throw err;
let DBModel = new DB(connection);
(async function() {
let oneUser = await DBModel.getUserById([1]);
let allUsers = await DBModel.getUsers();
res.render("index.ejs", {oneUser : oneUser, allUsers : allUsers});
})();
});
Notes :
if you need to do another query, you just have to write a new method in your class and calling it in your code with an await inside an async function, just copy/paste a method and modify it
there are two "core functions" in the class, doQuery and doQueryParams, the first one only takes a string as a parameter which basically is your mysql query. The second one is used for parameters in your query, it takes an array of values.
it's relevant to notice that the return value of your methods will always be an array of objects, it means that you'll have to do var[0] if you do a query which returns only one row. In case of multiple rows, just loop on it.

node.js synchronous call gather results

I have a table where Persons are saved an their time (in seconds) where they were active.
I would like to write a function that gathers the total time in another table called gather.
For each row I am checking if an entry in the gather table exists. Depending on that result I make an insert or an update.
db.serialize(function() {
db.each("SELECT * from TEST", function(err, row) {
db.get("SELECT * from GATHER where name = " + row.name "", function(err, row) {
if(row === undefined || row === null){
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([name, seconds], function(error){
console.log('lastInsert ' + this.lastID);
});
stmt.finalize();
}else{
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('lastInsert ' + row.idProcessed);
});
stmt.finalize();
}
});
});
});
The problem that I ecounter is that sqlite runs asynchronously. Therefore multiple entries are created in my gather table although lines should be updated.
What would be the right way to run this function sychronously? Should I limit the lines and call the function every second or is there a smarted way?
You could use async. For example (but first you should read final notes at the end):
var async = require('async');
var data = {}
var yourFirstSelect() = function(callback){
//You do your first select
//...
db.get("SELECT * from TEST", function(err, row) {
if(row){
data.name = row.name;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var yourSecondSelect() = function(data, callback){
//You do your second select
//...
db.get("SELECT * from GATHER where name = " + data.name "", function(err, row) { //Note that I'm using data instead of row
if(row){
data.seconds = row.seconds;
data.otherInterestingAttribute = row.otherInterestingAttribute;
callback(err, data);
}else{
callback('Row is null');
}
})
//..
}
var decide() = function(data, callback){
if (data.somethingExists) { //Do your validations
data.type = 'update';
callback(err, data);
} else {
data.type = 'insert';
callback(err, data);
}
}
var update() = function(data,callback){
if (data.type === 'update') {
//...
//Do your stuff in order to update
seconds += row.time;//increment time
var stmt = db.prepare("UPDATE gather SET time = ? WHERE name = ?");
stmt.run([seconds, row.name], function(error){
console.log('update ' + row.idProcessed);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var insert() = function(data,callback){
if (data.type === 'insert') {
//...
//Do your stuff in order to insert
var stmt = db.prepare("INSTER INTO gather (name, time) VALUE(?,?)");
stmt.run([data.name, data.seconds], function(error){
console.log('lastInsert ' + this.lastID);
callback(err,data);
});
stmt.finalize();
//...
} else {
callback(err,data);
}
}
var niceWorkflow = function(){
async.waterfall([
yourFirstSelect,
yourSecondSelect,
decide,
update,
insert
],
function (err, result) {
console.log('Done');
})
}
//and call your workflow
niceWorkflow();
Off course this is not a 100% working code, I wrote it in order you look another way to do what you are trying. Many variables, validations and more are just examples and I intentionally forgot the db.each to avoid being too extense and confusing you and trying to answer your final question Is there a smarter way?.
You can also use Q.all from Q library.
It will return a promise after all the promises are resolved. It one promise fails, then Q.all will be rejected.

Resources