This is my first personal project in Nodejs. I'm trying to get in live soon.
I have a Nodejs server that uses sqlite3. There are only 3000 rows with word, transform and a precalculated value each in a column of the table, which is already populated.
I need to just lookup the word in the DB to be sure it is valid.
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.get("SELECT * FROM tab WHERE w = ?", word, function(err, row) {
if(err) { console.log("Lookup:",word,", Error => ",err); return false; }
return true;
});
The problem is that the caller of this code has a lot of context and need the operation to wait. So, I tried this
function dbLookup(db, w) {
return function(cb) {
var rows = [];
db.exec('SELECT w FROM tab WHERE w = "'+w+'"')
.on('row', function(r) {
rows.push(r)
})
.on('result', function() {
cb(rows);
});
}
async.each([word], function(w) {
dbLookup(this.db, w);
}, function(err) {
if(err) {console.log("...ERROR..."); return false; }
else {console.log("...SUCCESS..."); return true; }
});
This doesn't solve the wait issue as the callback can fire at its own pace.
I read that promise using something like bluebird can solve my problem
but now I'm not able to get the value/result of the query out:
I've been pulling my hair for so long. Please help me either get the async working or get the result back from the promise approach.
var async = require('async');
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
function check(word, callback) {
db.get("SELECT count(1) cnt FROM tab WHERE w = ?", word, callback)
}
async.map(words, check, function(err, results) {
if (err)
return console.log('Query error')
var all_checked = results.filter(function(r) {
return r.cnt > 0
});
...
});
Or
var sqlite3 = require("sqlite3").verbose();
var db = new sqlite3.Database("validate.db");
db.all("SELECT distinct w FROM tab", function(err, rows) {
var all_checked = words.filter(function (w) {
return rows.indexOf(w) != -1;
})
...
})
Related
I am new to Javascript and am just getting familiar with promises so forgive my ignorance.
What I'm trying to do is request all the records from an Airtable base and filter them them based on a checkbox called "Email Sent" being unsent. When I have the records filtered there are linked records in 2 fields that I need to do additional requests for to get the values for. All of those values (various data, and recipients as an object) then are plugged into an HTML email template and fired off using AWS-SES. Then the "Email Sent" check box is checked for those records to prevent emails from being sent multiple times. The plan is to have this all running on an interval in AWS Lambda. And as records are added to the base they are automatically emailed to a list.
I am able to get the records, and filter them. I am also comfortable with executing the code within Lambda and using SES. But I have been struggling for days to get the values for the linked records. Both of which are lists of people and 1 of which is a recipient email address.
The numerous things I've tried end up either returning the original record called in the first step or returns undefined. I'm chaining a lot of promises and I don't think I'm doing it right because it seems they are running out of order. I'm also at the point of trying weird things like using a global array instead of an object that I'm passing down through the promises and updating.
Any help is appreciated. Thank you in advance.
var Airtable = require('airtable');
var base = new Airtable({apiKey: 'xxxxxxxxxx'}).base('xxxxxxxxxx');
var nodemailer = require("nodemailer")
var handlebars = require('handlebars');
const path = require('path');
var AWS = require("aws-sdk")
var ses = new AWS.SES();
var fs = require('fs');
var mainArray = [];
var readHTMLFile = function(path, callback) {
fs.readFile(path, {encoding: 'utf-8'}, function (err, html) {
if (err) {
throw err;
callback(err);
}
else {
callback(null, html);
}
});
};
function getRecords(){
return new Promise(function(resolve, reject) {
var reqArr = [];
base('Edit Requests').select({
// Selecting the first 3 records in Grid view:
maxRecords: 50,
view: "Grid view"
}).eachPage(function page(records, fetchNextPage) {
// This function (`page`) will get called for each page of records.
records.forEach(function(record) {
// console.log("108: ", record._rawJson.id)
var obj = {}
// obj = record.fields;
// obj.id = record._rawJson.id;
console.log("172", record.fields["Editor Preference"])
obj = record.fields;
obj.id = record._rawJson.id;
if(record.fields["Editor Preference"] != undefined){
obj["Editor Preference"] = obj["Editor Preference"]
// obj["Editor Preference"] = getEditorWrap(record.fields["Editor Preference"])
} else {
obj["Editor Preference"] = "";
}
if(record.fields["Production Manager"] != undefined){
obj["Production Manager"] = obj["Production Manager"]
} else {
obj["Production Manager"] = "";
}
mainArray.push(obj)
// console.log(record.fields["Email"])
// console.log('Retrieved', record.fields['Requested By']);
})
// To fetch the next page of records, call `fetchNextPage`.
// If there are more records, `page` will get called again.
// If there are no more records, `done` will get called.
fetchNextPage();
// console.log("123", reqArr)
// resolve(reqArr)
}, function done(err) {
if (err) { console.error(err); return; }
// console.log("123", mainArray)
resolve(mainArray)
});
// resolve(reqArr)
});
}
function filterRecords(arr){
return new Promise(function(resolve, reject) {
var filtered = []
mainArray = [];
// console.log("245", arr)
for (i in arr){
if(arr[i]['Email Sent'] == undefined){
// console.log("247", arr[i])
mainArray.push(arr[i])
};
}
console.log("filtered: ", mainArray)
resolve(mainArray)
});
}
function setObject(array){
return new Promise(function(resolve, reject) {
for (i in array){
var obj = array[i];
if(obj.id != undefined){
base('Editors').find(obj.id, function(err, record) {
if (err) { console.error(err); return; }
// console.log("281", record.fields);
});
}
}
resolve(mainArray)
});
}
function main1(){
return new Promise(function(resolve, reject) {
getRecords().
then(function(recordArr){
filterRecords(mainArray).
then(function(resultArr){
setObject(mainArray).
then(function(mainArray){
})
})
})
});
}
main1().
then(function(resultArray){
console.log(resultArray)
})
I am trying to generate a session ID when an API call is made to my 'generateSession' endpoint. I want to make sure that I don't have any duplicate session ID, so I query the database checking for a match on the generated ID. If there isn't a match, the ID is valid and I make a second query to add an active user with said session ID.
Although my first query executes, the 'id_is_valid' boolean never gets set to true so my program gets stuck in the while loop.
I am fairly new to JavaScript, but from some research, I'm pretty sure the problem is due to the asynchronous nature of the database call. However, I'm not sure where to go from there. Could anyone with more js knowledge offer me some direction?
Thanks!
var express = require('express');
var router = express.Router();
var myDB = require('../db-connection');
function generateSession() {
var session_id = '';
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for (var i = 0; i < 30; i++) session_id += possible.charAt(Math.floor(Math.random() * possible.length));
return session_id;
}
router.get('/generateSession', function(req, res){
var session_id = '';
var id_is_valid = false;
while (!id_is_valid){
session_id = generateSession();
myDB.query("SELECT * FROM activeUser WHERE session_id = ?", [session_id], function(error, results, field){
if(error) throw error;
else{
if (results.length === 0) is_is_valid = true;
}
});
}
myDB.query("INSERT INTO activeUser (is_registered, session_id) VALUES (0, ?)", [session_id], function(error, results, fields){
if (error) res.send('{"success": false}');
else res.send('{"success": true, "session_id": "' + session_id + '"}');
});
});
Although my first query executes, the id_is_valid boolean never gets set to true so my program gets stuck in the while loop.
This is because db call, by nature is asynchronous. If you run the following program you will know it.
'use strict';
let id_is_valid = false;
let count = 0;
while (!id_is_valid) {
count++;
console.log(`No of time setTimeout Invoked ${count}`);
setTimeout(function() { // simulating a db call that takes a second for execution
id_is_valid = true;
}, 1000);
}
console.log('This line wont be printed');
Outputs
No of time setTimeout Invoked 61415
No of time setTimeout Invoked 61416
No of time setTimeout Invoked 61417
^C //I killed it.
Like damitj07, I too suggest using npms like shortId for uniquely generating the sessionId. This will help you to eliminate a database call.
But if your business logic restricts and you need it in the current fashion you wrote. I think we can use async & await
'use strict';
let isFound = false;
let count = 0;
function doDbQuery() {
return new Promise((resolve, reject) => {
setTimeout(function () {
resolve(true);
}, 2000);
});
}
async function run() {
while (!isFound) {
count++;
console.log(`No of time setTimeout Invoked ${count}`);
isFound = await doDbQuery();
}
console.log('This line WILL BE printed');
}
run();
Output
No of time setTimeout Invoked 1
This line WILL BE printed
Making those changes to your code,
router.get('/generateSession', async function (req, res) {
var session_id = '';
var id_is_valid = false;
while (!id_is_valid) {
session_id = generateSession();
id_is_valid = await checkSessionIdInDb(session_id);
}
myDB.query('INSERT INTO activeUser (is_registered, session_id) VALUES (0, ?)', [session_id], function (error, results, fields) {
if (error) {
res.send('{"success": false}');
} else {
res.send('{"success": true, "session_id": "' + session_id + '"}');
}
});
});
function checkSessionIdInDb() {
return new Promise((resolve, reject) => {
myDB.query('SELECT * FROM activeUser WHERE session_id = ?', [session_id], function (error, results, field) {
if (error) {
return reject(error);
} else {
if (results.length === 0) {
resolve(true);
}
resolve(false);
}
});
});
From what I could make out of your code what you want to do is basically create a new user session for logged in user with a unique session Id, also making sure that the session ID does not already exist in the collection.
So the solution to this can be to the first query to check if the session ID is already present in Active_Users Collection, if not make a save call to save the user with generated session Id.
var express = require('express');
var router = express.Router();
var myDB = require('../db-connection');
function generateSession() {
var session_id = '';
var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
for (var i = 0; i < 30; i++) session_id += possible.charAt(Math.floor(Math.random() * possible.length));
return session_id;
}
router.get('/generateSession', function(req, res) {
var session_id = '';
myDB.query("SELECT * FROM activeUser WHERE session_id = ?", [session_id], function(error, results, field) {
if (error) {
throw error;
} else {
if (results.length === 0) {
//if session id is not present , insert new user
session_id = generateSession();
myDB.query("INSERT INTO activeUser (is_registered, session_id) VALUES (0, ?)", [session_id], function(error, results, fields) {
if (error) res.send('{"success": false}');
else res.send('{"success": true, "session_id": "' + session_id + '"}');
});
}
//else do nothing or inser nothing
}
});
});
But Ideally, if you are generating a truly random session id each time when you make a request to the /generate session, is it really necessary to check for duplicates in the collection.
Now if the logic to generate a random ID is not perfect, you can always use modules like shortId to do the work for you. This will avoid unnecessary database call and save on performance and your code will be much neater.
I am trying write a cron function in nodejs which fetches user_ids of all the users from the db and then I want to parse through each user_id.
Here is my code :
cron.schedule('43 11 * * *', function(){
var now = moment()
var formatted = now.format('YYYY-MM-DD HH:mm:ss')
console.log('Starting the cron boss!');
var dbSelectPromise = function(db, sql1) {
return new Promise(function(resolve, reject) {
db.select(sql1, function(err, data) {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
var users =[]
var sql = "select distinct(user_id) from user_level_task"
dbSelectPromise(db,sql).then(function(secondResult){
for(i=0;i<secondResult.length;i++){
var sql1 = "select max(level_id) as level from user_level_task where user_id ="+secondResult[i].user_id
dbSelectPromise(db,sql1).then(function(thirdResult){
console.log(thirdResult)
console.log(current)
var sql2 = "select task_id form user_level_task where user_id = '"+secondResult[i].user_id+"' and level_id = '"+thirdResult[0].level+"' "
dbSelectPromise(db,sql2).then(function(fourthResult){
var leng = fourthResult.length
for(i=0;i<leng;i++){
console.log(fourthResult[i])
}
})
})
}
})
});
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
I think what's happening is that i is no longer the same when you create those new promises because the for loop is still running. It appears that what you really need is the user_id and level_id. I suggest you restructure your code a bit to reduce nesting and pass on the values you need for future promises.
Perhaps something similar to this:
dbSelectPromise(db, sql)
.then(secondResult => {
const levelPromises = [];
secondResult.forEach(res => {
levelPromises.push(getLevelByUserId(res.user_id, db));
});
return Promise.all(levelPromises); // Promise.all only if you want to handle all success cases
})
.then(result => {
result.forEach( level => {
{ userId, queryResult } = level;
// ...
})
//...
})
.catch(err => {
console.log(err);
});
function getLevelByUserId(userId, db) {
const query = `select max(level_id) as level from user_level_task where user_id = ${userId}`;
return dbselectPromise(db, query).then(result => { userId, result });
}
It creates an array of all the get level queries as promises and then passes it along to the next step using Promise.all() which will only resolve if all queries were successful. At that point, you will have access to the userId again of each result because we returned it in our new function for your next set of queries.
I think you should abstract your queries a bit further instead of using a generic dbSelectPromise and don't forget to catch() at the end otherwise you won't know what's happening.
Note: It assumes your db variable instantiated properly and your original db.select doesn't need to be returned based on whatever library you're using. There's also some new syntax there.
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
This is because you're using reinitializing i without using let. When the loop is in process, the value will be different than what you expect.
each promise is dependant on the other and need to run synchronously
For this to work, You need to chain promises. Also, you can make use of Promise.all() to execute a bunches of promises at once. Remember, Promise.all() is all or nothing.
Making those changes to your code, I get the following structure.
'use strict';
let _ = require('lodash');
function dbSelectPromise(db, sql1) {
return new Promise((resolve, reject) => {
return db.select(sql1, (err, data) => {
if (err) {
return reject(err);
}
return resolve(data);
});
});
}
function process(secondResult) {
let sql1 = "select max(level_id) as level from user_level_task where user_id =" + secondResult[i].user_id;
return dbSelectPromise(db, sql1).then(function (thirdResult) {
console.log(thirdResult);
let sql2 = "select task_id form user_level_task where user_id = '" + secondResult[i].user_id + "' and level_id = '" + thirdResult[0].level + "' ";
return dbSelectPromise(db, sql2);
});
}
function getUsers() {
let sql = "select distinct(user_id) from user_level_task";
return dbSelectPromise(db, sql).then((users) => {
return users;
}).catch(() => {
return [];
});
}
cron.schedule('43 11 * * *', function () {
var now = moment();
var formatted = now.format('YYYY-MM-DD HH:mm:ss');
getUsers().then((users) => {
let batches = _.map(users, function (user) {
return process(user);
});
return Promise.all(batches);
}).then((fourthResult) => {
console.log('Your fourthResult [[],..]', fourthResult);
}).catch(() => {
console.log('err while processing', err);
});
});
While my database server is not available and any function of my node-express rest service like hiExpress is called, Nodejs crashes the node server and console of node reports
sql server connection closed
I do not want this to happen because either it should go to err function or at least it must be cautht by catch block. What could i do to avoid the crash of nodejs when database server is not available I am using following code which is absolutely fine as long as database server is available.
var sqlServer = require('seriate');
app.get('/hiExpress',function(req, res)
{
var sr = {error:'',message:''};
var sql= 'select * from table1 where id=? and name=?';
var params = {id: 5, name:'sami'};
exeDB(res,sr,sql, params);//sent only 4 parameters (not 6)
});
function exeDB(res, sr, sql, params, callback, multiple) {
try {
var obj = {};
for (p in params) {
if (params.hasOwnProperty(p)) {
obj[p] = {
type: sqlServer.VARCHAR,
val: params[p]
};
}
};
var exeOptions = {
query: sql,
params: obj
};
if (multiple) {
exeOptions.multiple = true;
}
sqlServer.execute(sqlServerConfigObject, exeOptions).then(function (results) {
sr.data = results;
if (callback)
callback(sr);
else
res.json(sr); //produces result when success
}, function (err) {
//sr.message = sql;
console.log(11);
sr.error = err.message;
res.json(sr);
});
}
catch (ex) {
console.log(21);
sr.error = ex.message;
res.json(sr);
}
}
Why I preferred to use seriate
I had not been much comfortable with node-SQL, especially when when it came to
multiple queries option even not using a transaction. It facilitates easy go to parameterized queries.
You can use transaction without seriate but with async like below
async.series([
function(callback) {db.run('begin transaction', callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run( ..., callback)},
function(callback) {db.run('commit transaction', callback)},
], function(err, results){
if (err) {
db.run('rollback transaction');
return console.log(err);
}
// if some queries return rows then results[query-no] contains them
})
The code is very dirty. Pass req and res params to db-layer is not a good idea.
Try change exeDB. I'm not sure, but probably you don't set error catcher to promise
function exeDB(res, sr, sql, params, callback, multiple) {
// It will execute with no error, no doubt
var obj = {};
for (p in params) {
if (params.hasOwnProperty(p)) {
obj[p] = {
type: sqlServer.VARCHAR,
val: params[p]
};
}
};
var exeOptions = {
query: sql,
params: obj
};
if (multiple) {
exeOptions.multiple = true;
}
// Potential problem is here.
// Catch is useless because code below is asynchronous.
sqlServer.execute(sqlServerConfigObject, exeOptions).then(function (results) {
sr.data = results;
if (callback)
callback(sr);
else
res.json(sr); //produces result when success
}).error(function(err){ // !!! You must provide on-error
console.log(err);
};
}
Considering that my server.js looks almost like this. Just send you the relevant part. I did not receive anything from the query, I do have data in the database, and "sendNotification" is triggered by the jQuery function in the client. Everything works and since var notis = []; returns an empty value and is what is shows as response. I know I have to debug SQL and that's what I'm going to do but anyway want to be sure of this other things. So my questions are:
1) Is a right syntax for node.js, considering this async behavior? (which I still don't understand )
2) The query always should be inside of the "io.sockets.on('connection')" part?
connection = mysql.createConnection({
host: 'localhost',
user: '',
password: "",
database: 'table' //put your database name
}),
...
connection.connect(function(err) {
// connected! (unless `err` is set)
console.log(err);
});
…
var sqlquery = function(uID,vs){
var notis = [];
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
return notis.push(data);
});
};
io.sockets.on('connection', function(socket) {
...
socket.on("sendNotification", function(data) {
var roomBName = data.room_name.replace("room-",""),
found = [];
var roomSelected = _.find(rooms, function (room) { return room.id == roomBName });
for (var person in people) {
for (var i = 0, numAttending = roomSelected.peopleAttending.length; i < numAttending; i++) {
if (people[person].name == roomSelected.peopleAttending[i]) {
found.push(person);
}
}
}
for (var i = 0, numFound = found.length; i < numFound; i++) {
**result = sqlquery(9,2);**
io.to(found[i]).emit('notification', result);
};
});
Your sqlquery() function will not accomplish anything useful. Because connection.query() is asynchronous, that means it provides the response sometime LATER after sqlquery() has already finished.
The only way in node.js to use an async result is to actually use it in the callback that provides it. You don't just stuff it into some other variable and expect the result to be there for you in other code. Instead, you use it inside that callback or you call some other function from the callback and pass it the data.
Here's one way, you could change your sqlquery() function:
var sqlquery = function(uID, vs, callback){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
callback(null, data);
});
// need to add error handling here if the query returns an error
// by calling callback(err)
};
Then, you could use the sqlquery function like this:
found.forEach(function(person, index) {
sqlquery(..., function(err, result) {
if (err) {
// handle an error here
} else {
io.to(person).emit('notification', result);
}
});
});
And, it looks like you probably have similar async issues in other places too like in connection.connect().
In addition to #jfriend00, this could be done with new ES6 feature Promise :
var sqlquery = function(uID, vs){
return new Promise(function(resolve, reject){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
resolve(data);
});
});
};
Now you can use it like :
found.forEach(function(person, index) {
sqlquery(...)
.then(function(result){
io.to(person).emit('notification', result);
});
});