page renders before getting all the values sorted - node.js

I think the rendering takes place before the searching of the string on the files, i have tried different methods but don't seems to get this working. any help will be appreciated. im a noob on to the nodejs. im trying to get the id of the user and query and get all the data and there after see if he is in any of the lists given and finally render the page.
const j = [];
let name = '';
const filename = [];
var ext = '';
module.exports = function(app, express) {
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/cusdetails', isLoggedIn, function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
connection.query(insertQuerys,[cusid],
function(err, rows){
rows.forEach( (row) => {
name=row.fncus;
});
fs.readdir('./views/iplist', function(err, files) {
if (err)
throw err;
for (var index in files) {
j.push(files[index])
}
j.forEach(function(value) {
var k = require('path').resolve(__dirname, '../views/iplist/',value);
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) throw err;
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
}
});
}
else {
console.log(" FileNotExist ");
}
});
});
});
console.log(filename);
res.render('cusdetails.ejs', {rows: rows, user:req.user , aml: filename });
});
})

You can create simple Promise wrapper and then use it inside async/await function to pause execution until resolved.
// use mysql2 package as it provides promise, less work to write promise wrappers
const mysql = require('mysql2/promise');
// create the connection to database
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'test'
});
// sample wrapper
function some(k) {
// more advisable to have local variables, why do you need this to be array?
var filename = [];
return new Promise((resolve, reject) => {
// doing this is also not recommended check nodejs documentation **fs.exists** for more info
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) reject(err);
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
resolve(filename)
}
});
}
else {
// reject(new Error("FileNotExist"))
console.log(" FileNotExist ");
}
});
})
}
// note the use of async
app.post('/cusdetails', isLoggedIn, async function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
// using await to pause excution, waits till query is finished
const [rows] = await connection.query(insertQuerys,[cusid])
rows.forEach( (row) => {
name=row.fncus;
});
// then you can
var result = await some(k)
...
Note however this way you loose the advantage of concurrent execution, as it's kindoff blocking. If the result of one call is not used in another, you can execute in parallel and await for result to achieve sequencing like
const [rows] = connection.query(insertQuerys,[cusid])
var result = some(k)
console.log(await rows) // do something
console.log(await result) // do something

JavaScript is asynchronous. This means that if you have a function with a callback (i.e. your query), the callback will be called asynchronously, at an unknown time, while the other code executes.
You need to look up some tutorials how to deal with callbacks, to get a proper understanding of it. Another method is using async/await and/or promises.
Basically, if you take the following code:
console.log("this will print first");
setTimeout(function () {
console.log("this will print last");
}, 1000);
console.log("this will print second");
If you run the code above, the top level is executed synchronously, so, it first calls console.log, then it executes setTimeout, which is synchronous. It sets a timeout, then says "I'm ready", and the code continues to the other console.log. After 1 second (1000 milliseconds), the callback in the setTimeout function is executed, and only then that console.log is called. You can not make the rest of the code wait this way, you need to restructure your code or read into promises.

Related

How To Bind Node-js DB Query to Web Form

I'm using node and postgres, I'm new to writing async function, what I'm trying to do is a very simple query that will do a total count of records in the database, add one to it and return the result. The result will be visible before the DOM is generated. I don't know how to do this, since async function doesn't return value to callers (also probably I still have the synchronous mindset). Here's the function:
function generateRTA(callback){
var current_year = new Date().getFullYear();
const qry = `SELECT COUNT(date_part('year', updated_on))
FROM recruitment_process
WHERE date_part('year', updated_on) = $1;`
const value = [current_year]
pool.query(qry, value, (err, res) => {
if (err) {
console.log(err.stack)
} else {
var count = parseInt(res.rows[0].count) + 1
var rta_no = String(current_year) + '-' + count
callback(null, rta_no)
}
})
}
For the front-end I'm using pug with simple HTML form.
const rta_no = generateRTA(function (err, res){
if(err){
console.log(err)
}
else{
console.log(res)
}
})
app.get('/new_application', function(req, res){
res.render('new_application', {rta_number: rta_no})
});
I can see the rta_no in console.log but how do I pass it back to the DOM when the value is ready?
Based on the ajax call async response, it will update the div id "div1" when it gets the response from the Node js .
app.js
app.get("/webform", (req, res) => {
res.render("webform", {
title: "Render Web Form"
});
});
app.get("/new_application", (req, res) => {
// Connect to database.
var connection = getMySQLConnection();
connection.connect();
// Do the query to get data.
connection.query('SELECT count(1) as cnt FROM test ', function(err, rows, fields) {
var person;
if (err) {
res.status(500).json({"status_code": 500,"status_message": "internal server error"});
} else {
// Check if the result is found or not
if(rows.length==1) {
res.status(200).json({"count": rows[0].cnt});
} else {
// render not found page
res.status(404).json({"status_code":404, "status_message": "Not found"});
}
}
});
// Close connection
connection.end();
});
webform.pug - Via asynchronous call
html
head
script(src='https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js')
script.
$(document).ready(function(){
$.ajax({url: "/new_application", success: function(result){
$("#div1").html(result.count);
}});
});
body
div
Total count goes here :
#div1
value loading ...
That seems okay, I'm just not sure of this:
The result will be visible before the DOM is generated
This constraint defeats the purpose of async, as your DOM should wait for the returned value to be there. Instead of waiting for it you could just render the page and once the function returns and runs your callback update the value.
Also, perhaps it's worth having a look into promises

Mongoose Promise not populating in script

I have an api server and some script jobs. They are using the same function to pull a roster using mongoose and populate the players in the roster.
On the api server, this function is called normally. Using the script, it doesn't.
API example
function getRoster(id) {
var deferred = Q.defer();
Roster.find({_id:id}, 'playerRoster userId tournamentId').populate('playerRoster').exec(
function(err, roster) {
if (err) {
deferred.resolve(err);
}
deferred.resolve(roster[0]);
});
return deferred.promise;
}
api.post('/get_roster', function(req, res) {
// get tournament
var id = req.body._id;
var playerId = req.body.playerId;
getRoster(id).then(function(data) {
var roster=data;
res.json(roster);
});
});
Script
module.exports = function(config) {
this.getRoster=function(id) {
//return Roster.find({_id:id}, 'playerRoster userId tournamentId').exec( - THIS RETURNS
return Roster.find({_id:id}, 'playerRoster userId tournamentId').populate('playerRoster').exec(
function(err, roster) {
if (err) {
return err;
}
console.log('roster[0]',roster);
return roster[0];
});
}
this.tallyPoints = function(tournamentPlayer,sportsPlayers) {
var deferred = Q.defer();
var totalPoints =0;
console.log("tallyPoints 0 ",tournamentPlayer);
var rosterId = tournamentPlayer.player.roster[0];
console.log("tallyPoints 1 ",rosterId);
this.getRoster(rosterId).then(function(roster2){
console.log("tallyPoints 2 ",roster2);
...
deferred.resolve(totalPoints);
});
return deferred.promise;
}
return this;
};
In the script, neither logging for the roster[0] or tallyPoints 2 lines print, but there is no error either.
Why doesn't Roster.find return when I add populate? The only thing I can imagine is because playerRoster collection has 2000 records searching for ~10 and it hits some timeout that isn't being caught.
Any suggestion to clean it up is also appreciated.
Thanks
Moongoose supports promises for a long time. It's unsuitable to use callback-based Mongoose API where promises are desirable and the use of Q.defer with existing promises is known as Deferred antipattern (similarly, new Promise results in promise construction antipattern).
In its current state getRoster doesn't return a promise and doesn't handle errors correctly.
function getRoster(id) {
return Roster.find({_id:id}, 'playerRoster userId tournamentId').populate('playerRoster').exec()
.then(roster => roster[0]);
}
api.post('/get_roster', function(req, res) {
// get tournament
var id = req.body._id;
var playerId = req.body.playerId;
getRoster(id)
.then(function(data) {
var roster=data;
res.json(roster);
})
.catch(err => {
// handle error
});
});
Considering that only roster[0] is used, it likely should be changed to Roster.findOne.
It doesn't matter whether getRoster is used in Express route or elsewhere, it should work. It's unknown how module.exports = function(config) {...} module is used, but this may refer to wrong context if it isn't used as class. If getRoster and tallyPoints don't use config, they shouldn't reside inside this function.

How to properly return a result from mysql with Node?

In the code
var stuff_i_want = '';
stuff_i_want = get_info(parm);
And the function get_info:
get_info(data){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
console.log(stuff_i_want); // Yep. Value assigned..
}
in the larger scope
stuff_i_want = null
What am i missing regarding returning mysql data and assigning it to a variable?
============ New code per Alex suggestion
var parent_id = '';
get_info(data, cb){
var sql = "SELECT a from b where info = data"
connection.query(sql, function(err, results){
if (err){
throw err;
}
return cb(results[0].objid); // Scope is larger than function
}
==== New Code in Use
get_data(parent_recording, function(result){
parent_id = result;
console.log("Parent ID: " + parent_id); // Data is delivered
});
However
console.log("Parent ID: " + parent_id);
In the scope outside the function parent_id is null
You're going to need to get your head around asynchronous calls and callbacks with javascript, this isn't C#, PHP, etc...
Here's an example using your code:
function get_info(data, callback){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
return callback(results[0].objid);
})
}
//usage
var stuff_i_want = '';
get_info(parm, function(result){
stuff_i_want = result;
//rest of your code goes in here
});
When you call get_info this, in turn, calls connection.query, which takes a callback (that's what function(err, results) is
The scope is then passed to this callback, and so on.
Welcome to javascript callback hell...
It's easy when you get the hang of it, just takes a bit of getting used to, coming from something like C#
I guess what you really want to do here is returning a Promise object with the results. This way you can deal with the async operation of retrieving data from the DBMS: when you have the results, you make use of the Promise resolve function to somehow "return the value" / "resolve the promise".
Here's an example:
getEmployeeNames = function(){
return new Promise(function(resolve, reject){
connection.query(
"SELECT Name, Surname FROM Employee",
function(err, rows){
if(rows === undefined){
reject(new Error("Error rows is undefined"));
}else{
resolve(rows);
}
}
)}
)}
On the caller side, you use the then function to manage fulfillment, and the catch function to manage rejection.
Here's an example that makes use of the code above:
getEmployeeNames()
.then(function(results){
render(results)
})
.catch(function(err){
console.log("Promise rejection error: "+err);
})
At this point you can set up the view for your results (which are indeed returned as an array of objects):
render = function(results){ for (var i in results) console.log(results[i].Name) }
Edit
I'm adding a basic example on how to return HTML content with the results, which is a more typical scenario for Node. Just use the then function of the promise to set the HTTP response, and open your browser at http://localhost:3001
require('http').createServer( function(req, res){
if(req.method == 'GET'){
if(req.url == '/'){
res.setHeader('Content-type', 'text/html');
getEmployeeNames()
.then(function(results){
html = "<h2>"+results.length+" employees found</h2>"
html += "<ul>"
for (var i in results) html += "<li>" + results[i].Name + " " +results[i].Surname + "</li>";
html += "</ul>"
res.end(html);
})
.catch(function(err){
console.log("Promise rejection error: "+err);
res.end("<h1>ERROR</h1>")
})
}
}
}).listen(3001)
Five years later, I understand asynchronous operations much better.
Also with the new syntax of async/await in ES6 I refactored this particular piece of code:
const mysql = require('mysql2') // built-in promise functionality
const DB = process.env.DATABASE
const conn = mysql.createConnection(DB)
async function getInfo(data){
var sql = "SELECT a from b where info = data"
const results = await conn.promise().query(sql)
return results[0]
}
module.exports = {
getInfo
}
Then, where ever I need this data, I would wrap it in an async function, invoke getInfo(data) and use the results as needed.
This was a situation where I was inserting new records to a child table and needed the prent record key, based only on a name.
This was a good example of understanding the asynchronous nature of node.
I needed to wrap the all the code affecting the child records inside the call to find the parent record id.
I was approaching this from a sequential (PHP, JAVA) perspective, which was all wrong.
Easier if you send in a promise to be resolved
e.g
function get_info(data, promise){
var sql = "SELECT a from b where info = data";
connection.query(sql, function(err, results){
if (err){
throw err;
}
console.log(results[0].objid); // good
stuff_i_want = results[0].objid; // Scope is larger than function
promise.resolve(results[0].objid);
}
}
This way Node.js will stay fast because it's busy doing other things while your promise is waiting to be resolved
I've been working on this goal since few weeks, without any result, and I finally found a way to assign in a variable the result of any mysql query using await/async and promises.
You don't need to understand promises in order to use it, eh, I don't know how to use promises neither anyway
I'm doing it using a Model class for my database like this :
class DB {
constructor(db) {
this.db = db;
}
async getUsers() {
let query = "SELECT * FROM asimov_users";
return this.doQuery(query)
}
async getUserById(array) {
let query = "SELECT * FROM asimov_users WHERE id = ?";
return this.doQueryParams(query, array);
}
// CORE FUNCTIONS DON'T TOUCH
async doQuery(queryToDo) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
async doQueryParams(queryToDo, array) {
let pro = new Promise((resolve,reject) => {
let query = queryToDo;
this.db.query(query, array, function (err, result) {
if (err) throw err; // GESTION D'ERREURS
resolve(result);
});
})
return pro.then((val) => {
return val;
})
}
}
Then, you need to instantiate your class by passing in parameter to constructor the connection variable given by mysql. After this, all you need to do is calling one of your class methods with an await before. With this, you can chain queries without worrying of scopes.
Example :
connection.connect(function(err) {
if (err) throw err;
let DBModel = new DB(connection);
(async function() {
let oneUser = await DBModel.getUserById([1]);
let allUsers = await DBModel.getUsers();
res.render("index.ejs", {oneUser : oneUser, allUsers : allUsers});
})();
});
Notes :
if you need to do another query, you just have to write a new method in your class and calling it in your code with an await inside an async function, just copy/paste a method and modify it
there are two "core functions" in the class, doQuery and doQueryParams, the first one only takes a string as a parameter which basically is your mysql query. The second one is used for parameters in your query, it takes an array of values.
it's relevant to notice that the return value of your methods will always be an array of objects, it means that you'll have to do var[0] if you do a query which returns only one row. In case of multiple rows, just loop on it.

Nodejs with express jumps after 1 function is called directly to callback

i have a sample function in which i want to trigger 3 sql queries in one express post request
function getRelatedSalespersonByCardcode(req, res) {
var reqJson = JSON.parse(req.body.json);
var count = Object.keys(reqJson.cardcode).length;
var tmpResult1, tmpResult2, tmpResult3 = [];
var q = sql.open(connstr, function (err) {
if (err) {
console.log(err);
return;
}
for (var i = 0; i < count; i++) {
q.queryRaw("SELECT Division, Salesperson FROM SomeDB.dbo.MS2_Rel_BusinessPartnerSalesperson WHERE CardCode = " + reqJson.cardcode[i], function (e, results) {
if (e) {
console.log(e);
return;
}
tmpResult1.push(results);
});
q.queryRaw("SELECT SlpCode, SlpName, Memo, Commission, GroupCode, UserSign, Active, U_wpABIS, U_sweDW," +
" U_sweATT, U_sweDIV, U_sweEMPLOYEE, U_sweRETAILER FROM SomeDB.dbo.OSLP WHERE U_sweId = " + tmpResult1[1], function (e, results) {
if (e) {
console.log(e);
return;
}
tmpResult2.push(results);
});
q.queryRaw("SELECT Code, Name, U_sweSALES FROM SomeDB.dbo.[#SWEDIV] WHERE Code = " + tmpResult3[0], function (e, results) {
if (e) {
console.log(e);
return;
}
tmpResult3.push(results);
});
}
});
res.send(200, tmpResult2);
}
Anyway after 1 function is called inside my req, res function the callback is triggert.. so it jumps directly to the res.send(...) line.
i've played a bit around and it seems that this is how express works.
after a bit of googleing around i found out that i have to use the async lib.
i'd like to ask why express is working like this and if anybody maybe have a better solution than the async approach. i simply need a way to realize my scenario.
This is no issue of express and no issue at all. This is like node.js async programming is working. Let's examine what's happening in your code:
var tmpResult2;
// 1. async open sql connection
var q = sql.open(connstr, function (err) {
// 3. query sql connection
});
// 2. render result
res.send(200, tmpResult2);
SQL connection is opened asynchronously.
Express render is called.
SQL queries are executed.
As a consequence this code piece sends data to the client before the data was fetched. So the most simple solution is to invoke res.send inside of the callback like this:
var tmpResult2;
// 1. async open sql connection
var q = sql.open(connstr, function (err) {
// 2. render result
res.send(200, tmpResult2);
});
You can use the wonderful async module to deal with the queries (Take a look at the async waterfall function).

NodeJS and MongoDB send response when finish all [duplicate]

I want to drop some mongodb collections, but that's an asynchronous task. The code will be:
var mongoose = require('mongoose');
mongoose.connect('mongo://localhost/xxx');
var conn = mongoose.connection;
['aaa','bbb','ccc'].forEach(function(name){
conn.collection(name).drop(function(err) {
console.log('dropped');
});
});
console.log('all dropped');
The console displays:
all dropped
dropped
dropped
dropped
What is the simplest way to make sure all dropped will be printed after all collections has been dropped? Any 3rd-party can be used to simplify the code.
Use Promises.
var mongoose = require('mongoose');
mongoose.connect('your MongoDB connection string');
var conn = mongoose.connection;
var promises = ['aaa', 'bbb', 'ccc'].map(function(name) {
return new Promise(function(resolve, reject) {
var collection = conn.collection(name);
collection.drop(function(err) {
if (err) { return reject(err); }
console.log('dropped ' + name);
resolve();
});
});
});
Promise.all(promises)
.then(function() { console.log('all dropped)'); })
.catch(console.error);
This drops each collection, printing “dropped” after each one, and then prints “all dropped” when complete. If an error occurs, it is displayed to stderr.
Previous answer (this pre-dates Node’s native support for Promises):
Use Q promises or Bluebird promises.
With Q:
var Q = require('q');
var mongoose = require('mongoose');
mongoose.connect('your MongoDB connection string');
var conn = mongoose.connection;
var promises = ['aaa','bbb','ccc'].map(function(name){
var collection = conn.collection(name);
return Q.ninvoke(collection, 'drop')
.then(function() { console.log('dropped ' + name); });
});
Q.all(promises)
.then(function() { console.log('all dropped'); })
.fail(console.error);
With Bluebird:
var Promise = require('bluebird');
var mongoose = Promise.promisifyAll(require('mongoose'));
mongoose.connect('your MongoDB connection string');
var conn = mongoose.connection;
var promises = ['aaa', 'bbb', 'ccc'].map(function(name) {
return conn.collection(name).dropAsync().then(function() {
console.log('dropped ' + name);
});
});
Promise.all(promises)
.then(function() { console.log('all dropped'); })
.error(console.error);
I see you are using mongoose so you are talking about server-side JavaScript. In that case I advice looking at async module and use async.parallel(...). You will find this module really helpful - it was developed to solve the problem you are struggling with. Your code may look like this
var async = require('async');
var calls = [];
['aaa','bbb','ccc'].forEach(function(name){
calls.push(function(callback) {
conn.collection(name).drop(function(err) {
if (err)
return callback(err);
console.log('dropped');
callback(null, name);
});
}
)});
async.parallel(calls, function(err, result) {
/* this code will run after all calls finished the job or
when any of the calls passes an error */
if (err)
return console.log(err);
console.log(result);
});
The way to do it is to pass the tasks a callback that updates a shared counter. When the shared counter reaches zero you know that all tasks have finished so you can continue with your normal flow.
var ntasks_left_to_go = 4;
var callback = function(){
ntasks_left_to_go -= 1;
if(ntasks_left_to_go <= 0){
console.log('All tasks have completed. Do your stuff');
}
}
task1(callback);
task2(callback);
task3(callback);
task4(callback);
Of course, there are many ways to make this kind of code more generic or reusable and any of the many async programing libraries out there should have at least one function to do this kind of thing.
Expanding upon #freakish answer, async also offers a each method, which seems especially suited for your case:
var async = require('async');
async.each(['aaa','bbb','ccc'], function(name, callback) {
conn.collection(name).drop( callback );
}, function(err) {
if( err ) { return console.log(err); }
console.log('all dropped');
});
IMHO, this makes the code both more efficient and more legible. I've taken the liberty of removing the console.log('dropped') - if you want it, use this instead:
var async = require('async');
async.each(['aaa','bbb','ccc'], function(name, callback) {
// if you really want the console.log( 'dropped' ),
// replace the 'callback' here with an anonymous function
conn.collection(name).drop( function(err) {
if( err ) { return callback(err); }
console.log('dropped');
callback()
});
}, function(err) {
if( err ) { return console.log(err); }
console.log('all dropped');
});
I do this without external libaries:
var yourArray = ['aaa','bbb','ccc'];
var counter = [];
yourArray.forEach(function(name){
conn.collection(name).drop(function(err) {
counter.push(true);
console.log('dropped');
if(counter.length === yourArray.length){
console.log('all dropped');
}
});
});
All answers are quite old. Since the beginning of 2013 Mongoose started to support promises gradually for all queries, so that would be the recommended way of structuring several async calls in the required order going forward I guess.
With deferred (another promise/deferred implementation) you can do:
// Setup 'pdrop', promise version of 'drop' method
var deferred = require('deferred');
mongoose.Collection.prototype.pdrop =
deferred.promisify(mongoose.Collection.prototype.drop);
// Drop collections:
deferred.map(['aaa','bbb','ccc'], function(name){
return conn.collection(name).pdrop()(function () {
console.log("dropped");
});
}).end(function () {
console.log("all dropped");
}, null);
If you are using Babel or such transpilers and using async/await you could do :
function onDrop() {
console.log("dropped");
}
async function dropAll( collections ) {
const drops = collections.map(col => conn.collection(col).drop(onDrop) );
await drops;
console.log("all dropped");
}

Resources