I have a problem in a nodeJS app with mongoDB, i'm trying to do a forum and for each topic i want a button to display every sub topics.
So i need to get everything in the request:
One array with main topics
Another map array with ['topic1'] containing sub topics
Without the mapping (not an actual problem) i have this:
Post.find({'path': path})
.exec(function (err, posts){
if(err)
console.log("Get post list:" + err);
else
{
var sub_posts = new Array; // Second array with sub topics
for (var i = 0; posts[i]; i++) //Here is the loop for each topic
{
var tmp_path = ... // Path and next request works
Post.find({'path': tmp_path}) // Here is the second request
.exec(function(err, bis_posts) {
if (err) console.log('Error loading subforum');
else sub_posts.push(bis_posts); // Affectation fail !!!
})
}
res.render(... 'post_list': posts, 'sub_posts': sub_posts); // Send result
}
})}
So i get it's a scope problem and i should use callback but with the loop i can't resolve this problem.
Sorry for my english and thanks for your answers !
I have no idea what you mean by "affectation fail", but it looks like you're calling res.render too early — callbacks are invoked asynchronously after your current context finishes executing, so when you call res.render(...) after your for loop has finished, your Post.find(...)... operations still haven't finished and their callbacks haven't been invoked, so sub_posts will be still empty.
My node.js and Mongo are rusty, so perhaps this isn't the canonical way to do it, but I'd add a counter to track the state of the pending requests and only call res.render when all subposts have been fetched:
var sub_posts = new Array;
var pending = 0;
for (var i = 0; posts[i]; i++)
{
var tmp_path = ...
Post.find({'path': tmp_path})
.exec(function(err, bis_posts) {
if (err) console.log('Error loading subforum');
else sub_posts.push(bis_posts);
pending -= 1;
if (!pending) {
// all pending subpost lookups finished, render the response:
res.render(... 'post_list': posts, 'sub_posts': sub_posts);
}
});
pending += 1;
}
Related
I am new in node js / express I am trying to query, foreach id in an array, some data from the sqlite3 database. My for loop looks like as follows
response.patients = results;
for (var i = 0; i < results.length; i++) {
response.patients[i].check = "false";
var patient = response.patients[i];
db.each("SELECT visit_id FROM patient_visits where patient_id='"+patient.id+"' AND visitdate >='"+moment().format('YYYY-MM-DD')+"'", function(err, row) {
if (row) {
response.patients[i].check = "true";
}
});
}
res.send(response);
the problem is that the for loop continues before the query is finished. Is there a way to check if the query has finished?
I only need to set a flag true / false.
The problem is that db.each is a callback function. This function executes and the response takes some time to be retrieved, however the system doesn't stop and the for loop continues. When it ends, res.send is called while the responses from the db.each are still being processed.
You can try
db.all(
SELECT visit_id FROM patient_visits where patient_id IN ?
AND visitdate >='"+moment().format('YYYY-MM-DD')+"'", response.patients,
function(err, rows) {
if (err) {
throw err;
}
rows.forEach((row) => {
response.patients[row.visit_id].check = "true";
});
res.send(response);
});
I'm not sure about the content of row variable to get the visit_id but before print the variable and check if it works.
If you have any doubt let me know. And I strongly advise you to read some content about callbacks and how they work and then promises and async/await because callbacks aren't used now, with async await, your problem would be solved easily.
This is some of my code that I have in my index.js. Its waiting for the person to visit url.com/proxy and then it loads up my proxy page, which is really just a form which sends back an email and a code. From my MongoDB database, I grab the users order using the code, which contains some information I need (like product and the message they're trying to get). For some reason, it seems like its responding before it gets this information and then holds onto it for the next time the form is submitted.
The newline in my res.send(product + '\n' + message) isnt working either, but thats not a big deal right now.
But.. for example, the first time I fill out the form ill get a blank response. The second time, I'll get the response to whatever I filled in for the first form, and then the third time ill get the second response. I'm fairly new to Web Development, and feel like I'm doing something obviously wrong but can't seem to figure it out. Any help would be appreciated, thank you.
app.get('/proxy', function(req,res){
res.sendFile(__dirname+ "/views/proxy.html");
});
var message = "";
var product = "";
app.post('/getMessage', function(req,res)
{
returnMsg(req.body.user.code, req.body.user.email);
//res.setHeader('Content-Type', 'text/plain');
res.send(product + "\n" + message);
});
function returnMsg(code, email){
MongoClient.connect(url, function(err, db){
var cursor = db.collection('Orders').find( { "order_id" : Number(code) })
cursor.each(function(err, doc){
assert.equal(err, null);
if (doc!= null)
{
message = doc["message"];
product = doc["product"];
}
else {
console.log("wtf");
// error code here
}
});
console.log(email + " + " + message);
var document = {
"Email" : email,
"Message" : message
}
db.collection("Users").insertOne(document);
db.close();
});
}
You need to do lots of reading about your asynchronous programming works in node.js. There are significant design problems with this code:
You are using module level variables instead of request-level variables.
You are not correctly handling asynchronous responses.
All of this makes a server that simply does not work correctly. You've found one of the problems already. Your async response finishes AFTER you send your response so you end up sending the previously saved response not the current one. In addition, if multiple users are using your server, their responses will tromp on each other.
The core design principle here is first that you need to learn how to program with asynchronous operations. Any function that uses an asynchronous respons and wants to return that value back to the caller needs to accept a callback and deliver the async value via the callback or return a promise and return the value via a resolved promise. The caller then needs to use that callback or promise to fetch the async value when it is available and only send the response then.
In addition, all data associated with a request needs to stay "inside" the request handle or the request object - not in any module level or global variables. That keeps the request from one user from interfering with the requests from another user.
To understand how to return a value from a function with an asynchronous operation in it, see How do I return the response from an asynchronous call?.
What ends up happening in your code is this sequence of events:
Incoming request for /getMessage
You call returnMsg()
returnMsg initiates a connection to the database and then returns
Your request handler calls res.send() with whatever was previously in the message and product variables.
Then, sometime later, the database connect finishes and you call db.collection().find() and then iterate the cursor.
6/ Some time later, the cursor iteration has the first result which you put into your message and product variables (where those values sit until the next request comes in).
In working out how your code should actually work, there are some things about your logic that are unclear. You are assigning message and product inside of cursor.each(). Since cursor.each() is a loop that can run many iterations, which value of message and product do you actually want to use in the res.send()?
Assuming you want the last message and product value from your cursor.each() loop, you could do this:
app.post('/getMessage', function(req, res) {
returnMsg(req.body.user.code, req.body.user.email, function(err, message, product) {
if (err) {
// send some meaningful error response
res.status(500).end();
} else {
res.send(product + "\n" + message);
}
});
});
function returnMsg(code, email, callback) {
let callbackCalled = false;
MongoClient.connect(url, function(err, db) {
if (err) {
return callback(err);
}
var cursor = db.collection('Orders').find({
"order_id": Number(code)
});
var message = "";
var product = "";
cursor.each(function(err, doc) {
if (err) {
if (!callbackCalled) {
callback(err);
callbackCalled = true;
}
} else {
if (doc != null) {
message = doc["message"];
product = doc["product"];
} else {
console.log("wtf");
// error code here
}
}
});
if (message) {
console.log(email + " + " + message);
var document = {
"Email": email,
"Message": message
}
db.collection("Users").insertOne(document);
}
db.close();
if (!callbackCalled) {
callback(null, message, product);
}
});
}
Personally, I would use promises and use the promise interface in your database rather than callbacks.
This code is still just conceptual because it has other issues you need to deal with such as:
Proper error handling is still largely unfinished.
You aren't actually waiting for things like the insert.One() to finish before proceeding.
Here is the situation:
I am new to node.js, I have a 40MB file containing multilevel json file like:
[{},{},{}] This is an array of objects (~7000 objects). Each object has properties and a one of those properties is also an array of objects
I wrote a function to read the content of the file and iterate it. I succeeded to get what I wanted in terms of content but not usability. I thought that I wrote an async function that would allow node to serve other web requests while iterating the array but that is not the case. I would be very thankful if anyone can point me to what I've done wrong and how to rewrite it so I can have a non-blocking iteration. Here's the function that handles the situation:
function getContents(callback) {
fs.readFile(file, 'utf8', function (err, data) {
if (err) {
console.log('Error: ' + err);
return;
}
js = JSON.parse(data);
callback();
return;
});
}
getContents(iterateGlobalArr);
var count = 0;
function iterateGlobalArr() {
if (count < js.length) {
innerArr = js.nestedProp;
//iterate nutrients
innerArr.forEach(function(e, index) {
//some simple if condition here
});
var schema = {
//.....get props from forEach iteration
}
Model.create(schema, function(err, post) {
if(err) {
console.log('\ncreation error\n', err);
return;
}
if (!post) {
console.log('\nfailed to create post for schema:\n' + schema);
return;
}
});
count++;
process.nextTick(iterateGlobalArr);
}
else {
console.log("\nIteration finished");
next();
}
Just so it is clear how I've tested the above situation. I open two tabs one loading this iteration which takes some time and second with another node route which does not load until the iteration is over. So essentially I've written a blocking code but not sure how to re-factor it! I suspect that just because everything is happening in the callback I am unable to release the event loop to handle another request...
Your code is almost correct. What you are doing is inadvertently adding ALL the items to the very next tick... which still blocks.
The important piece of code is here:
Model.create(schema, function(err, post) {
if(err) {
console.log('\ncreation error\n', err);
return;
}
if (!post) {
console.log('\nfailed to create post for schema:\n' + schema);
return;
}
});
// add EVERYTHING to the very same next tick!
count++;
process.nextTick(iterateGlobalArr);
Let's say you are in tick A of the event loop when getContents() runs and count is 0. You enter iterateGlobalArr and you call Model.create. Because Model.create is async, it is returning immediately, causing process.nextTick() to add processing of item 1 to the next tick, let's say B. Then it calls iterateGlobalArr, which does the same thing, adding item 2 to the next tick, which is still B. Then item 3, and so on.
What you need to do is move the count increment and process.nextTick() into the callback of Model.create(). This will make sure the current item is processed before nextTick is invoked... which means next item is actually added to the next tick AFTER the model item has been created... which will give your app time to handle other things in between. The fixed version of iterateGlobalArr is here:
function iterateGlobalArr() {
if (count < js.length) {
innerArr = js.nestedProp;
//iterate nutrients
innerArr.forEach(function(e, index) {
//some simple if condition here
});
var schema = {
//.....get props from forEach iteration
}
Model.create(schema, function(err, post) {
// schedule our next item to be processed immediately.
count++;
process.nextTick(iterateGlobalArr);
// then move on to handling this result.
if(err) {
console.log('\ncreation error\n', err);
return;
}
if (!post) {
console.log('\nfailed to create post for schema:\n' + schema);
return;
}
});
}
else {
console.log("\nIteration finished");
next();
}
}
Note also that I would strongly suggest that you pass in your js and counter with each call to iterageGlobalArr, as it will make your iterateGlobalArr alot easier to debug, among other things, but that's another story.
Cheers!
Node is single-threaded so async will only help you if you are relying on another system/subsystem to do the work (a shell script, external database, web service etc). If you have to do the work in Node you are going to block while you do it.
It is possible to create one node process per core. This solution would result in only blocking one of the node processes and leave the rest to service your requests, but this feature is still listed as experimental http://nodejs.org/api/cluster.html.
A single instance of Node runs in a single thread. To take advantage
of multi-core systems the user will sometimes want to launch a cluster
of Node processes to handle the load.
The cluster module allows you to easily create child processes that
all share server ports.
First post here, so apologies if I get things wrong...
I'm creating some standalone code to read a folder structure and return all .mp3 files in an Array. Once this has been returned I then loop through the array and for each item I create a Mongoose Object and populate the fields, before saving the object with .save()
I am looping through the array using async.forEach - and while it does loop through all items in the Array they do not save, and there is no error produced to help me identify what is wrong.
If I move the logic of the loop elsewhere then the MP3s are stored in the mongodb database - if I have the example shown nothing is saved.
var saveMP3s = function(MP3Files, callback) {
console.log('start loop -> saving MP3s');
async.forEach( MP3Files, function(mp3file, callback) {
newTrack = new MP3Track();
newTrack.title = mp3file.title;
newTrack.track = mp3file.track;
newTrack.disk = mp3file.disk;
newTrack.metadata = mp3file.metadata;
newTrack.path = mp3file.path;
console.log('....:> Song Start: ');
console.log(newTrack.title);
console.log(newTrack.track);
console.log(newTrack.disk);
console.log(newTrack.metadata);
console.log(newTrack.path);
console.log('....:> Song End: ');
newTrack.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('saving Track: '+ newTrack.title);
callback();
}
});
}, function(err) {
if (err) { console.log(err); }
});
console.log('end loop -> finished saving MP3s');
};
The trouble I have is that when the code is NOT in the async loop the code works and the MP3 is saved in the MongoDB database, inside the async code nothing is saved and no errors are given as to why.
I did try (in an earlier incarnation of the code) create the objects once I had read the metadata of the MP3 files - but for some reason it would not save the last 2 objects in the list (of 12)... So I've rewritten it to scan all the items first and then populate mongoDB using mongoose from an array; just to split things up. But having no luck in finding out why nothing happens and why there are no errors on the .save()
Any help with be greatly appreciated.
Regards,
Mark
var saveMP3s = function(MP3Files, callback) {
console.log('start loop -> saving MP3s');
async.forEach( MP3Files, function(mp3file, callback) {
var newTrack = new MP3Track(); // <--- USE VAR HERE
newTrack.title = mp3file.title;
newTrack.track = mp3file.track;
newTrack.disk = mp3file.disk;
newTrack.metadata = mp3file.metadata;
newTrack.path = mp3file.path;
console.log('....:> Song Start: ');
console.log(newTrack.title);
console.log(newTrack.track);
console.log(newTrack.disk);
console.log(newTrack.metadata);
console.log(newTrack.path);
console.log('....:> Song End: ');
newTrack.save(function (err) {
if (err) {
console.log(err);
} else {
console.log('saving Track: '+ newTrack.title);
callback();
}
});
}, function(err) {
if (err) { console.log(err); }
});
console.log('end loop -> finished saving MP3s');
};
I suspect that the missing var keyword is declaring a global variable and you are overriding it on each iteration of the loop. Meaning, before the first newTrack can complete it's async save operation, you've already moved on in the loop and overridden that variable with the next instance.
Also, in async.forEach you MUST invoke the callback when the operation is completed. You are only calling it if the record saves successfully. You should also be calling it if an error occurs and pass in the error.
Finally, the callback argument to your saveMP3s function never gets called at all. The call to callback() inside the newTrack.save function only refers to the callback argument passed into the anonymous function by async.forEach.
SOLVED
Hi,
I ended up solving it by having the async loop higher up the in the logic so the saveMP3s function was called from within the async loop itself.
Thank you all for your thoughts and suggestions.
Enjoy your day.
mark
(not allowed to Accept my answer until tomorrow, so will update the question status then)
I have the following code and am attempting to make a Turntable bot using node.js. this piece of code says when a user types "q+" we have to make sure its not already on the queue, that its not already DJing, and if it meets those 2 requirements, add them to the queue. Otherwise if it doesnt meet one of those first 2 criteria, tell the user and do not touch the queue.
My problem is the "isCurrentDJ(userId)". When I pass a userId through that function, the function gives me the correct answer. However the function ALWAYS passes back "false" even when the answer is "true" and the console.log() function within the isCurrentDJ(userId) function proves so.
I am not the most js-savvy person, so I think this may be a variable scope issue. But I am really not sure and have been struggling with it for hours! Any help would be greatly appreciated. Thanks!
// When someone speaks, listen to see if it is one of the q commands
bot.on('speak', function (data) {
var name = data.name;
var text = data.text;
var userId = data.userid;
// q+ :: Add to Queue
if (text.match(/^q\+$/)) {
//Index is the position in the queue that this person's name is found.
//If its not found, -1 is returned.
var index = queue.indexOf(name);
//Function to determine if the user is currently a DJ
function isCurrentDJ(user_id, callback){
bot.roomInfo(false, function (data) {
var djList = data.room.metadata.djs;
for (i = 0; i < djList.length; i++){
if (djList[i] == user_id){
console.log('recognized as a DJ'); //Consistently printed!
callback(true);
}
}
callback(false);
});
}
isCurrentDJ(userId, function(isDJ) {
//If the user is already in the queue
if(index > -1){
//Tell them they are already in there
bot.speak('You are already on the list');
} else if(isDJ){
//Otherwise if they are already a DJ tell them that
bot.speak('You are already a DJ, '+name);
}else{
//Otherise if they are not in the queue add user to end of queue
queue.push(name);
//Tell them about it and the updated q
bot.speak(name+' has been added to queue.');
}
});
}
Your problem is that bot.roomInfo is an asynchronous function.
When you call it, it immediately returns and currDJ is still false. A little while later, the callback (function(data) {...) is called. Most of node.js's API are async so that your code never blocks.
Here's how you should rewrite your code:
// When someone speaks, listen to see if it is one of the q commands
bot.on('speak', function (data) {
var name = data.name;
var text = data.text;
var userId = data.userid;
// q+ :: Add to Queue
if (text.match(/^q\+$/)) {
//Index is the position in the queue that this person's name is found.
//If its not found, -1 is returned.
var index = queue.indexOf(name);
//Function to determine if the user is currently a DJ
function testCurrentDJ(user_id, cb){
bot.roomInfo(false, function (data) {
var djList = data.room.metadata.djs;
for (i = 0; i < djList.length; i++){
if (djList[i] == user_id){
console.log('recognized as a DJ'); //Consistently printed!
return cb(true);
}
}
cb(false);
});
}
//If the user is already in the queue
if(index > -1){
//Tell them they are already in there
bot.speak('You are already on the list');
return;
}
testCurrentDJ(userId, function(isDJ) {
//Otherwise if they are already a DJ tell them that
if(isDJ) {
bot.speak('You are already a DJ, '+name);
} else {
//Otherise if they are not in the queue add user to end of queue
queue.push(name);
//Tell them about it and the updated q
bot.speak(name+' has been added to queue. Is Current DJ? '+isDJ);
}
})
}
I've just updated your code to show you the basic idea. In node.js's API, the first argument of callbacks is usually an error object that is null if everything went fine.
Is bot.roomInfo perhaps an asynchronous function? If so the value of currDJ will be set to true, but too late because you've already returned it. You can no operate on the value of currDJ untill that callback is called.
How familiar are you with the concept of asynchronous functions?