Sequelizejs model create callback is not running synchronously? - node.js

I'm using Sequelize and PostgreSQL in my node.js application. I have two tables with one-to-one relationship - Customers and Users.
There is foreign key UserId in the Customers table. So, I firstly insert into the Users table and then insert into the Customers with the last inserted UserId. Here is my controller code:
var self = this;
async.each(data, function(row, callback) {
var userData = {
name: row.name,
/** **/
}
// self.User is User model
self.User.create(userData).then(function(user) {
console.log("[User.create] succeeded");
/** **/
// self.Model is Customer model
self.Model.create(cusData).then(function(customer) {
console.log("[Customer.create] succeeded");
/** **/
}).catch(function(err) {
throw err;
}); // EOL self.Model.create
}).catch(function(err) {
throw err;
}); // EOL self.User.create
callback();
}, function(err){
if (err) {
throw err;
}
}
I'm using async.each() to loop the array of 2 records synchronously. When I inserted the two records, the console output is:
[User.create] succeeded
[User.create] succeeded
[Customer.create] succeeded
[Customer.create] succeeded
What I expected is:
[User.create] succeeded
[Customer.create] succeeded
[User.create] succeeded
[Customer.create] succeeded
I think it could be a problem of synchronous flow in asynchronous programming. What am I wrong? I think I'm using the callbacks correctly.

You're initiating the two inserts asynchronously, so you've no guarantee on the order of the follow-up queries. For all you know, depending on locks, you could get the current or your expected result.
To force the order, move the iteration forward from within the inner callback:
self.Model.create(cusData).then(function(customer) {
console.log("[Customer.create] succeeded");
/** move the iterator forward here, using e.g. recursion **/
})

Related

"before all" hook randomly showing up in my tests

I'm currently running a stack that consists of Express and MongoClient with Mocha and Chai for testing. I'm working on writing test cases for my endpoint and am getting a random error that pops up from time to time. Below is a snippet of one of the suits I'm writing:
describe('Recipes with populated database', () => {
before((done) => {
var recipe1 = {"search_name": "mikes_mac_and_cheese", "text_friendly_name": "Mikes Mac and Cheese","ingredients": [{"name": "elbow_noodles","text_friendly_name": "elbow noodles","quantity": 12,"measurement": "oz"},{"name": "cheddar_cheese","text_friendly_name": "cheddar cheese","quantity": 6,"measurement": "oz"},{"name": "gouda_cheese","text_friendly_name": "gouda cheese","quantity": 6,"measurement": "oz"},{"name": "milk","text_friendly_name": "milk","quantity": 2,"measurement": "oz"}],"steps": ["Bring water to a boil","Cook noodels until al dente.","Add the milk and cheeses and melt down.","Stir constantly to ensure even coating and serve."],"course": ["dinner","lunch","side"],"prep_time": {"minutes": 15,"hours": 0},"cook_time":{"minutes": 25,"hours": 1},"cuisine": "italian","submitted_by": "User1","searchable": true};
db.collectionExists('recipes').then((exists) => {
if (exists) {
db.getDb().dropCollection('recipes', (err, results) => {
if (err)
{
throw err;
}
});
}
db.getDb().createCollection('recipes', (err, results) => {
if (err)
{
throw err;
}
});
db.getDb().collection('recipes').insertOne(recipe1, (err, result) => {
done();
});
});
});
The collectionExists() method simply takes in a name and returns a promise that is resolved to a true/false value. I've already done some debugging and it is working just fine. Where I am getting a problem is when I hit the section of the code where I call createCollection. I get an error about how the collection already exists thus leading to my tests failing. This appears to be happening on every third time I'm running my tests as well.
The purpose of all this is to ensure that my database collection called recipes is completely empty before I start testing so I'm not stuck with old data or in an uncontrolled environment.
You have a race condition between .createCollection and .insertOne. In other words, they start at the same time and go in parallel. There is no way to tell which will be done first.
The way .insert works in MongoDB is that if the collection is missing and you try inserting - it's going to create a collection. So if .insertOne is executed first - the collection is created and that is why you're getting the already exists error in an attempt to createCollection.
Due to the async nature of DB calls you'd have to place the subsequent calls inside the callback of a prev. one. This way there will be no parallel execution:
before((done) => {
var recipe1 = {/**/};
db.collectionExists('recipes')
.then((exists) => {
if (exists) {
// Drop the collection if it exists.
db.getDb().dropCollection('recipes', (err) => {
if (err) {
// If there's an error we want to pass it up to before.
done(err);
return;
}
// Just insert a first document into a non-existent collection.
// It's going to be created.
// Notice the done callback.
db.getDb().collection('recipes').insertOne(recipe1, done);
});
}
// If there were no such collection - simply insert the first doc to create it.
// Note that I'm passing before's done callback inside.
db.getDb().collection('recipes').insertOne(recipe1, done);
})
// We don't want to lose the error from this promise always.
.catch(err => done(err));
});
But. Actually, there is no need to drop and re-create a collection each time you run the tests. You can simply .remove all the objects in the before block. So probably the right solution would be:
before((done) => {
var recipe1 = {/**/};
const recipes = db.getDb().collection('recipes');
// Simply wipe out the data
recipes.remove({}, err => {
if (err) {
done(err);
return;
}
recipes.insertOne(recipe1, done);
});
});

What is the best way to query mongodb with mongoose with an array of IDs?

I query one collection (messages) with mongoose. The result is an array of documents. Each document contains an ID for a different collection (users). Now I want to query the users collection for each ID from the messages collection.
The idea is to update each message object with the information from the user collection before returning it to the front end.
I tried using async.each. For some reason the final function is never called even though I am making sure the callback() function is called after each iteration.
app.get('/getmsg', function(req, res){
messages.find({query})
.exec(function(err, ms){
if(err) throw err;
async.each(ms, function(m, callback){
users.findOne({_id : m.userId})
.lean()
.exec(function(err, user){
if(err) {
console.log('error' , err);
callback();
} else {
m.userName = user.name;
// everything is working up to here
callback();
}
}), function(err){
res.send(ms); // this is never returned!
}
});
});
});
Is there a better way of achieving this? I assume this must be a common issue.
Thanks!
You can't use res.send. Instead create a function to get notified about it. Something like this.
// 1st para in async.each() is the array of items
async.each(items,
// 2nd param is the function that each item is passed to
function(item, callback){
// Call an asynchronous function, often a save() to DB
item.someAsyncCall(function (){
// Async call is done, alert via callback
callback();
});
},
// 3rd param is the function to call when everything's done
function(err){
// All tasks are done now
doSomethingOnceAllAreDone();
}
);

nodejs pg transactions without nesting

I would like to know if it's possible to run a series of SQL statements and have them all committed in a single transaction.
The scenario I am looking at is where an array has a series of values that I wish to insert into a table, not individually but as a unit.
I was looking at the following item which provides a framework for transactions in node using pg. The individual transactions appear to be nested within one another so I am unsure of how this would work with an array containing a variable number of elements.
https://github.com/brianc/node-postgres/wiki/Transactions
var pg = require('pg');
var rollback = function(client, done) {
client.query('ROLLBACK', function(err) {
//if there was a problem rolling back the query
//something is seriously messed up. Return the error
//to the done function to close & remove this client from
//the pool. If you leave a client in the pool with an unaborted
//transaction weird, hard to diagnose problems might happen.
return done(err);
});
};
pg.connect(function(err, client, done) {
if(err) throw err;
client.query('BEGIN', function(err) {
if(err) return rollback(client, done);
//as long as we do not call the `done` callback we can do
//whatever we want...the client is ours until we call `done`
//on the flip side, if you do call `done` before either COMMIT or ROLLBACK
//what you are doing is returning a client back to the pool while it
//is in the middle of a transaction.
//Returning a client while its in the middle of a transaction
//will lead to weird & hard to diagnose errors.
process.nextTick(function() {
var text = 'INSERT INTO account(money) VALUES($1) WHERE id = $2';
client.query(text, [100, 1], function(err) {
if(err) return rollback(client, done);
client.query(text, [-100, 2], function(err) {
if(err) return rollback(client, done);
client.query('COMMIT', done);
});
});
});
});
});
My array logic is:
banking.forEach(function(batch){
client.query(text, [batch.amount, batch.id], function(err, result);
}
pg-promise offers a very flexible support for transactions. See Transactions.
It also supports partial nested transactions, aka savepoints.
The library implements transactions automatically, which is what should be used these days, because too many things can go wrong, if you try organizing a transaction manually as you do in your example.
See a related question: Optional INSERT statement in a transaction
Here's a simple TypeScript solution to avoid pg-promise
import { PoolClient } from "pg"
import { pool } from "../database"
const tx = async (callback: (client: PoolClient) => void) => {
const client = await pool.connect();
try {
await client.query('BEGIN')
try {
await callback(client)
await client.query('COMMIT')
} catch (e) {
await client.query('ROLLBACK')
}
} finally {
client.release()
}
}
export { tx }
Usage:
...
let result;
await tx(async client => {
const { rows } = await client.query<{ cnt: string }>('SELECT COUNT(*) AS cnt FROM users WHERE username = $1', [username]);
result = parseInt(rows[0].cnt) > 0;
});
return result;

How to handle async concurrent requests correctly?

Let's say I have some sort of game. I have a buyItem function like this:
buyItem: function (req, res) {
// query the users balance
// deduct user balance
// buy the item
}
If I spam that route until the user balance is deducted (the 2nd query) the user's balance is still positive.
What I have tried:
buyItem: function (req, res) {
if(req.session.user.busy) return false;
req.session.user.busy = true;
// query the users balance
// deduct user balance
// buy the item
}
The problem is req.session.user.busy will be undefined for the first ~5 requests. So that doesn't work either.
How do we handle such situations? I'm using the Sails.JS framework if that is important.
Update 2
Sails 1.0 now has full transaction support, via the .getDatastore() method. Example:
// Get a reference to the default datastore, and start a transaction.
await sails.getDatastore().transaction(async (db, proceed)=> {
// Now that we have a connection instance in `db`, pass it to Waterline
// methods using `.usingConnection()` to make them part of the transaction:
await BankAccount.update({ balance: 5000 }).usingConnection(db);
// If an error is thrown, the transaction will be rolled back.
// Or, you can catch errors yourself and call `proceed(err)`.
// To commit the transaction, call `proceed()`
return proceed();
// You can also return a result with `proceed(null, result)`.
});
Update
As several commenters have noted, the code below doesn't work when connection pooling is enabled. At the time that this was originally posted, not all of the adapters pooled by default, but at this point it should be assumed that they do, so that each individual method call (.query(), .findOne(), etc.) could be on a different connection, and operating outside of the transaction. The next major version of Waterline will have transaction support, but until then, the only way to ensure that your queries are transactional is to use the raw database driver package (e.g. pg or mysql).
It sounds like what you need is a transaction. Sails doesn't support transactions at the framework level yet (it's on the roadmap) but if you're using a database that supports them (like Postgres or MySQL), you can use the .query() method of your model to access the underlying adapter and run native commands. Here's an example:
buyItem: function(req, res) {
try {
// Start the transaction
User.query("BEGIN", function(err) {
if (err) {throw new Error(err);}
// Find the user
User.findOne(req.param("userId").exec(function(err, user) {
if (err) {throw new Error(err);}
// Update the user balance
user.balance = user.balance - req.param("itemCost");
// Save the user
user.save(function(err) {
if (err) {throw new Error(err);}
// Commit the transaction
User.query("COMMIT", function(err) {
if (err) {throw new Error(err);}
// Display the updated user
res.json(user);
});
});
});
});
}
// If there are any problems, roll back the transaction
catch(e) {
User.query("ROLLBACK", function(err) {
// The rollback failed--Catastrophic error!
if (err) {return res.serverError(err);}
// Return the error that resulted in the rollback
return res.serverError(e);
});
}
}
I haven't tested this out. But as long as your not using multiple instances or clusters, you should just be able to store the status in memory. Because node is single threaded there shouldn't be any problems with atomicity.
var inProgress = {};
function buyItem(req, res) {
if (inProgress[req.session.user.id]) {
// send error response
return;
}
inProgress[req.session.user.id] = true;
// or whatever the function is..
req.session.user.subtractBalance(10.00, function(err, success) {
delete inProgress[req.session.user.id];
// send success response
});
}

SailsJS + Waterline concurrent db requests with promises

I'm a bit confused about concurrency in SailsJS's waterline.
Currently I'm doing data retrieval like this;
var results = {};
// Get user by id 5
User.find('5', function(err, user) {
results.user = user;
// when it resolves, get messages
Message.find({userId: '5'}, function(err, messages) {
results.messages = messages;
// when message query resolves, get other stuff
OtherStuff.find({userId: '5'}, function(err, otherStuff) {
results.otherStuff = otherStuff;
res.view({results});
});
});
});
The problem is that the DB calls are not concurrent. Every request launches after previous one's promise has been fulfilled. I'd like to launch all requests at the same time and then see somehow if all promises are fulfilled and if so, proceed to pass results to the view.
How am I gonna achieve this concurrency with db requests?
Thanks!
Use async.auto. The async module is globalized in Sails:
async.auto({
user: function(cb) {
// Note--use findOne here, not find! "find" doesn't accept
// an ID argument, only an object.
User.findOne('5').exec(cb);
},
messages: function(cb) {
Message.find({userId: '5'}).exec(cb);
},
otherStuff: function(cb) {
OtherStuff.find({userId: '5'}).exec(cb);
}
},
// This will be called when all queries are complete, or immediately
// if any of them returns an error
function allDone (err, results) {
// If any of the queries returns an error,
// it'll populate the "err" var
if (err) {return res.serverError(err);}
// Otherwise "results" will be an object whose keys are
// "user", "messages" and "otherStuff", and whose values
// are the results of those queries
res.view(results);
}
);

Resources