I'm doing a load test on user signup with the same email address for a webservice and the first 10 users which connect simultaneously will always register.
I'm using WATCH and MULTI but that doesn't seem to work any way.
I'm calling save() to save the user.
this.insert = function(callback) {
this.preInsert();
created = new Date();
updated = new Date();
// Also with these uncommented it still doesn't work
// Common.client.watch("u:" + this.username);
// Common.client.watch("em:" + this.email);
console.log(ID + " email is locked " + this.email);
Common.client.multi()
.set("u:" + this.username, ID)
.hmset("u:" + ID,
{"username": this.username
,"password": this.password
,"email": this.email
,"payment_plan": payment_plan
,"created": created.getTime()
,"updated": updated.getTime()
,"avatar": this.avatar})
.zadd("u:users", 0, ID)
.sadd("u:emails", this.email)
.set("u:"+ ID + ":stats", 0)
.set("em:" + this.email, ID)
.exec();
this.postInsert();
if (callback != null)
callback(null, this);
}
this.save = function(callback) {
// new user
if (ID == -1) {
var u = this;
Common.client.watch("u:" + this.username);
Common.client.exists("u:" + this.username, function(error, exists) {
// This username already exists
if (exists == 1) {
Common.client.unwatch();
if (callback != null)
callback({code: 100, message: "This username already exists"});
}
else {
Common.client.watch("em:" + u.email);
Common.client.get("em:" + u.email, function(err, emailExists) {
if (emailExists != null) {
Common.client.unwatch();
if (callback != null)
callback({code: 101, message: "This email is already in use"});
}
else {
Common.client.incr("u:nextID", function(error, id) {
if (error) callback(error);
else {
ID = id;
u.insert(callback);
}
});
}
});
}
});
}
// existing user
else {
var u = this;
Common.client.get("em:" + this.email, function(err, emailExists) {
if (emailExists != ID && emailExists) {
if (callback != null) {
callback({code: 101, message: "This email is already in use " + ID + " " + emailExists});
}
}
else {
u.update(callback);
}
});
}
}
The output almost always is:
1 email is locked test#test.com
2 email is locked test#test.com
3 email is locked test#test.com
4 email is locked test#test.com
5 email is locked test#test.com
6 email is locked test#test.com
7 email is locked test#test.com
8 email is locked test#test.com
9 email is locked test#test.com
10 email is locked test#test.com
Am I doing something wrong or redis can't handle that much concurrency.
Also this is the definition of Common:
var Common = {
client: redis.createClient(),
...
};
YES! After a night's rest of course the solution came to me in the shower.
The problem was that I used a single redis thread for the whole app and all the connections registered the watch on that thread. Of course it didn't signal that the keys were modified by a different client because there was no other client.
I know this thread is a 8 months old, but anyway, my thoughts still can help someone. There is a problem I still cannot understand, I even started my own thread dedicated to this issue Redis WATCH MULTI EXEC by one client, where I refer to yours. I now use "connection per transaction" method, which means I create new connection if I need to do transactions with WATCH-MULTI-EXEC. In other cases for atomic operations I use connection, which is created during an app launch. Not sure this method is effective, because creating a new connection means creating + authorizing and this produces latency, but it works.
Related
I am trying to do the following code in cloud code:
//Code for sending push notifications after an update is created
Parse.Cloud.afterSave('AssignmentMod', function(request){
//Only send if the mod is new
console.log("MOD SAVED. NOTIFICATIONS CHECK.")
if(!request.object.existed()) {
console.log("NEW MOD ADDED. NOTIFICATIONS START.")
var mod = request.object
//Get the users who have these classes
var userType = Parse.Object.extend('User')
var usersQuery = new Parse.Query(userType)
usersQuery.equalTo('currentClasses', mod.get("parentClass"))
//Get the users who removed the assignment
var modsType = Parse.Object.extend('AssignmentMod')
//If if was an assignment and removed at some point
var mods1Query = new Parse.Query(modsType)
mods1Query.notEqualTo("assignment", null)
mods1Query.equalTo("assignment", mod.get("assignment"))
mods1Query.equalTo("type", "remove")
//If it was an assignment mod and was removed at some point
var mods2Query = new Parse.Query(modsType)
mods2Query.notEqualTo("assignmentMod", null)
mods2Query.equalTo("assignmentMod", mod.id)
mods2Query.equalTo("type", "remove")
//Get the remove mods for this particular update
var modsQuery = new Parse.Query.or(mods1Query,mods2Query)
//Run the user and mods queries
Parse.Promise.when(
usersQuery.find(),
modsQuery.find()
).then( function(users, removeMods) {
console.log("QUERY 1 COMPLETE.")
//Get all users that copied this remove mod
var copyType = Parse.Object.extend('ModCopy')
var copyQuery = new Parse.Query(copyType)
copyQuery.containedIn("assignmentMod", removeMods)
copyQuery.find().then(function(copies){
console.log("QUERY 2 COMPLETE.")
var copyUsers = copies.map(function(copy){
return copy.get("user")
})
//Get the devices of users that belong to the class, did not remove the assignment, and have an ios devices
var deviceQuery = new Parse.Query(Parse.Installation)
deviceQuery.equalTo("deviceType", "ios")
deviceQuery.containedIn("user", users)
deviceQuery.notContainedIn("user", copyUsers)
//Send the push notification
console.log("PUSHING NOTIFICATIONS")
Parse.Push.send(
{
where: deviceQuery,
data: {
alert: "You have new updates."
}
},
{userMasterKey: true}
).then(
function(){
console.log("SUCCESSFULLY SEND NOTIFICATIONS.")
},
function(error){
throw "Error sending push notifications:" + error.code + " : " + error.message
console.log("FAILED TO SEND NOTIFICATIONS")
}
)
},
function(reason){
throw "Error: " + reason
print("ERROR: " + reason)
})
},
function(reason){
throw "Error: " + reason
print("ERROR: " + reason)
})
}
})
I can see in my logs that "MOD SAVED. NOTIFICATIONS CHECK." and "NEW MOD ADDED> NOTIFICATIONS START." are both logged out. However, I get no other logs after that and no push notifications are sent. I should at leas see the logs, and see none. Is there an issue with parse server using promises inside an afterSave or something? Why am is my code seemingly halting execution when it hits the first promise?
Your afterSave() cloud code must be ended by a response.success() or response.error(error)
You should wait for all your Promises complete end then finish your code with something like:
promise.then( function(){
console.log("END of afterSave() " ;
response.success();
}, function(error){
console.error("error in afterSave() " + " : " + error.message);
response.error(JSON.stringify({code: error.code, message: error.message}));
});
This turned out to be an issue with Heroku hosting. I have not tracked down passed that. If the same code is hosted on AWS or another hosting platform, these issues do not occur. There is likely some server settings Heroku uses in it's default setup that need changed. If you encounter this issue and have more of a solution beyond "Switch off Heroku hosting" then submit an answer and I will accept.
I am using node js to send push notifications to Android devices using GCM and want to delete the device id if I get an error message from Google. The code snippet is attached and runs in a loop (i is number of device ids).
My issue is that if the call returns an error (err4), all future loops return if(err4) as true and all the ids after the first error are deleted. Why is the err4 not getting reset?
for(var i = 0; i < noofdeviceIds; i++){
// some more code here to create the 'message'.
(function(i) {
gcm.send(message, function(err4, messageId){
if (err4) {
console.log("\nError occured: Notification could not be sent with error: ", err4);//
if(err4 == 'NotRegistered' || err4 == 'InvalidRegistration'){
console.log("\nRemoving the entry from the DB.");
var removeEntryQuery = "DELETE FROM devicereg WHERE deviceregId = '" + deviceregIds[i] + "'";
req._dbConnection.query(removeEntryQuery, function(err5, row) {
if(row!=0){console.log("\nDB returned: ", row);}
});
}
}
else {
console.log("\nMessage sent with message ID: ", messageId);
}
});
})(i);
}
I am creating an insert script that does some business logic.
Basically, I want to check to see if a value in the inserted item exists in a table. But, it seems like if I find a problem Request.Send() doesn't stop execution and get an error.
I think there is an async issue here. I'm not 100% sure how to solve.
Is there a way to stop execution of the script?
if (item.memberType === 'Family' && item.primaryFamilyMember) {
table
.where({
memberNumber: item.primaryFamilyMember,
memberType: 'Family',
primaryFamilyMember: null })
.read({
success: function(results) {
if (results.length == 0) {
request.respond(statusCodes.BAD_REQUEST,
'Invalid Primary Family Member specified.');
console.error('Invalid Primary Family Member specified:' + item.primaryFamilyMember);
validInsert = false;
} else {
item.memberType = results[0].memberType;
item.memberLevel = results[0].memberLevel;
item.dateOfExpiry = results[0].dateOfExpiry;
}
}
});
}
if (validInsert) {
var today = new Date();
var prefix = today.getFullYear().toString().substr(2,2) + ('0' + (today.getMonth() + 1)).slice(-2);
table.includeTotalCount().where(function(prefix){
return this.memberNumber.substring(0, 4) === prefix;
}, prefix)
.take(0).read({
success: function (results) {
if (isNaN(results.totalCount)) {
results.totalCount = 0;
}
item.memberNumber = prefix + ('00' + (results.totalCount + 1)).slice(-3);
request.execute();
}
});
}
Yes, validInsert is declared at the top of the insert function.
I assume what's happening is the if(validInsert) runs before the read callback. But if so, i'm not sure why I'm getting "Error: Execute cannot be called after respond has been called." That implies the callback is running first.
Also, the record is being inserted when it shouldn't be even though the 400 error is sent back to the client.
This is an express app right? Should I just call response.end() after the error occurs?
Yes, there are definitely asyn issues in that code. To solve get rid of your validInsert flag and simply move the if (validInsert) section into the success callback (or make it a function called from the success callback). For example:
success: function(results) {
if (results.length == 0) {
request.respond(statusCodes.BAD_REQUEST,
'Invalid Primary Family Member specified.');
console.error('Invalid Primary Family Member specified:' + item.primaryFamilyMember);
} else {
item.memberType = results[0].memberType;
item.memberLevel = results[0].memberLevel;
item.dateOfExpiry = results[0].dateOfExpiry;
var today = new Date();
var prefix = today.getFullYear().toString().substr(2,2) + ('0' + (today.getMonth() + 1)).slice(-2);
...
//respond successfully
}
}
buses_near_stops begins as an empty array. Inside the asynchronous calls to the database, it is supposed to be filled. Then after the calls finish, I want to use the data inside of it. When I run this code, the final console log of buses_near_stops executes before the inner database calls, even though I have the looped calls inside of a closure. According to this post, a closure should work, but here it is doing nothing for me.
var buses_near_stops = [];
buses_near_stops.test = "TEST";
// return these fields of each location document in the database
Location.find({}, 'service_name coordinates vehicle_id last_gps_fix', function(err, doc) {
//console.log('location found ' + JSON.stringify(doc));
if(err){return next(err);}
doc.forEach(function(j,k) {
//Find a stop that is near enough to each given bus that we can say the bus is 'at' that stop
//Making sure it returns 1 stop now because I don't know proper distance
(function(buses_near_stops) {
Stop.findOne({coordinates: { $near : j.coordinates, $maxDistance: .0001}
}, function(err, stop){
if(err){return next(err);}
console.log('stop found ' + j.service_name + " " + JSON.stringify(stop));
// service_name is null if bus is out of service (I believe)
if(stop !== null && j.service_name !== null) {
var service_name_of_bus = j.service_name;
console.log('service name of bus ' + service_name_of_bus);
// Find the service document associated with service_name_of_bus
var service_of_name = Service.findOne({name: service_name_of_bus}, function(err, service_of_name){
if(err){return next(err);}
// If the service has 'stop' on its route
if(service_of_name != null && service_of_name.routes[0].stops.indexOf(stop.stop_id) > -1) {
console.log('stop found on service');
// We have now found a bus that is stopped at a stop on its route
console.log('test ' + buses_near_stops.test);
buses_near_stops.push(
{
time: j.last_gps_fix,
bus_coords: j.coordinates,
stop_coords: stop.coordinates,
vehicle_id: j.vehicle_id,
stop_id: stop.stop_id,
service_name: service_name_of_bus
});
console.log('length ' + buses_near_stops.length);
}
});
}
})}(buses_near_stops));
});
console.log('buses near stops ' + JSON.stringify(buses_near_stops));
});
Hope someone can assist with a (simple) async question on node-redis. I'm trying to load a set from a hash in the redis db and then use that populated set further on. Here's the code snippet :-
var redis_client = redis.createClient(REDIS_PORT, REDIS_URL);
redis_client.hgetall(target_hash,function(e,o){
Object.keys(o).forEach(function(target){
// get the "name" from the hash
redis_client.hget(o[target],"name",function(e,o){
if (e){
console.log("Error occurred getting key: " + e);
}
else {
redis_client.sadd("newset",o);
}
});
});
// the following line prints nothing - why ??
redis_client.smembers("newset",redis.print);
When I examine the contents of "newset" in redis it is populated as expected, but at runtime it displayed as empty. I'm sure it's an async issue - any help much appreciated !
hgetall is an asynchronous call: when it receives a reply from the redis server, it will eventually call your callback function (target) { ... }. But within your script, it actually returns immediately. Since hgetall returns very fast, Node will immediately run the next statement, smembers. But at this point the sadd statements haven’t run yet (even if your system is very fast because there hasn’t been a context switch yet).
What you need to do is to make sure smembers isn’t called before all the possible sadd calls have executed. redis_client provides the multi function to allow you to queue up all the sadd calls and run a callback when they’re all done. I haven’t tested this code, but you could try this:
var redis_client = redis.createClient(REDIS_PORT, REDIS_URL);
redis_client.hgetall(target_hash, function(e, o) {
var multi = redis_client.multi();
var keys = Object.keys(o);
var i = 0;
keys.forEach(function (target) {
// get the "name" from the hash
redis_client.hget(o[target], "name", function(e, o) {
i++;
if (e) {
console.log("Error occurred getting key: " + e);
} else {
multi.sadd("newset", o);
}
if (i == keys.length) {
multi.exec(function (err, replies) {
console.log("MULTI got " + replies.length + "replies");
redis_client.smembers("newset", redis.print);
});
}
});
});
});
Some libraries have a built-in equivalent of forEach that allows you to specify a function to be called when the loop is all done. If not, you have to manually keep track of how many callbacks there have been and call smembers after the last one.
You shouldn't use multi unless you need actually need a transaction.
just keep a counter of the transactions and call smembers in the final callback
var redis_client = redis.createClient(REDIS_PORT, REDIS_URL);
var keys = Object.keys(o);
var i = 0;
redis_client.hgetall(target_hash,function(e,o){
Object.keys(o).forEach(function(target){
// get the "name" from the hash
redis_client.hget(o[target],"name",function(e,o){
i++
if (e){
console.log("Error occurred getting key: " + e);
}
else {
redis_client.sadd("newset",o);
if (i == keys.length) {
redis_client.smembers("newset", redis.print);
}
}});