DocumentDB, How to work with continuationToken in a SP - azure

The next SP suppose to run over the collection and keep query for the next batch of documents (10 docs every batch). but instead return the same 10 documents every time.
function sample(prefix) {
var continuations = [],
ids = [],
context = getContext(),
collection = context.getCollection(),
response = context.getResponse();
var queryOptions = { pageSize: 10, continuation: null };
for (i = 0; i < 10; i++) {
// get all user wish list actions
var query = "select * from w",
accept = collection.queryDocuments(collection.getSelfLink(), query, queryOptions, processMultiUsers);
if (!accept) throw "Unable to read user's sessions";
}
getContext().getResponse().setBody(ids);
function processMultiUsers(err, docs, options) {
if (err) throw new Error("Error: " + err.message);
if (docs == undefined || docs.length == 0) throw new Error("Warning: Users not exists");
for (j = 0; j < docs.length; j++) {
ids.push(docs[j].UserId);
}
queryOptions.continuation = options.continuation;
continuations.push(options.continuation);
}}

In the script that you wrote, the execution of the queries are done synchronously and they are queued up with the same initial continuation token, which is null. Instead, we need to take the token from the first query and then queue the next and continue.
The below sample should help achieve what you are looking for
function sample(continuationToken) {
var collection = getContext().getCollection();
var maxResult = 10;
var documentsProcessed = 0;
var ids = [];
var filterQuery = "select * from w";
tryQuery(continuationToken);
function tryQuery(nextContinuationToken) {
var responseOptions = { continuation: nextContinuationToken, pageSize: maxResult };
if (documentsProcessed >= maxResult || !query(responseOptions)) {
setBody(nextContinuationToken);
}
}
function query(responseOptions) {
return (filterQuery && filterQuery.length) ?
collection.queryDocuments(collection.getSelfLink(), filterQuery, responseOptions, onReadDocuments) :
collection.readDocuments(collection.getSelfLink(), responseOptions, onReadDocuments);
}
function onReadDocuments(err, docFeed, responseOptions) {
if (err) {
throw 'Error while reading document: ' + err;
}
documentsProcessed += docFeed.length;
for (var i = 0; i < documentsProcessed; i++) {
ids.push(docFeed[i].UserId);
}
if (responseOptions.continuation) {
tryQuery(responseOptions.continuation);
} else {
setBody(null);
}
}
function setBody(continuationToken) {
var body = { continuationToken: continuationToken, documentsProcessed: documentsProcessed, ids: ids };
getContext().getResponse().setBody(body);
}
}

Related

node.js how to get data from redis database inside in middleware

I'm doing middleware module that will extract data from redis and put to req.my_session.[here]
This is function that call inside app.use();
function parse_cookies(req){
if(req.headers.cookie != null){
var result = req.headers.cookie.match(new RegExp('m:[^=]*=[^; ]*', 'ig'));
if(result != null){
for(var i = 0; i < result.length; i++){
var result1 = result[i].split('=');
req.my_session[result1[0].substr(2)] = result1[1];
// get from redis value
client.get('sess:'+result1[1], function(err, reply) {
// reply is null when the key is missing
console.log(reply);
let li = i;
req.my_session[result1[0].substr(2)] = reply;
console.log('li = ' + li);
console.log('result1.lenght= ' + result.length);
if(i == result.length){
console.log('call the next');
}
});
}
}
}
} // parse_cookies
in console i outputs always 3, how can I get all data from database using redis.get and on last data call next() function for get out from my function?
problem it's get data from database in my middleware, I can't because redis has callback function
client.get("missingkey", function(err, reply) {
// reply is null when the key is missing
console.log(reply);
});
I think the issue is becuase of async in loop you can try the following
function parse_cookies(req){
if(req.headers.cookie != null){
var result = req.headers.cookie.match(new RegExp('m:[^=]*=[^; ]*', 'ig'));
if(result != null){
var promises = [];
for(var i = 0; i < result.length; i++){
var result1 = result[i].split('=');
promises.push(getFromRd(req, result1));
}
return Promise.all(promises)
.then(() => {
return next()
})
.catch((e) => {
return next({error: e})
})
}
}
} // parse_cookies
function getFromRd(req, result1) {
req.my_session[result1[0].substr(2)] = result1[1];
// get from redis value
return client.get('sess:'+result1[1], function(err, reply) {
if (err) {
throw Error(' failed to find ' + 'sess:' + result1[1])
}
// reply is null when the key is missing
console.log(reply);
let li = i;
req.my_session[result1[0].substr(2)] = reply;
console.log('li = ' + li);
console.log('result1.lenght= ' + result.length);
return {success:true}
});
}

DocumentDB update based on select statement fails

I am planning to update the document, if similar values already exists in other records of the document.
Say, if root document has few records in which few fields(track,time) are similar, then I want to update the isOriginal field for such record as false.
function updateArticlesDetailsX() {
var collection = getContext().getCollection();
var collectionLink = collection.getSelfLink();
var response = getContext().getResponse();
var docCount = 0;
var counter = 0;
tryQueryAndUpdate();
function tryQueryAndUpdate(continuation) {
var query = {
query: "select * from root r ORDER BY r.created.epoch DESC"
};
var requestOptions = {
continuation: continuation
};
var isAccepted =
collection
.queryDocuments(collectionLink,
query,
requestOptions,
function queryCallback(err, documents, responseOptions) {
//response.setBody(responseOptions);
if (err) throw err;
if (documents.length > 0) {
document.upd
// If at least one document is found, update it.
docCount = documents.length;
//response.setBody("Found " + docCount + " documents");
for (var i=0; i<docCount; i++){
tryUpdate(documents[i]);
}
//response.setBody("Updated " + docCount + " documents");
}
if (responseOptions.continuation) {
// Else if the query came back empty, but with a continuation token;
// repeat the query w/ the token.
tryQueryAndUpdate(responseOptions.continuation);
}
//else {
// throw new Error("Document not found.");
// }
});
if (!isAccepted) {
//throw new Error("The stored procedure timed out");
}
}
function tryUpdate(document) {
//Optimistic concurrency control via HTTP ETag.
var requestOptions = { etag: document._etag };
//Update statement goes here:
document.update({"track":{document.track},"Time":{document.Time} }, {"$set":{"isOriginal":"false!"}});
document.created = {
date: "2016-06-22 19:18:14",
epoch: 1466623094582
}
//response.setBody(document);
var isAccepted = collection
.replaceDocument(document._self,
document,
requestOptions,
function replaceCallback(err, updatedDocument, responseOptions) {
if (err) throw err;
counter++;
response.setBody("Updated " + counter + " documents");
// response.setBody(updatedDocument);
});
// If we hit execution bounds - throw an exception.
if (!isAccepted) {
//throw new Error("The stored procedure timed out");
}
}
}

DocumentDB: Issue with updating list with multiple columns

I have a document where I want turn of is_approved = false, if multiple columns are similar(posted_by and location).
I am not able to write a query which can get multiple duplicate columns..
function Prop() {
var collection = getContext().getCollection();
var collectionLink = collection.getSelfLink();
var response = getContext().getResponse();
var counter = 0;
var responseBody = {
updated: 0,
continuation: true,
error: "",
log: ""
};
// Validate input.
getFullListOfPosts();
// Recursively queries for a document by id w/ support for continuation tokens.
// Calls findDuplicates(document) as soon as the query returns a document.
function getFullListOfPosts(continuation) {
var query = {
query: "select * from root r ORDER BY r.created.epoch DESC"
};
var requestOptions = {
continuation: continuation
};
var isAccepted = collection.queryDocuments(collectionLink, query, requestOptions, proccessFullListOfPosts);
// If we hit execution bounds - throw an exception.
if (!isAccepted) {
responseBody.log += "Query not accepted";
response.setBody(responseBody);
}
}
function proccessFullListOfPosts(err, documents, responseOptions) {
if (err) {
responseBody.error = err;
throw err;
}
if (documents.length > 0) {
// If documents are found, update them.
responseBody.log += "Found documents: " + documents.length;
findDuplicates(documents);
}
}
// Updates the supplied document according to the update object passed in to the sproc.
function findDuplicates(documents) {
if (documents.length > 0) {
responseBody.log += "Updating documents " + documents.length;
//for (var i = 0; i < documents.length; i++) {
var first = documents[0];
//The below query is not right. I want the above query to fetch all records with same posted_by and location and turn its is_approved to false
var query = 'select * from root r WHERE r.is_approved = true AND r.posted_by = "' + first.posted_by + '"';
var requestOptions = {
etag: first._etag
};
var isAccepted = collection.queryDocuments(collectionLink, query, requestOptions, identifyRecordsToDisable);
// Update the document. // If we hit execution bounds - throw an exception.
if (!isAccepted) {
responseBody.log += "Update not accepted";
response.setBody(responseBody);
}
} else {
getFullListOfPosts();
}
}
function identifyRecordsToDisable(err, documents, responseOptions) {
if (err) {
responseBody.error = err;
throw err;
}
if (documents.length > 0) {
// If documents are found, update them.
responseBody.log += "Found documents: " + documents.length;
for (var i = 0; i < documents.length; i++) {
var j = documents[i];
j.is_approved = false;
responseBody.log += "^Updated" + j.id + j.is_approved + ""
var requestOptions = {
etag: j._etag
};
var isAccepted = collection.replaceDocument(j._self, j, requestOptions, onReplaced);
responseBody.log += "*" + j.id + j.is_approved + "*"
}
findDuplicates(documents);
}
}
function onReplaced(err, updatedDocument, responseOptions) {
if (err) {
responseBody.error = err;
//throw err;
}
counter++;
response.setBody("Updated " + counter + " documents");
}
}
In the above code, in the function 'findDuplicates', I am unable to write a correct query through which I can get all the duplicate records.
I tried using distinct, group by. having. I think documented doesn't support them
The following query should help you find documents that match both property values.
SELECT *
FROM root r
WHERE r.is_approved = true
AND r.posted_by = "' + first.posted_by + '"
AND r.location = "' + first.location + '"

Notifying caller about error from callback in node.js on error from sqlite3 call

In node.js + sqlite3:
What is a good way to notify a caller from a callback about errors and get the caller to try again ? When we get a database locked error - would like to try and run the query again.
TestController.prototype.addDevices = function (number_of_devices, callback_after_all_devices_have_been_added) {
var controller = this;
var db = controller.connectToDb();
for(i = 0; i < number_of_devices; i++) {
db.each("SELECT api_key as ak FROM user_table ORDER BY RANDOM() LIMIT 1", function(err, row) {
//sometimes we get a SQLITE: database locked error
if (err !== null) {
console.log("Error found");
//i-- this will not work - but how do we do it then?
return;
}
console.log("Error: " + err);
//code to process if we get an entry from the user_table ..
});
}
}
This is how I managed to solve this - added a retry counter and moved the problem code to its own function, passing in the epilogue as a callback. It works - but is this a good way?
(other than using promises or the async module?)
TestController.prototype.addDevices = function (number_of_devices, callback_after_all_devices_have_been_added) {
var number_of_users = 0;
var controller = this;
controller.pending_db_writes_for_device = number_of_devices;
for(i = 0; i < number_of_devices; i++) {
var api_key = controller.getRandomAPIKeyFromDB(10,function(access_key) {
var mac_address = controller.generateRandomMAC();
var friendly_name = Faker.random.array_element(["Washing Machine","Television","Dishwasher","Set top box"]);
var description = Faker.Lorem.sentence(5);
controller.registerDeviceWithAPI(mac_address, friendly_name, description, access_key, callback_after_all_devices_have_been_added);
});
}
}
TestController.prototype.getRandomAPIKeyFromDB = function (retry_counter, callback) {
var controller = this;
var db = controller.connectToDb();
console.log("Retry counter: "+ retry_counter);
db.each("SELECT api_key as ak FROM user_table ORDER BY RANDOM() LIMIT 1", function(err, row) {
if (err !== null) {
console.log("Error: " + err + "retry count: "+ retry_counter);
console.log("Error found");
if(retry_counter === 0) {
console.log("Bailing out after retry limit reached");
return;
}
controller.getRandomAPIKeyFromDB(--retry_counter,callback);
}
else {
console.log("Successfully got api key: " + row.ak);
callback(row.ak);
}
}
);
}

Loop synchronous multiple async.whilst

I want to use many included loops in node.js in synchronous mode.
Example :
for (var i = 0; i < length1; i++) {
for (var j = 0; j < length2; j++) {
for (var k = 0; k < length3; k++) {
//completed 3
}
//completed 2
}
//do completed 1
}
How to do this with async? I tried this :
exports.myFunction = function (callback) {
var finalListA = new Array();
var pos = 0;
Model_A.find().populate('listOfItems')
.lean().exec(function (err, As) {
if (err) {
console.log(err);
return callback(err, null);
} else {
//For each A
var i = 0;
async.whilst(
function () {
return i < As.length;
},
function (callback1) {
var isActive = false;
//For each B into the A
var j = 0;
async.whilst(
function () {
return j < As[i].Bs.length;
},
function (callback2) {
Model_B.findById(AS[i].Bs[j]._id, function (err, B) {
if (err) {} else {
var k = 0;
// For each C in the B
async.whilst(
function () {
return k < B.Cs.length;
},
function (callback3) {
if (B.Cs[k].dateEnd >= Date.now()) {
isActive = true;
}
k++;
callback3();
},
function (err) {
console.log("3 COMPLETED");
}
);
}
});
j++;
callback2();
},
function (err) {
console.log("2 COMPLETED");
if (err) {} else {
if (isActive == true) {
finalListA[pos] = As[i];
pos = pos + 1;
}
}
}
);
i++;
callback1();
},
function (err) {
console.log("1 COMPLETED");
if (err) {} else {
return callback(null, finalListA);
}
}
);
}
});
}
The trace shows me :
COMPLETED 2
COMPLETED 2
COMPLETED 1
COMPLETED 3
COMPLETED 3
The order expected is :
COMPLETED 3
COMPLETED 3
COMPLETED 2
COMPLETED 2
COMPLETED 1
You must call the callbacks of the higher loops from the end callback of your whilst loop (like you did with the outermost callback), instead of calling them synchronously from the whilst body in which you just started the next level iteration.
Btw, I don't know what you actually want to do, but whilst does not seem the best choice for iterating arrays. Use the parallel each or the serial eachSeries (or their map or reduce equivalents).
I've recently created simpler abstraction called wait.for to call async functions in sync mode (based on Fibers). It's at an early stage but works. It is at:
https://github.com/luciotato/waitfor
Using wait.for, you can call any standard nodejs async function, as if it were a sync function.
I do not understand exactly what are you trying to do in your code. Maybe you can explain your code a little more, or give some data example.
I dont' know what Model_A or Model_B are... I'm guessing most of your code, but...
using wait.for your code migth be:
var wait=require('wait.for');
exports.myFunction = function(callback) {
//launchs a Fiber
wait.launchFiber(inAFiber,callback);
}
function inAFiber(callback) {
var finalListA = new Array();
var pos = 0;
var x= Model_A.find().populate('listOfItems').lean();
As = wait.forMethod(x,"exec");
//For each A
for(var i=0;i<As.length;i++){
var isActive = false;
//For each B into the A
for(var j=0; j < As[i].Bs.length;j++){
var B=wait.forMethod(Model_B,"findById",AS[i].Bs[j]._id);
// For each C in the B
for(var k=0; k < B.Cs.length;k++){
if(B.Cs[k].dateEnd >= Date.now()) {
isActive = true;
}
}
console.log("3 COMPLETED");
}
console.log("2 COMPLETED");
if(isActive == true) {
finalListA[pos] = As[i];
pos = pos + 1;
}
};
console.log("1 COMPLETED");
return callback(null,finalListA);
}
Also, for what I see, you should break the loops as soon as you find one item (isActive), and you don't need the var pos. Doing that your code will be:
var wait=require('wait.for');
exports.myFunction = function(callback) {
//launchs a Fiber
wait.launchFiber(inAFiber,callback);
}
function inAFiber(callback) {
var finalListA = [];
var x= Model_A.find().populate('listOfItems').lean();
As = wait.forMethod(x,"exec");
var isActive;
//For each A
for(var i=0;i<As.length;i++){
isActive = false;
//For each B into the A
for(var j=0; j < As[i].Bs.length;j++){
var B=wait.forMethod(Model_B,"findById",AS[i].Bs[j]._id);
// For each C in the B
for(var k=0; k < B.Cs.length;k++){
if(B.Cs[k].dateEnd >= Date.now()) {
isActive = true;
break;//for each C
}
} //loop for each C
console.log("3 COMPLETED");
if (isActive) break;//for each B
} //loop for each B
if (isActive) finalListA.push(As[i]);
console.log("2 COMPLETED");
} //loop for each A
console.log("1 COMPLETED");
return callback(null,finalListA);
}

Resources