How to make the setTimeout sequential for multiple database updates? - node.js

I am updating database fields by using setTimeout(). So when the updates are multiple what happens is the the last primary key is used for all the updates. How do I run the setTimeout() function sequentially. Below is the portion of code which does that.
for( var i = 0; i < req.body.devicelist.length; i++) { //running for loop for multiple elements
var data = JSONPARSE.toObject(req.body);
mac_id = req.body.devicelist[i];
data.mac_id = mac_id;
var gateway_config;
for (let j = 0; j < gateways_config.length; j++) { //code for fetching specific element. IGNORE
if(gateways_config[j].latest_config.mac_id == mac_id){
gateway_config = gateways_config[j]
break;
}
gateway_config = undefined
}
await syncConfig(req.body,gateway_config, req.decoded.id);
..........
..........
}
syncConfig(body,gateway,user_id){
var jsonObj = body;
...
...
...
config_timeout_array[jsonObj.mac_id] = setTimeout(() => { //Causing problem
commandTimeout(jsonObj.org_id,jsonObj.mac_id)
}, 10000);
...
...
}
commandTimeout:(org_id, mac_id) =>{
console.log(mac_id); //prints same mac_id (the last in the array)
return gateway_model.findOneAndUpdate({ org_id: org_id, mac_id: mac_id }, { 'sync_sent': false }, {"new": true})
.then((updated_gateway) => {
...
...
...
}
}

config_timeout_array[jsonObj.mac_id] = setTimeout(() => { //Causing problem
commandTimeout(jsonObj.org_id,jsonObj.mac_id)
}, 10000);
Instead of doing the above logic directly, call a function and do it there. I don't know why but it worked!
seqTimeout(jsonObj.org_id,jsonObj.mac_id); //function call
seqTimeout(org_id,mac_id){
config_timeout_array[mac_id] = setTimeout(() => {
GatewayController.commandTimeout(org_id, mac_id);
}, COMMAND_TIMEOUT);
}

Related

How to handle a long request with Express

I'm working on a simple function I have for a specific GET request triggered in the browser. The objective of this request is to make multiple queries to a mongodb (mongoose) database and then perform some calculation and structure formating on the results to send it back to the browser.
The only problem is that everything takes too long and it results in an error in the browser:
net::ERR_EMPTY_RESPONSE
to give an example of part of the function I'm trying to build here it goes:
async function getPriceByMake(makes, id) {
return new Promise(async (resolve, reject) => {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
}
resolve(pMakes);
});
}
In this function, if I leave the async / await out, I get an empty {} on the other end. Which is obviously not the objective.
I've been searching the web a little and was able to find an article pointing to this scheme:
Browser:
Initiates request
displays progress
Show result
WebServer:
Submit event
Checks for completion
Return result
BackEndApp:
Picks up event
Runs task
Returns results
My question is the following:
How can I do that with NodeJS and Express?
In this code:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
Your await isn't working because you're passing a callback to Listing.find(). When you do that, it does NOT return a promise and therefore the await does nothing useful. You get the empty response because the await doesn't work and thus you call resolve() before there's any actual data there.
Change the code to this:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
And, then the await will work properly.
You also should remove the return new Promise() wrapper. You don't want that. Just make the function async and use await and it will already return a promise.
Here's your function with the unnecessary promise wrapper removed:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
}
return pMakes;
}
Then, keep in mind that whatever code sends your actual response needs to use .then() or await when calling this async function in order to get the final result.
Your best bet to speed up this code would be to refactor either your queries or your database structure or both to not have to do N * M separate queries to get your final result. That's likely where your slowness is coming from. The biggest performance gains will probably come from reducing the number of queries you have to run here to far fewer.
Depending upon your database configuration and capabilities, it might speed things up to run the inner loop queries in parallel as shown here:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
await Promise.all(modelsArr.map(async currModel => {
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}));
}
return pMakes;
}

How do I get all child terms of a termset in Sharepoint?

How do I get a flattened version of child terms, or just the last level/tier of terms.
Ex: TermSet -->
Term -->
ChildTerm
How do I grab the child term in an SPFX webpart? I can grab the Terms under the termset, but I cant go any deeper.
Tested code based on this thread(test the code to get term of current site).
private getTermsetWithChildren(termStoreName: string, termsetId: string) {
return new Promise((resolve, reject) => {
//const taxonomy = new Session(siteCollectionURL);
const store: any = taxonomy.termStores.getByName(termStoreName);
store.getTermSetById(termsetId).terms.select('Name', 'Id', 'Parent').get()
.then((data: any[]) => {
let result = [];
// build termset levels
do {
for (let index = 0; index < data.length; index++) {
let currTerm = data[index];
if (currTerm.Parent) {
let parentGuid = currTerm.Parent.Id;
insertChildInParent(result, parentGuid, currTerm, index);
index = index - 1;
} else {
data.splice(index, 1);
index = index - 1;
result.push(currTerm);
}
}
} while (data.length !== 0);
// recursive insert term in parent and delete it from start data array with index
function insertChildInParent(searchArray, parentGuid, currTerm, orgIndex) {
searchArray.forEach(parentItem => {
if (parentItem.Id == parentGuid) {
if (parentItem.children) {
parentItem.children.push(currTerm);
} else {
parentItem.children = [];
parentItem.children.push(currTerm);
}
data.splice(orgIndex, 1);
} else if (parentItem.children) {
// recursive is recursive is recursive
insertChildInParent(parentItem.children, parentGuid, currTerm, orgIndex);
}
});
}
resolve(result);
}).catch(fail => {
console.warn(fail);
reject(fail);
});
});
}
Call the function.
this.getTermsetWithChildren(
'Taxonomy_hAIlyuIrZSNizRU+uUbanA==',
'70719569-ae34-4f24-81b9-0629d68c05aa'
).then(data => {
console.log(data);
});

What should I do if I want to add clearInterval in diferent methods?

I've moveMarker methods with setInterval to making my Marker in mapbox-gl is moving along my Routes Line and I've play button to trigger this function . My question is , what should I do if I want to create pause button with clearInterval?
I've tried create function to clearInterval in moveMarker but doesn't work
this is my function to move the marker :
moveMarker () {
const moveCoordinate = []
const loop = setInterval(() => {
if (this.index + 1 === (this.coordinates.length - 1)) {
clearInterval(loop)
}
for (let index = 0; index < moveCoordinate.length; index++) {
moveCoordinate[index].remove()
}
this.map.panTo(this.coordinates[this.index])
const lat = this.coordinates[this.index][0]
const lng = this.coordinates[this.index][1]
const newMarker = new mapboxgl.LngLat(lat, lng)
console.log('new', newMarker)
const markerMapbox = new mapboxgl.Marker()
.setLngLat(newMarker)
.addTo(this.map)
moveCoordinate.push(markerMapbox)
this.index++
}, 1000)
},
and this is stop function :
stop () {
clearInterval(this.moveMarker)
},
First off you should store you interval in data property, to have access to it in the stop method. Then in stop method just call clearInterval with stored interval:
export default {
...
data() {
return {
interval: null
}
},
methods: {
moveMarker () {
const moveCoordinate = []
this.interval = setInterval(() => {
if (this.index + 1 === (this.coordinates.length - 1)) {
clearInterval(this.interval)
}
for (let index = 0; index < moveCoordinate.length; index++) {
moveCoordinate[index].remove()
}
this.map.panTo(this.coordinates[this.index])
const lat = this.coordinates[this.index][0]
const lng = this.coordinates[this.index][1]
const newMarker = new mapboxgl.LngLat(lat, lng)
console.log('new', newMarker)
const markerMapbox = new mapboxgl.Marker()
.setLngLat(newMarker)
.addTo(this.map)
moveCoordinate.push(markerMapbox)
this.index++
}, 1000)
},
stop() {
clearInterval(this.interval)
},
},
...
}
Calling clearInterval on the moveMarker method isn't going to do anything. You need to save the interval id somewhere that stop can access it.
e.g. Inside moveMarker:
this.intervalId = loop
Then:
stop () {
clearInterval(this.intervalId)
}
There's no need to add intervalId to your data as you don't need it to be reactive.

Is it possible to build a dynamic task list in nodejs Async (waterfall, series, etc...)

I am pulling information from some collections in mongo that contain node and edge data. First i must get the node so i can grab its edges. Once i have a list of edges i then go back out and grab more nodes (etc.. based on a depth value). The following code is an loose example of how i am attempting to use async.waterfall and the task list.
Initially i have only a single task but once i make my first query i add to the task array. Unfortunately this does not seem to register with async and it does not continue to process the tasks i am adding.
Is there a better way to do this?
var async = require('async')
var mongoose = require('mongoose')
var _ = requrie('underscore')
var Client = this.Mongo.connect(/*path to mongo*/)
var Node = mongoose.Schema({
id : String,
graph_id : String
})
var Edge = mongoose.Schema({
id : String,
source_id : String,
destination_id : String
})
var Nodes = Client.model('myNode', Node)
var Edges = Client.model('myEdge', Edge)
var funcs = []
var round = 1
var depth = 2
var query = {
node : {
id : '12345'
},
edge : {
id : '12345'
}
}
var addTask = function(Nodes, Edges, query, round, depth) {
return function(callback) {
queryData(Nodes, Edges, query, function(err, node_list) {
if(depth > round) {
round++
function_array.push(addTask(Nodes, Edges, query, round, depth))
}
})
}
}
var queryData = function(Nodes, Edges, query, cb) {
async.waterfall([
function(callback) {
Nodes.find(query.node, function(err, nodes) {
var node_keys = _.map(nodes, function(node) {
return node.id
})
callback(null, nodes, node_keys)
})
},
function(nodes, node_keys, callback) {
query.edge.$or = [ {'source_id' : {$in:node_keys}}, {'destination_id' : {$in:node_keys}} ]
Edges.find(query.edge, function(err, edges) {
var edge_keys = _.map(edges, function(edge) {
if(edge['_doc']['source_id'] != query.node.id) {
return edge['_doc']['source_id']
} else {
return edge['_doc']['destination_id']
}
callback(null, nodes, edges, node_keys, edge_keys)
})
})
}
], function(err, nodes, edges, node_keys, edge_keys) {
// update the results object then...
cb(null, _.uniq(edge_keys)
})
}
var function_array = []
function_array.push(addTask(Nodes, Edges, query, round, depth))
async.waterfall(function_array, function(err) {
Client.disconnect()
//this should have run more than just the initial task but does not
})
--------------------- UPDATE ---------------------------
So after playing around with trying to get Async waterfall or series to do this by adding trailing functions I decided to switch to using async.whilst and am now happy with the solution.
function GraphObject() {
this.function_array = []
}
GraphObject.prototype.doStuff = function() {
this.function_array.push(this.buildFunction(100))
this.runTasks(function(err) {
console.log('done with all tasks')
}
}
GraphObject.prototype.buildFunction = function(times) {
return function(cb) {
if(times != 0) {
this.function_array.push(this.buildFunction(times - 1))
}
cb(null)
}
}
GraphObject.prototype.runTasks = function(cb) {
var tasks_run = 0
async.whilst(
function(){
return this.function_array.length > 0
}.bind(this),
function(callback) {
var func = this.function_array.shift()
func.call(this, function(err) {
tasks_run++
callback(err)
})
}.bind(this),
function(err) {
console.log('runTasks ran '+tasks_run+' tasks')
if(err) {
cb(500)
}
cb(null)
}.bind(this)
)
}
A task in your function_array can only add a new task to the array provided it is NOT the last task in the array.
In your case, your function_array contained only 1 task. That task itself cannot add additional tasks since it's the last task.
The solution is to have 2 tasks in the array. A startTask to bootstrap the process, and a finalTask that is more of a dummy task. In that case,
function_array = [startTask, finalTask];
Then startTask would add taskA, taskB will add task C and eventually
function_array = [startTask, taskA, taskB, taskC, finalTask];
The sample code below that illustrates the concepts.
var async = require('async');
var max = 6;
var nodeTask = function(taskId, value, callback){
var r = Math.floor(Math.random() * 20) + 1;
console.log("From Node Task %d: %d", taskId, r);
// add an edge task
if (taskId < max) {
function_array.splice(function_array.length-1, 0, edgeTask);
}
callback(null, taskId + 1, value + r);
};
var edgeTask = function(taskId, value, callback){
var r = Math.floor(Math.random() * 20) + 1;
console.log("From Edge Task %d: %d", taskId, r);
// add a node task
if (taskId < max) {
function_array.splice(function_array.length-1, 0, nodeTask);
}
callback(null, taskId + 1, value + r);
};
var startTask = function(callback) {
function_array.splice(function_array.length-1, 0, nodeTask);
callback(null, 1, 0);
};
var finalTask = function(taskId, value, callback) {
callback(null, value);
};
var function_array = [startTask, finalTask];
async.waterfall(function_array, function (err, result) {
console.log("Sum is ", result);
});

mongodb data async in a for-loop with callbacks?

Here is a sample of the working async code without mongodb. The problem is, if i replace the vars (data1_nodb,...) with the db.collection.find(); function, all needed db vars received at the end and the for()-loop ends not correct. Hope someone can help. OA
var calc = new Array();
function mach1(callback){
error_buy = 0;
// some vars
for(var x_c99 = 0; x_c99 < array_temp_check0.length;x_c99++){
// some vars
calc[x_c99] = new Array();
calc[x_c99][0]= new Array();
calc[x_c99][0][0] = "dummy1";
calc[x_c99][0][1] = "dummy2";
calc[x_c99][0][2] = "dummy3";
calc[x_c99][0][3] = "dummy4";
calc[x_c99][0][4] = "dummy5";
function start_query(callback) {
data1_nodb = "data1";
data2_nodb = "data2";
data3_nodb = "data3";
data4_nodb = "data4";
calc[x_c99][0][0] = data1_nodb;
calc[x_c99][0][1] = data2_nodb;
calc[x_c99][0][2] = data3_nodb;
callback(data1_nodb,data2_nodb,etc..);
}
start_query(function() {
console.log("start_query OK!");
function start_query2(callback) {
data4_nodb = "data5";
data5_nodb = "data6";
data6_nodb = "data7";
calc[x_c99][0][3] = data4_nodb;
calc[x_c99][0][4] = data5_nodb;
callback(data5_nodb,data6_nodb,etc..);
}
start_query2(function() {
console.log("start_query2 OK!");
function start_query3(callback) {
for(...){
// do something
}
callback(vars...);
}
start_query3(function() {
console.log("start_query3 OK!");
});
});
});
}
callback(calc);
};
function mach2(callback){
mach1(function() {
console.log("mach1 OK!");
for(...){
// do something
}
});
callback(calc,error_buy);
};
mach2(function() {
console.log("mach2 OK 2!");
});
You need to work with the async nature of the collection.find() method and wait for all of them to be done. A very popular approach is to use the async module. This module allows you run several parallel tasks and wait for them to finish with its async.parallel() method:
async.parallel([
function (callback) {
db.foo.find({}, callback);
},
function (callback) {
db.bar.find({}, callback);
},
function (callback) {
db.baz.find({}, callback);
}
], function (err, results) {
// results[0] is the result of the first query, etc
});

Resources