mongodb data async in a for-loop with callbacks? - node.js

Here is a sample of the working async code without mongodb. The problem is, if i replace the vars (data1_nodb,...) with the db.collection.find(); function, all needed db vars received at the end and the for()-loop ends not correct. Hope someone can help. OA
var calc = new Array();
function mach1(callback){
error_buy = 0;
// some vars
for(var x_c99 = 0; x_c99 < array_temp_check0.length;x_c99++){
// some vars
calc[x_c99] = new Array();
calc[x_c99][0]= new Array();
calc[x_c99][0][0] = "dummy1";
calc[x_c99][0][1] = "dummy2";
calc[x_c99][0][2] = "dummy3";
calc[x_c99][0][3] = "dummy4";
calc[x_c99][0][4] = "dummy5";
function start_query(callback) {
data1_nodb = "data1";
data2_nodb = "data2";
data3_nodb = "data3";
data4_nodb = "data4";
calc[x_c99][0][0] = data1_nodb;
calc[x_c99][0][1] = data2_nodb;
calc[x_c99][0][2] = data3_nodb;
callback(data1_nodb,data2_nodb,etc..);
}
start_query(function() {
console.log("start_query OK!");
function start_query2(callback) {
data4_nodb = "data5";
data5_nodb = "data6";
data6_nodb = "data7";
calc[x_c99][0][3] = data4_nodb;
calc[x_c99][0][4] = data5_nodb;
callback(data5_nodb,data6_nodb,etc..);
}
start_query2(function() {
console.log("start_query2 OK!");
function start_query3(callback) {
for(...){
// do something
}
callback(vars...);
}
start_query3(function() {
console.log("start_query3 OK!");
});
});
});
}
callback(calc);
};
function mach2(callback){
mach1(function() {
console.log("mach1 OK!");
for(...){
// do something
}
});
callback(calc,error_buy);
};
mach2(function() {
console.log("mach2 OK 2!");
});

You need to work with the async nature of the collection.find() method and wait for all of them to be done. A very popular approach is to use the async module. This module allows you run several parallel tasks and wait for them to finish with its async.parallel() method:
async.parallel([
function (callback) {
db.foo.find({}, callback);
},
function (callback) {
db.bar.find({}, callback);
},
function (callback) {
db.baz.find({}, callback);
}
], function (err, results) {
// results[0] is the result of the first query, etc
});

Related

Get data from nested foreach

I'm building an app using firebase and Node.js. I need to get data from nested foreach. How to do it correctly? Need to return the results of all iterations simultaneously.
exports.userParty = function (userInfo, cb) {
var userID = userInfo.userID;
var clubID = userInfo.clubID;
var refUserParty = ref.child(userID).child('my_party_id');
var party = {};
refUserParty.orderByValue().once("value", function (snapshot) {
var party = {};
snapshot.forEach(function (partyID) {
var refParty = dbb.ref('clubs').child(clubID).child('party').child(partyID.val());
refParty.once('value', function (partyBody) {
party[partyID.val()] = partyBody.val();
//console.log(party);
});
});
cb(party); // {}
});
};
You need to call the callback after all the async functions in the forEach block have completed. You can do this using a simple counter to check all async functions are complete:
...
let completedSnapshots = 0;
snapshot.forEach(function (partyID) {
var refParty = dbb.ref('clubs').child(clubID).child('party').child(partyID.val());
refParty.once('value', function (partyBody) {
party[partyID.val()] = partyBody.val();
completedSnapshots++;
if (completedSnapshots === snapshot.val().length) {
cb(party);
}
});
});
...

Resolving a deferred Q multiple times

As I know deferred can be resolved only once and also nodejs caches the required module at the first step in order to prevent loading it every single time, but what I want to do is renewing deferred Q object each time I want to resolve/reject a returned value.
This is my server code :
// server.js
app.get('/api/posts', function(req,res,next){
var insta = require('./upRunner'); // this's the main obstacle
insta.returner().then(function(data){
// ....
};
});
and the code for ./upRunner.js is :
// upRunner.js
...
var defer = Q.defer();
upRunner();
function extractor(body) {
var $ = cheerio.load(body), scripts= [];
$('script').each(function(i, elem){
scripts[i] = $(this).text();
});
var str = scripts[6],
newData = JSON.parse(str.substring(str.indexOf("{"), str.lastIndexOf(";"))).entry_data.TagPage[0].tag.media.page_info;
grabber(str);
return newData;
}
function grabber(str) {
newData = JSON.parse(str.substring(str.indexOf("{"), str.lastIndexOf(";"))).entry_data.TagPage[0].tag.top_posts.nodes;
newData.sort(dynamicSort('-date'));
newData.forEach(function(elem,index,array){
if (instaImages.length >= 10) {
defer.resolve(instaImages);
} else {
instaImages.push(elem);
}
});
}
function upRunner(newData){
profilePage = !(tagPage = URL.includes("/tags/") ? true : false);
if (!newData) {
request(URL,{jar:true}, function(err, resp, body){
var $ = cheerio.load(body), scripts= [];
$('script').each(function(i, elem){
scripts[i] = $(this).text();
});
var str = scripts[6],
newData = JSON.parse(str.substring(str.indexOf("{"), str.lastIndexOf(";"))).entry_data.TagPage[0].tag.media.page_info;
upRunner(newData);
});
} else {
if (newData.has_next_page) {
requester(URL, newData.end_cursor).then(function(body){
newData = extractor(body);
upRunner(newData);
});
} else {
console.log('it\'s finished\n');
}
function returner() {
return deferred.promise;
}
exports.returner = returner;
As you see I'm almost renewing upRunner returner deferred promise each time server get /api/posts address, but the problem is the deferred object still return the old resolved value.
There is the grabber function which resolve value, so defer can not be localized in a single function.
Try to initialized deferred value locally to get new resolve value.
function returner() {
var defer= $q.defer();
return deferred.promise;
};
exports.returner = returner;

Variable precedence (global in node js?)

"use strict";
var Tabletop = require("tabletop");
var base64 = require('base-64');
Tabletop.init( { key: 'xxxxxg46hgfjd',
callback: showInfo,
simpleSheet: true } )
function showInfo(data, tabletop) {
console.log(data);
console.log(base64.encode(data));
}
var vGlobals = {
dataString: base64.encode(data)
};
module.exports = vGlobals;
How can I access the data variable from showInfo, to use in vGlobals? It says that it hasn't been defined.
Your approach is wrong, you can't do it this way because TableTop call your callback asynchronously.
My suggestion (a quick one) :
var dataString = null;
module.exports = function(cb) {
if (dataString == null)
Tabletop.init({
key: 'xxxxxg46hgfjd',
callback: function(data, tabletop) {
dataString = base64.encode(data);
cb(dataString);
},
simpleSheet: true
});
else cb(dataString);
};
And to get the data :
var dataManager = require('./myfile');
dataManager(function(dataString) {
//here is your data do what you want with it
})
You should look/learn more about node/javascript and asynchronous/event-driven programing.

What is the best way to loop bluebird promises

Now I've working on NodeJS and Sequelize to query and process an database data.
I've call findAll from Table1 and I want to query each rows to apply some data to Table2 then I want to add all data to array before send output, I did like this
var last_promise;
var output_results = {};
Table1Model.findAll()
.then(function(results1)
{
for (var i = 0; i < results1.length; ++i)
{
var result1 = results1[i];
output_results[result1.id] = result1;
var add_promise = Table2Model
.create({
id_from_table1: result1.id,
data_from_table1: result1.data
});
.then(function(result2) {
output_results[result2.id_from_table1].data2 = result2;
});
if (last_promise)
{
last_promise.then(function()
{
return add_promise;
});
} else {
last_promise = add_promise;
}
}
}
}
last_promise.then(function() {
return output_results;
}
I want to know that there any better way to execute promises sequentially in a loop like this ?
It looks like you can do that with .all() method:
Table1Model
.findAll()
.then(function(results1) {
return Promise.all(results1.map(function(result) {
return Table2Model
.create({
id_from_table1: result1.id,
data_from_table1: result1.data
})
.then(function(result2) {
...
});
}));
})
.then(function(output_results) {
});

Generic wrapper to harmonise functions to async style

I would love to write a generic wrapper that takes a function, and returns the "async-style" version of that function IF it wasn't async to start with.
Trouble is, there is no easy way to know whether the call is sync or async. So... this basically "cannot be done". Right?
(Note that the wrapper should harmonise sync functions to async style, and LEAVE async functions alone)
var wrapper = function( fn ){
return function(){
var args = Array.prototype.splice.call(arguments, 0);
var cb = args[ args.length - 1 ];
// ?!?!?!?!?
// I cannot actually tell if `fn` is sync
// or async, and cannot determine it!
console.log( fn.toString() );
}
}
var f1Async = wrapper( function( arg, next ){
next( null, 'async' + arg );
})
var f2Sync = wrapper( function( arg ){
return 'sync' + arg;
})
f1Async( "some", function(err, ret ){
console.log( ret );
});
f2Sync( "some other", function(err, ret ){
console.log( ret );
});
You cannot find out what the accepted arguments of a function are, so you cannot find out if it takes a callback or not.
In javascript there is no way to check if the last argument of a function is a function, because in javascript you do not define the types of your arguments.
My solution works by getting a list of the parameters in the function, then using a RegExp to see if that parameter is used as a function. Also, in the case that the callback is not being used directly (like passing it to something else), it has a list of argument names to be considered as a callback.
And the code is:
var STRIP_COMMENTS = /((\/\/.*$)|(\/\*[\s\S]*?\*\/))/mg;
var CALLBACK_NAMES = [ "next", "done", "callback", "cb"];
function getParamNames(func) {
var fnStr = func.toString().replace(STRIP_COMMENTS, '')
var result = fnStr.slice(fnStr.indexOf('(')+1, fnStr.indexOf(')')).match(/([^\s,]+)/g)
if(result === null)
result = []
return result
}
function checkIfParamIsFunction(func, paramName){
var fnStr = func.toString();
if (fnStr.replace(new RegExp("(" + paramName + "\s*\([A-Za-z0-9,\.]*\)?!{|" + paramName + ".apply\([A-Za-z0-9,\.]*\)|" + paramName + ".call\([A-Za-z0-9,\.]*\))", ""), "{<._|/}") != fnStr) { // Remove All Calls to the arg as a function, then check to see if anything was changed.
return true;
} else {
return false;
}
}
function makeAsync(func) {
var paramNames = getParamNames(func);
if (checkIfParamIsFunction(func, paramNames[paramNames.length - 1])
|| CALLBACK_NAMES.indexOf(paramNames[paramNames.length - 1]) != -1) {
// Function Is Already async
return func;
} else {
return function () {
var args = Array.prototype.slice.call(arguments);
var cb = args.pop();
cb(func.apply(this, args));
}
}
}
function test1(a){
return (a+' test1');
};
function test2(a, callback){
return callback(a+' test2')
};
function main(){
var tested1 = makeAsync(test1);
var tested2 = makeAsync(test2);
tested1('hello', function(output){
console.log('holy shit it\'s now async!!');
console.log(output);
});
tested2('world', function(output){
console.log('this one was already async tho');
console.log(output);
});
}
main();
Simply call makeAsync(function) and it will return an async function. This will work if you use function.apply or .call.
It simply cannot be done. End of story.
Though, this is not the answer, but a good alternative. I have provided example of browser based JavaScript but same class can be used on Node as well.
To solve this problem, promises were developed. However we use a modified version of promise as follow.
function AtomPromise(f)
{
// Success callbacks
this.sc = [];
// Error callbacks
this.ec = [];
this.i = f;
this.state = 'ready';
}
AtomPromise.prototype ={
finish: function(s,r) {
this.result = r;
var c = s ? this.sc : this.ec;
this.state = s ? 'done' : 'failed' ;
for(var i=o ; i< c.length; i++){
c[i](this);
}
},
invoke: function(f) {
this.state = 'invoking';
this.i(this);
},
then: function(f) {
this.sc.push(f);
},
failed: function(f){
this.ec.push(f);
},
value: function(v) {
if(v !== undefined ) this.result = v;
return this.result;
},
pushValue: function(v) {
var _this = this;
setTimeout(100, function () {
_this.finish(true, v);
});
}
}
//wrap jQuery AJAX
AtomPromise.ajax = function( url, options ) {
return new AtomPromise(function (ap){
$.ajax(url, options)
.then( function(r){ ap.finish(true, r); })
.failed( function (){ ap.finish( false, arguments) });
}) ;
}
//Wrape sync function
AtomPromise.localStorage = function( key ) {
return new AtomPromise(function (ap){
var v = localStorage[key];
ap.pushValue(v);
}) ;
}
// Calling Sequence
AtomPromise.ajax( 'Url' ).then( function(ap) {
alert(ap.value());
}).invoke();
AtomPromise.localStorage( 'Url' ).then( function(ap) {
alert(ap.value());
}).invoke();
Both functions are now asynchronous. Push Value method makes result route through setTimeout that makes further calls asynchronous.
This is used in Web Atoms JS to wrape async code into single attribute and by following one pattern you can get rid of async callback hell. http://webatomsjs.neurospeech.com/docs/#page=concepts%2Fatom-promise.html
Disclaimer: I am author of Web Atoms JS.

Resources