I'm trying to write a Node.js code that does the below.
Connect to a Salesforce instance.
Get the past 7 days, and loop through them.
Run 2 queries inside them and push the result to an Array.
Display the value in another function.
Here is my JS code.
var jsforce = require("jsforce");
var moment = require('moment');
function connectToEP() {
var main_Obj = {};
var response_Obj = {};
var pastSevenDaysArray = [];
var conn = new jsforce.Connection();
var beforeSevenDays = moment().subtract(7, 'days').format('YYYY-MM-DD');
var today = moment().startOf('day');
var i = 0;
conn.login("myUid", "myPwd").then(() => {
console.log("Connected To Dashboard");
for (var m = moment(beforeSevenDays); m.diff(today, 'days') <= 0; m.add(1, 'days')) {
conn.query("SELECT SUM(Total_ETA_of_all_tasks__c), SUM(Total_ETA__C) from Daily_Update__c where DAY_ONLY(createddate)= " + m.format('YYYY-MM-DD')).then(() => {
console.log("B1");
var z = response_Obj.aggrRes;
response_Obj.aggrRes = res;
pastSevenDaysArray.push({ z: res });
console.log("B1 Exit");
}).then(() => {
conn.query("SELECT count(Id), Task_Type__c FROM Daily_Task__c where DAY_ONLY(createddate) = " + m.format('YYYY-MM-DD') + " group by Task_Type__c").then(() => {
console.log("B2");
var z = response_Obj.aggrRes;
response_Obj.aggrRes = res;
pastSevenDaysArray.push({ z: res });
console.log("B2 Exit");
})
})
}
return Promise.resolve(pastSevenDaysArray);
}).then((data) => {
console.log(typeof data);
updateMessage(JSON.stringify(data));
console.log(typeof data);
});
}
function updateMessage(message) {
console.log("XXXXXXXXXXXX");
console.log(message);
console.log("XXXXXXXXXXXX");
}
function socketNotificationReceived() {
console.log("socket salesforce rec");
connectToEP();
}
socketNotificationReceived();
when I run this code, the output that I get is.
socket salesforce rec
Connected To Dashboard
object
XXXXXXXXXXXX
[]
XXXXXXXXXXXX
object
B1
B1
B1
B1
B1
B1
B1
B1
I'm very new to this js platform, unable to get the promises concepts :(. please let me know on were am I going wrong and how can I fix it.
An explanation of what's going is very helpful in my future projects.
Thanks
The thing I always do when I get confused is to decompose. Build the pieces one by one, and make sure each works. Trying to understand your code, I get something like this...
A function each for logging in, getting a "task sum" from the db and getting a "task count" from the db. (Task sum/count is what I guessed the queries were up to. Rename as you see fit).
var jsforce = require("jsforce");
var moment = require('moment');
function login(conn) {
return conn.login("myUid", "myPwd");
}
function queryTaskSumForDay(conn, m) {
return conn.query("SELECT SUM(Total_ETA_of_all_tasks__c), SUM(Total_ETA__C) from Daily_Update__c where DAY_ONLY(createddate)= " + m.format('YYYY-MM-DD'));
}
function queryTaskCountForDay(conn, m) {
return conn.query("SELECT count(Id), Task_Type__c FROM Daily_Task__c where DAY_ONLY(createddate) = " + m.format('YYYY-MM-DD') + " group by Task_Type__c");
}
With those working, it should be easy to get a sum and a count for a given day. Rather than returning these in an array (containing two objects that each have a "z" property as your code did), I opted for the simpler single object that has a sum and count property. You may need to change this to suit your design. Notice the use of Promise.all() to resolve two promises together...
function sumAndCountForDay(conn, m) {
let sum = queryTaskSumForDay(conn, m);
let count = queryTaskCountForDay(conn, m);
return Promise.all([sum, count]).then(results => {
return { sum: results[0], count: results[1] };
});
}
With that working, it should be easy to get an array of sum-count objects for a period of seven days using your moment logic and the Promise.all() idea...
function sumAndCountForPriorWeek(conn) {
let promises = [];
let beforeSevenDays = moment().subtract(7, 'days').format('YYYY-MM-DD');
let today = moment().startOf('day');
for (let m = moment(beforeSevenDays); m.diff(today, 'days') <= 0; m.add(1, 'days')) {
promises.push(sumAndCountForDay(conn, m));
}
return Promise.all(promises);
}
With that working (notice the pattern here?), your OP function is tiny and nearly fully tested because we tested all of it's parts...
function connectToEP() {
let conn = new jsforce.Connection();
return login(conn).then(() => {
return sumAndCountForPriorWeek(conn)
}).then(result => {
console.log(JSON.stringify(result));
return result;
}).catch(error => {
console.log('error: ' + JSON.stringify(error));
return error;
});
}
I think your general structure should be something like this. The biggest issue is not returning promises when you need to. A "for loop" of promises is a little difficult to step into, but if you can do them in parallel then the easiest thing to do is Promise.all If you need to aggregate the data before you can perform the next query then you need to do multiple Promise.all().then()'s. The reason you get an empty array [] is because your for loop creates the promises but doesn't wait until they finish.
var jsforce = require("jsforce");
var moment = require('moment');
function connectToEP() {
// connectToEP now returns a promise
return conn.login("myUid", "myPwd").then(() => {
console.log("Connected To Dashboard");
let myQueries = [];
for (start ; condition ; incrementer) {
myQueries.push( // Add all these query promises to the parallel queue
conn.query(someQuery)
.then((res) => {
return res;
})
.then((res) => {
return conn.query(someQuery).then((res) => {
return someData;
})
})
)
}
return Promise.all(myQueries); // Waits for all queries to finish...
}).then((allData) => { // allData is an array of all the promise results
return updateMessage(JSON.stringify(allData));
});
}
Related
I am new to promises. I want to read two csv files at time using promises. How to read 2 csv files parallel and then proceed to chain. I have gone through this but they have used some library called RSVP. can any one help me how to do this without using any of library functions. I want to read 2 files at a time and should able to access both files in next .then()?
file = require('fs')
// Reading file
let readFile =(filename)=>{
return new Promise(function(resolve, reject){
file.readFile(filename, 'utf8', function(err, data){
if(err){
reject(err)
}else{
resolve(data)
}
});
});
}
// Get the match id for season 2016
getMatchId=(matches)=>{
let allRows = matches.split(/\r?\n|\r/);
let match_id = []
for(let i=1; i<allRows.length-1; i++){
let x = allRows[i].split(',')
if(x[1] === '2016'){
match_id.push(parseInt((x[0])))
}
}
return match_id
}
// Final calculation to get runs per team in 2016
getExtraRunsin2016=(deliveries, match_id)=>{
let eachRow = deliveries.split(/\r?\n|\r/);
result = {}
let total = 0
for(let i=1; i<eachRow.length-1;i++){
let x = eachRow[i].split(',')
let team_name = x[3]
let runs = parseInt(x[16])
let id = parseInt(x[0])
if(match_id.indexOf(id)){
total+=runs
result[team_name] += runs
}else{
result[team_name] = 0
}
}
console.log(result, total)
}
// Promise
readFile('fileOne.csv')
.then((matchesFile)=>{
return getMatchId(matchesFile)
})
.then((data)=>{
readFile('fileTwo.csv')
.then((deliveries)=>{
getExtraRunsin2016(deliveries, data)
})
})
.catch(err=>{
console.log(err)
})
You can Promise.all() to combine things without using any other libraries
"use strict";
Promise.all([
readFile('fileOne.csv'),
readFile('fileTwo.csv'),
]).then((results) => {
let [rawFileOneValues, rawFileTwoValues] = results;
// Your other function to process them
}).catch(err => {
console.log(err)
});
You want to use Promise.all().
// Promise
Promise.all([
readFile('fileOne.csv').then(getMatchId),
readFile('fileTwo.csv'),
])
.then(([data, deliveries]) => getExtraRunsin2016(deliveries, data))
.catch(err => console.log(err));
I also recommend using fs-extra, which would replace your readFile implementation.
I am trying to call CompaniesHouse API and fetch companies registered between November and February. The approach I took is to pick a starting index(a company registered in November) and a stop index(a company registered in February) and loop through to get the companies registered between the start and stop index. Like so:
var needle = require("needle");
var startIdx = 11059000;
var stopIdx = 11211109;
for(idx = startIdx; idx < stopIdx; idx++)
{
needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
})
.then(function(data) {
})
.catch(function(err) {
console.log('Call the locksmith!' + err)
})
}
But this doesn't work as gives either a timeout or socket hangup error.
The API is currently in beta and some features are still yet to be implemented.
Because the for loop runs synchronously and your calls to needle() are asynchronous and therefore do not block, you end up attempting to start more than 100,000 network requests at once. This overwhelms either your local computer or the target server and you start getting socket errors.
For this many requests, you need to run them X at a time so no more than X are in flight at the same time. To maximize performance, you will have to figure out what value of X you want to use because it will depend upon the target server and how it handles lots of simultaneous requests. It is generally safe to start with a value of 5 and then increase it from there to test higher values.
If you were processing an array, there are a number of pre-built options to run X requests at once. The simplest is to use a pre-built concurrency management operation such as Bluebird. Or you can write your own. You can see examples of both here: Make several requests to an API that can only handle 20 request a minute
But, since you are not processing an array, but are just incrementing a number for each successive request, I couldn't find a pre-built option that does that. So, I wrote a general purpose one where you can fill in the function that will increment your index:
// fn gets called on each iteration - must return a promise
// limit is max number of requests to be in flight at once
// cnt is number of times to call fn
// options is optional and can be {continueOnError: true}
// runN returns a promise that resolves with results array.
// If continueOnError is set, then results array
// contains error values too (presumed to be instanceof Error so caller can discern
// them from regular values)
function runN(fn, limit, cnt, options = {}) {
return new Promise((resolve, reject) => {
let inFlightCntr = 0;
let results = [];
let cntr = 0;
let doneCnt = 0;
function run() {
while (inFlightCntr < limit && cntr < cnt) {
let resultIndex = cntr++;
++inFlightCntr;
fn().then(result => {
--inFlightCntr;
++doneCnt;
results[resultIndex] = result;
run(); // run any more that still need to be run
}).catch(err => {
--inFlightCntr;
++doneCnt;
if (options.continueOnError) {
// assumes error is instanceof Error so caller can tell the
// difference between a genuine result and an error
results[resultIndex] = err;
run(); // run any more that still need to be run
} else {
reject(err);
}
});
}
if (doneCnt === cnt) {
resolve(results);
}
}
run();
});
}
Then, you could use this like this:
const needle = require("needle");
const startIdx = 11059000;
const stopIdx = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;
runN(function() {
let idx = idxCntr++;
return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
});
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
To minimize memory use, you can use a .then() handler on your call to needle() and trim down the response to only what you need in the final array:
const needle = require("needle");
const startIdx = 11059000;
const stopIdx = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;
runN(function() {
let idx = idxCntr++;
return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
}).then(response => {
// construct the smallest possible response here and then return it
// to minimize memory use for your 100,000+ requests
return response.someProperty;
});
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
var needle = require("needle");
var startIdx = 11059000;
var stopIdx = 11211109;
const promises = [];
for(idx = startIdx; idx < stopIdx; idx++)
{
promises.push(
needle('get', "https://api.companieshouse.gov.uk/company/"+idx, {
username: key,password:""
})
)
}
Promise.all(promises).then(results => {console.log(results);}).catch(err => console.log(err));
A simple Promise.all implementation can help.
How can I get a return after I loop through all my firebase results to properly finish my cloud function?
var count = 0;
return ref.child('/records').once('value').then(snap=>{
snap.forEach(snapChild=>{
var ucName = ${snapChild.val().name.toUpperCase();
var update = {'name':ucName);
ref.child(`/records/${snapChild.key}`).update(update).then(()=>{
count++;
res.write(`<p>${ucName} updated</p>`);
});
})
}).then(()=>{
return res.end(`<p>end of list (${count} records)</p>`);
})
It actually does what it is supposed to do but the counter stays at 0 and I get an error 'write after end' - I guess due to the missing return from forEach.
It's because the last then callback is called, when snap children is not processed. You need to use Array#map to produce an array of promises, and Promise.all to wait until all promises are not resolved:
var count = 0;
return ref
.child('/records').once('value')
.then(snap => {
let ops = snap.map(snapChild => {
var ucName = ${snapChild.val().name.toUpperCase();
var update = {'name':ucName);
return ref.child(`/records/${snapChild.key}`).update(update).then(() => {
count++;
res.write(`<p>${ucName} updated</p>`);
});
});
return Promise.all(ops);
})
.then(()=>{
return res.end(`<p>end of list (${count} records)</p>`);
});
I am trying write a cron function in nodejs which fetches user_ids of all the users from the db and then I want to parse through each user_id.
Here is my code :
cron.schedule('43 11 * * *', function(){
var now = moment()
var formatted = now.format('YYYY-MM-DD HH:mm:ss')
console.log('Starting the cron boss!');
var dbSelectPromise = function(db, sql1) {
return new Promise(function(resolve, reject) {
db.select(sql1, function(err, data) {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
var users =[]
var sql = "select distinct(user_id) from user_level_task"
dbSelectPromise(db,sql).then(function(secondResult){
for(i=0;i<secondResult.length;i++){
var sql1 = "select max(level_id) as level from user_level_task where user_id ="+secondResult[i].user_id
dbSelectPromise(db,sql1).then(function(thirdResult){
console.log(thirdResult)
console.log(current)
var sql2 = "select task_id form user_level_task where user_id = '"+secondResult[i].user_id+"' and level_id = '"+thirdResult[0].level+"' "
dbSelectPromise(db,sql2).then(function(fourthResult){
var leng = fourthResult.length
for(i=0;i<leng;i++){
console.log(fourthResult[i])
}
})
})
}
})
});
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
I think what's happening is that i is no longer the same when you create those new promises because the for loop is still running. It appears that what you really need is the user_id and level_id. I suggest you restructure your code a bit to reduce nesting and pass on the values you need for future promises.
Perhaps something similar to this:
dbSelectPromise(db, sql)
.then(secondResult => {
const levelPromises = [];
secondResult.forEach(res => {
levelPromises.push(getLevelByUserId(res.user_id, db));
});
return Promise.all(levelPromises); // Promise.all only if you want to handle all success cases
})
.then(result => {
result.forEach( level => {
{ userId, queryResult } = level;
// ...
})
//...
})
.catch(err => {
console.log(err);
});
function getLevelByUserId(userId, db) {
const query = `select max(level_id) as level from user_level_task where user_id = ${userId}`;
return dbselectPromise(db, query).then(result => { userId, result });
}
It creates an array of all the get level queries as promises and then passes it along to the next step using Promise.all() which will only resolve if all queries were successful. At that point, you will have access to the userId again of each result because we returned it in our new function for your next set of queries.
I think you should abstract your queries a bit further instead of using a generic dbSelectPromise and don't forget to catch() at the end otherwise you won't know what's happening.
Note: It assumes your db variable instantiated properly and your original db.select doesn't need to be returned based on whatever library you're using. There's also some new syntax there.
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
This is because you're using reinitializing i without using let. When the loop is in process, the value will be different than what you expect.
each promise is dependant on the other and need to run synchronously
For this to work, You need to chain promises. Also, you can make use of Promise.all() to execute a bunches of promises at once. Remember, Promise.all() is all or nothing.
Making those changes to your code, I get the following structure.
'use strict';
let _ = require('lodash');
function dbSelectPromise(db, sql1) {
return new Promise((resolve, reject) => {
return db.select(sql1, (err, data) => {
if (err) {
return reject(err);
}
return resolve(data);
});
});
}
function process(secondResult) {
let sql1 = "select max(level_id) as level from user_level_task where user_id =" + secondResult[i].user_id;
return dbSelectPromise(db, sql1).then(function (thirdResult) {
console.log(thirdResult);
let sql2 = "select task_id form user_level_task where user_id = '" + secondResult[i].user_id + "' and level_id = '" + thirdResult[0].level + "' ";
return dbSelectPromise(db, sql2);
});
}
function getUsers() {
let sql = "select distinct(user_id) from user_level_task";
return dbSelectPromise(db, sql).then((users) => {
return users;
}).catch(() => {
return [];
});
}
cron.schedule('43 11 * * *', function () {
var now = moment();
var formatted = now.format('YYYY-MM-DD HH:mm:ss');
getUsers().then((users) => {
let batches = _.map(users, function (user) {
return process(user);
});
return Promise.all(batches);
}).then((fourthResult) => {
console.log('Your fourthResult [[],..]', fourthResult);
}).catch(() => {
console.log('err while processing', err);
});
});
Considering that my server.js looks almost like this. Just send you the relevant part. I did not receive anything from the query, I do have data in the database, and "sendNotification" is triggered by the jQuery function in the client. Everything works and since var notis = []; returns an empty value and is what is shows as response. I know I have to debug SQL and that's what I'm going to do but anyway want to be sure of this other things. So my questions are:
1) Is a right syntax for node.js, considering this async behavior? (which I still don't understand )
2) The query always should be inside of the "io.sockets.on('connection')" part?
connection = mysql.createConnection({
host: 'localhost',
user: '',
password: "",
database: 'table' //put your database name
}),
...
connection.connect(function(err) {
// connected! (unless `err` is set)
console.log(err);
});
…
var sqlquery = function(uID,vs){
var notis = [];
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
return notis.push(data);
});
};
io.sockets.on('connection', function(socket) {
...
socket.on("sendNotification", function(data) {
var roomBName = data.room_name.replace("room-",""),
found = [];
var roomSelected = _.find(rooms, function (room) { return room.id == roomBName });
for (var person in people) {
for (var i = 0, numAttending = roomSelected.peopleAttending.length; i < numAttending; i++) {
if (people[person].name == roomSelected.peopleAttending[i]) {
found.push(person);
}
}
}
for (var i = 0, numFound = found.length; i < numFound; i++) {
**result = sqlquery(9,2);**
io.to(found[i]).emit('notification', result);
};
});
Your sqlquery() function will not accomplish anything useful. Because connection.query() is asynchronous, that means it provides the response sometime LATER after sqlquery() has already finished.
The only way in node.js to use an async result is to actually use it in the callback that provides it. You don't just stuff it into some other variable and expect the result to be there for you in other code. Instead, you use it inside that callback or you call some other function from the callback and pass it the data.
Here's one way, you could change your sqlquery() function:
var sqlquery = function(uID, vs, callback){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
callback(null, data);
});
// need to add error handling here if the query returns an error
// by calling callback(err)
};
Then, you could use the sqlquery function like this:
found.forEach(function(person, index) {
sqlquery(..., function(err, result) {
if (err) {
// handle an error here
} else {
io.to(person).emit('notification', result);
}
});
});
And, it looks like you probably have similar async issues in other places too like in connection.connect().
In addition to #jfriend00, this could be done with new ES6 feature Promise :
var sqlquery = function(uID, vs){
return new Promise(function(resolve, reject){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
resolve(data);
});
});
};
Now you can use it like :
found.forEach(function(person, index) {
sqlquery(...)
.then(function(result){
io.to(person).emit('notification', result);
});
});