reading two csv files at a time with promises nodejs - node.js

I am new to promises. I want to read two csv files at time using promises. How to read 2 csv files parallel and then proceed to chain. I have gone through this but they have used some library called RSVP. can any one help me how to do this without using any of library functions. I want to read 2 files at a time and should able to access both files in next .then()?
file = require('fs')
// Reading file
let readFile =(filename)=>{
return new Promise(function(resolve, reject){
file.readFile(filename, 'utf8', function(err, data){
if(err){
reject(err)
}else{
resolve(data)
}
});
});
}
// Get the match id for season 2016
getMatchId=(matches)=>{
let allRows = matches.split(/\r?\n|\r/);
let match_id = []
for(let i=1; i<allRows.length-1; i++){
let x = allRows[i].split(',')
if(x[1] === '2016'){
match_id.push(parseInt((x[0])))
}
}
return match_id
}
// Final calculation to get runs per team in 2016
getExtraRunsin2016=(deliveries, match_id)=>{
let eachRow = deliveries.split(/\r?\n|\r/);
result = {}
let total = 0
for(let i=1; i<eachRow.length-1;i++){
let x = eachRow[i].split(',')
let team_name = x[3]
let runs = parseInt(x[16])
let id = parseInt(x[0])
if(match_id.indexOf(id)){
total+=runs
result[team_name] += runs
}else{
result[team_name] = 0
}
}
console.log(result, total)
}
// Promise
readFile('fileOne.csv')
.then((matchesFile)=>{
return getMatchId(matchesFile)
})
.then((data)=>{
readFile('fileTwo.csv')
.then((deliveries)=>{
getExtraRunsin2016(deliveries, data)
})
})
.catch(err=>{
console.log(err)
})

You can Promise.all() to combine things without using any other libraries
"use strict";
Promise.all([
readFile('fileOne.csv'),
readFile('fileTwo.csv'),
]).then((results) => {
let [rawFileOneValues, rawFileTwoValues] = results;
// Your other function to process them
}).catch(err => {
console.log(err)
});

You want to use Promise.all().
// Promise
Promise.all([
readFile('fileOne.csv').then(getMatchId),
readFile('fileTwo.csv'),
])
.then(([data, deliveries]) => getExtraRunsin2016(deliveries, data))
.catch(err => console.log(err));
I also recommend using fs-extra, which would replace your readFile implementation.

Related

Node waiting for sqlite to return results

I'm fairly new to Node from Python so I'm used to my code executing in the order in which I write it and this has got me a little stumped.
I'm basically getting rows back from my SQLite database and performing an operation on each row then keeping a running total. I then want the final total to reply back to the user (This is a Discord bot)
I've attempted to wrap my code inside a Promise but not when it runs my bot crashes saying 'query is not defined'
async function feelingCommand(username,receivedMessage){
let sqlStatement = `SELECT Message FROM discord WHERE User = ? ORDER BY Time DESC LIMIT 2`;
let total = 0;
const query = await new Promise((resolve,reject) => {
db.run(sqlStatement, (err) => {
if(err) {
return console.log(err.message);
reject(err);
}
db.each(sqlStatement, [username], (err, row) => {
if (err) {
reject(err);
}
let score = sentiment.analyze(`${row.Message}`).comparative;
total = total + score;
});
resolve(query);
});
});
console.log(total)
}
I'd be very grateful for any help in pointing me in the right direction. To be honest, I'm surprised I got this far :)
Well my addition searching indicated that db.each is a pain to promisify so I switched to using db.all and a foreach loop with a promise
async function feelingCommand(username,receivedMessage){
let sqlStatement = `SELECT Message FROM discord WHERE User = ? ORDER BY Time DESC LIMIT 2`;
let total = 0;
const query = await new Promise((resolve,reject) => {
db.all(sqlStatement, [username], (err, rows) => {
if (err) {
reject(err);
}
rows.forEach((row) => {
console.log(row.name);
let score = sentiment.analyze(`${row.Message}`).comparative;
total = total + score;
});
resolve(total);
});
})

Create promises on queries inside a for loop

I'm trying to write a Node.js code that does the below.
Connect to a Salesforce instance.
Get the past 7 days, and loop through them.
Run 2 queries inside them and push the result to an Array.
Display the value in another function.
Here is my JS code.
var jsforce = require("jsforce");
var moment = require('moment');
function connectToEP() {
var main_Obj = {};
var response_Obj = {};
var pastSevenDaysArray = [];
var conn = new jsforce.Connection();
var beforeSevenDays = moment().subtract(7, 'days').format('YYYY-MM-DD');
var today = moment().startOf('day');
var i = 0;
conn.login("myUid", "myPwd").then(() => {
console.log("Connected To Dashboard");
for (var m = moment(beforeSevenDays); m.diff(today, 'days') <= 0; m.add(1, 'days')) {
conn.query("SELECT SUM(Total_ETA_of_all_tasks__c), SUM(Total_ETA__C) from Daily_Update__c where DAY_ONLY(createddate)= " + m.format('YYYY-MM-DD')).then(() => {
console.log("B1");
var z = response_Obj.aggrRes;
response_Obj.aggrRes = res;
pastSevenDaysArray.push({ z: res });
console.log("B1 Exit");
}).then(() => {
conn.query("SELECT count(Id), Task_Type__c FROM Daily_Task__c where DAY_ONLY(createddate) = " + m.format('YYYY-MM-DD') + " group by Task_Type__c").then(() => {
console.log("B2");
var z = response_Obj.aggrRes;
response_Obj.aggrRes = res;
pastSevenDaysArray.push({ z: res });
console.log("B2 Exit");
})
})
}
return Promise.resolve(pastSevenDaysArray);
}).then((data) => {
console.log(typeof data);
updateMessage(JSON.stringify(data));
console.log(typeof data);
});
}
function updateMessage(message) {
console.log("XXXXXXXXXXXX");
console.log(message);
console.log("XXXXXXXXXXXX");
}
function socketNotificationReceived() {
console.log("socket salesforce rec");
connectToEP();
}
socketNotificationReceived();
when I run this code, the output that I get is.
socket salesforce rec
Connected To Dashboard
object
XXXXXXXXXXXX
[]
XXXXXXXXXXXX
object
B1
B1
B1
B1
B1
B1
B1
B1
I'm very new to this js platform, unable to get the promises concepts :(. please let me know on were am I going wrong and how can I fix it.
An explanation of what's going is very helpful in my future projects.
Thanks
The thing I always do when I get confused is to decompose. Build the pieces one by one, and make sure each works. Trying to understand your code, I get something like this...
A function each for logging in, getting a "task sum" from the db and getting a "task count" from the db. (Task sum/count is what I guessed the queries were up to. Rename as you see fit).
var jsforce = require("jsforce");
var moment = require('moment');
function login(conn) {
return conn.login("myUid", "myPwd");
}
function queryTaskSumForDay(conn, m) {
return conn.query("SELECT SUM(Total_ETA_of_all_tasks__c), SUM(Total_ETA__C) from Daily_Update__c where DAY_ONLY(createddate)= " + m.format('YYYY-MM-DD'));
}
function queryTaskCountForDay(conn, m) {
return conn.query("SELECT count(Id), Task_Type__c FROM Daily_Task__c where DAY_ONLY(createddate) = " + m.format('YYYY-MM-DD') + " group by Task_Type__c");
}
With those working, it should be easy to get a sum and a count for a given day. Rather than returning these in an array (containing two objects that each have a "z" property as your code did), I opted for the simpler single object that has a sum and count property. You may need to change this to suit your design. Notice the use of Promise.all() to resolve two promises together...
function sumAndCountForDay(conn, m) {
let sum = queryTaskSumForDay(conn, m);
let count = queryTaskCountForDay(conn, m);
return Promise.all([sum, count]).then(results => {
return { sum: results[0], count: results[1] };
});
}
With that working, it should be easy to get an array of sum-count objects for a period of seven days using your moment logic and the Promise.all() idea...
function sumAndCountForPriorWeek(conn) {
let promises = [];
let beforeSevenDays = moment().subtract(7, 'days').format('YYYY-MM-DD');
let today = moment().startOf('day');
for (let m = moment(beforeSevenDays); m.diff(today, 'days') <= 0; m.add(1, 'days')) {
promises.push(sumAndCountForDay(conn, m));
}
return Promise.all(promises);
}
With that working (notice the pattern here?), your OP function is tiny and nearly fully tested because we tested all of it's parts...
function connectToEP() {
let conn = new jsforce.Connection();
return login(conn).then(() => {
return sumAndCountForPriorWeek(conn)
}).then(result => {
console.log(JSON.stringify(result));
return result;
}).catch(error => {
console.log('error: ' + JSON.stringify(error));
return error;
});
}
I think your general structure should be something like this. The biggest issue is not returning promises when you need to. A "for loop" of promises is a little difficult to step into, but if you can do them in parallel then the easiest thing to do is Promise.all If you need to aggregate the data before you can perform the next query then you need to do multiple Promise.all().then()'s. The reason you get an empty array [] is because your for loop creates the promises but doesn't wait until they finish.
var jsforce = require("jsforce");
var moment = require('moment');
function connectToEP() {
// connectToEP now returns a promise
return conn.login("myUid", "myPwd").then(() => {
console.log("Connected To Dashboard");
let myQueries = [];
for (start ; condition ; incrementer) {
myQueries.push( // Add all these query promises to the parallel queue
conn.query(someQuery)
.then((res) => {
return res;
})
.then((res) => {
return conn.query(someQuery).then((res) => {
return someData;
})
})
)
}
return Promise.all(myQueries); // Waits for all queries to finish...
}).then((allData) => { // allData is an array of all the promise results
return updateMessage(JSON.stringify(allData));
});
}

How to bulk insert in Mongoose cursor.eachAsync?

I have tried several solutions to get this working but all failed. I am reading the Mongo DB docs using cursor.eachAsync() and converting some doc fields. I need to move these docs to another collection after conversion.My idea is that after 1000 docs are processed, they should be bulk-inserted into the destination collection. This works good until the last batch of records which are less than 1000. To phrase the same problem differently, if the number of records are <1000 then they are not inserted.
1. First version - bulk insert after async()
Just like any other code, I should have docs < 1000 in bulk object after async() and should be able to insert. But I find bulk.length is 0. (I have removed those statements in the code snippet below).
```js`async function run() {
await mongoose.connect(dbPath, dbOptions);
const cursor = events.streamEvents(query, 10);
let successCounter = 0;
let bulkBatchSize = 1000;
let bulkSizeCounter = 0;
let sourceDocsCount = 80;
var bulk = eventsConvertedModel.collection.initializeOrderedBulkOp();
await cursor.eachAsync((doc) => {
let pPan = new Promise((resolve, reject) => {
getTokenSwap(doc.panTokenIdentifier, doc._id)
.then((swap) => {
resolve(swap);
});
});
let pXml = new Promise((resolve, reject) => {
let xmlObject;
getXmlObject(doc)
.then(getXmlObjectToken)
.then((newXmlString) => {
resolve(newXmlString);
})
.catch(errFromPromise1 => {
});
})
.catch(error => {
reject(error);
});
});
Promise.all([pPan, pXml])
.then(([panSwap, xml]) => {
doc.panTokenIdentifier = panSwap;
doc.eventRecordTokenText = xml;
return doc;
})
.then((newDoc) => {
successCounter++;
bulkSizeCounter++;
bulk.insert(newDoc);
if (bulkSizeCounter % bulkBatchSize == 0) {
bulk.execute()
.then(result => {
bulkSizeCounter = 0;
let msg = "Conversion- bulk insert =" + result.nInserted;
console.log(msg);
bulk = eventsConvertedModel.collection.initializeOrderedBulkOp();
Promise.resolve();
})
.catch(bulkErr => {
logger.error(bulkErr);
});
}
else {
Promise.resolve();
}
})
.catch(err => {
console.log(err);
});
});
console.log("outside-async=" + bulk.length); // always 0
console.log("run()- Docs converted in this run =" + successCounter);
process.exit(0);
}`
2. Second version (track expected number of iterations and after all iterations, change batch size to say 10).
Result - The batch size value changes but it's not reflected in bulk.insert. The records are lost.
3. Same as 2nd but insert one record at a time after bulk inserts are done.
```js
let d = eventsConvertedModel(newDoc);
d.isNew = true;
d._id = mongoose.Types.ObjectId();
d.save().then(saved => {
console.log(saved._id)
Promise.resolve();
}).catch(saveFailed => {
console.log(saveFailed);
Promise.resolve();
});
```
Result - I was getting DocumentNotFound error, so I added d.isNew = true. But for some reason only few records get inserted and many of them get lost.
I have also tried other variations using the number of expected bulk insert iterations. Finally, I changed the code to write to file (one doc at a time) but I am still wondering if there is any way to make write to DB make work.
Dependencies:
Node v8.0.0
Mongoose 5.2.2

how to promises in a loop?

I am trying write a cron function in nodejs which fetches user_ids of all the users from the db and then I want to parse through each user_id.
Here is my code :
cron.schedule('43 11 * * *', function(){
var now = moment()
var formatted = now.format('YYYY-MM-DD HH:mm:ss')
console.log('Starting the cron boss!');
var dbSelectPromise = function(db, sql1) {
return new Promise(function(resolve, reject) {
db.select(sql1, function(err, data) {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
var users =[]
var sql = "select distinct(user_id) from user_level_task"
dbSelectPromise(db,sql).then(function(secondResult){
for(i=0;i<secondResult.length;i++){
var sql1 = "select max(level_id) as level from user_level_task where user_id ="+secondResult[i].user_id
dbSelectPromise(db,sql1).then(function(thirdResult){
console.log(thirdResult)
console.log(current)
var sql2 = "select task_id form user_level_task where user_id = '"+secondResult[i].user_id+"' and level_id = '"+thirdResult[0].level+"' "
dbSelectPromise(db,sql2).then(function(fourthResult){
var leng = fourthResult.length
for(i=0;i<leng;i++){
console.log(fourthResult[i])
}
})
})
}
})
});
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
I think what's happening is that i is no longer the same when you create those new promises because the for loop is still running. It appears that what you really need is the user_id and level_id. I suggest you restructure your code a bit to reduce nesting and pass on the values you need for future promises.
Perhaps something similar to this:
dbSelectPromise(db, sql)
.then(secondResult => {
const levelPromises = [];
secondResult.forEach(res => {
levelPromises.push(getLevelByUserId(res.user_id, db));
});
return Promise.all(levelPromises); // Promise.all only if you want to handle all success cases
})
.then(result => {
result.forEach( level => {
{ userId, queryResult } = level;
// ...
})
//...
})
.catch(err => {
console.log(err);
});
function getLevelByUserId(userId, db) {
const query = `select max(level_id) as level from user_level_task where user_id = ${userId}`;
return dbselectPromise(db, query).then(result => { userId, result });
}
It creates an array of all the get level queries as promises and then passes it along to the next step using Promise.all() which will only resolve if all queries were successful. At that point, you will have access to the userId again of each result because we returned it in our new function for your next set of queries.
I think you should abstract your queries a bit further instead of using a generic dbSelectPromise and don't forget to catch() at the end otherwise you won't know what's happening.
Note: It assumes your db variable instantiated properly and your original db.select doesn't need to be returned based on whatever library you're using. There's also some new syntax there.
The problem i am facing is i cannot access value of i in third and fourth promises. Please help!
This is because you're using reinitializing i without using let. When the loop is in process, the value will be different than what you expect.
each promise is dependant on the other and need to run synchronously
For this to work, You need to chain promises. Also, you can make use of Promise.all() to execute a bunches of promises at once. Remember, Promise.all() is all or nothing.
Making those changes to your code, I get the following structure.
'use strict';
let _ = require('lodash');
function dbSelectPromise(db, sql1) {
return new Promise((resolve, reject) => {
return db.select(sql1, (err, data) => {
if (err) {
return reject(err);
}
return resolve(data);
});
});
}
function process(secondResult) {
let sql1 = "select max(level_id) as level from user_level_task where user_id =" + secondResult[i].user_id;
return dbSelectPromise(db, sql1).then(function (thirdResult) {
console.log(thirdResult);
let sql2 = "select task_id form user_level_task where user_id = '" + secondResult[i].user_id + "' and level_id = '" + thirdResult[0].level + "' ";
return dbSelectPromise(db, sql2);
});
}
function getUsers() {
let sql = "select distinct(user_id) from user_level_task";
return dbSelectPromise(db, sql).then((users) => {
return users;
}).catch(() => {
return [];
});
}
cron.schedule('43 11 * * *', function () {
var now = moment();
var formatted = now.format('YYYY-MM-DD HH:mm:ss');
getUsers().then((users) => {
let batches = _.map(users, function (user) {
return process(user);
});
return Promise.all(batches);
}).then((fourthResult) => {
console.log('Your fourthResult [[],..]', fourthResult);
}).catch(() => {
console.log('err while processing', err);
});
});

NodeJS- How to sequentially create a file and then read from?

I'm entirely new to NodeJS and this problem has been bugging me for days now. I'm pulling my hairs to find a working solution. I'm trying to get information from the database and pass it to a text file where I later read from it. I cannot do it sequentially. It always reads it first and then creates it. I don't know what way I should take to overcome the issue. Any working solution/ways will help tremendously.
My connection file that retrieves information from the database:
this.getInfo = function() {
return new Promise(function(resolve, reject){
db.query('SELECT ai_code from user_code',
function(err,rows){
if(err)
reject(err);
resolve(rows);
});
});
}
module.exports =
{
getInfo: this.getInfo
}
Functions that calls the method to receive data.
function team1Code(){
db.getInfo().then(function(result){
var code = JSON.stringify(result[0]);
var c = json2plain(code, options);
c = c.replace('Ai_code:','');
fs.writeFile('./scr1.js', c, { overwrite: true, encoding: 'ascii' },function (err) {
if (err) return console.log(err);
});
});
}
function team2Code(){
db.getInfo().then(function(result){
var code = JSON.stringify(result[1]);
var c = json2plain(code, options);
c = c.replace('Ai_code:','');
fs.writeFile('./scr2.js', c, { overwrite: true, encoding: 'ascii' },function (err) {
if (err) return console.log(err);
});
});
}
Finally, this is where we try to read the content of the files.
vmHandler.init = function(apiValues) {
team1Code();
team2Code();
// Team 1
try{
vmHandler.team1.scriptCode = fs.readFileSync('./scr1.js');
vmHandler.team1.script = new vm.Script(vmHandler.team1.scriptCode);
vmHandler.team1.sandbox = { api: new Api(apiValues, 1) }
vmHandler.team1.context = new vm.createContext(vmHandler.team1.sandbox);
}catch(err){}
// Team 2
try {
vmHandler.team2.scriptCode = fs.readFileSync('./scr2.js');
vmHandler.team2.script = new vm.Script(vmHandler.team2.scriptCode);
vmHandler.team2.sandbox = { api: new Api(apiValues, 2) }
vmHandler.team2.context = new vm.createContext(vmHandler.team2.sandbox);
} catch(err) {
console.log("ERROR: " + err);
}
};
The approach you are taking is slightly unfavorable since the function calls
team1Code();
team2Code();
doesn't make sure to accomplish before the next try-catch block gets executed. This is because both the calls are asynchronous hence the next lines get executed before they finish even though they are working with promises. promises themselves are asynchronous, what they make easy is all the code inside any then won't be executed until the promises get settled but the rest of the code will be executed as usual. So, here is the way to do your tasks with updated code.
function writeFile(fileName,data){
return new Promise(function(resolve, reject){
var code = JSON.stringify(data);
var c = json2plain(code, options);
c = c.replace('Ai_code:','');
fs.writeFile(fileName, c, { overwrite: true, encoding: 'ascii' },function (err) {
if(err)
reject(err);
resolve();
});
})
}
//Finally, this is where we try to read the content of the files.
vmHandler.init = function(apiValues) {
var files = ['./scr1.js','./scr2.js'];
db.getInfo().then(function(result){
var allPromise = [];
for(var key in files){
allPromise.push(writeFile(files[key], result[key]));
}
return Promise.all(allPromise);
}).then(function(res){
// Team 1
try{
vmHandler.team1.scriptCode = fs.readFileSync('./scr1.js');
vmHandler.team1.script = new vm.Script(vmHandler.team1.scriptCode);
vmHandler.team1.sandbox = { api: new Api(apiValues, 1) }
vmHandler.team1.context = new vm.createContext(vmHandler.team1.sandbox);
}catch(err){}
// Team 2
try {
vmHandler.team2.scriptCode = fs.readFileSync('./scr2.js');
vmHandler.team2.script = new vm.Script(vmHandler.team2.scriptCode);
vmHandler.team2.sandbox = { api: new Api(apiValues, 2) }
vmHandler.team2.context = new vm.createContext(vmHandler.team2.sandbox);
} catch(err) {
console.log("ERROR: " + err);
}
});
};
In the wmHandler.init function, you are starting 2 asynchronous operations (querying and storing) and reading from the files that the aforementioned async operations should write.
However, the file reading is performed right after the 2 async operations are started. Therefore, it is expected that the files are read before written.
To resolve this, make team1Code and team2Code return Promises of their own, and do not read the files until they have been written.
team1Code().
.then(team2Code)
.then(readFiles)
Where readFiles is the function that does the file reading, and team1Code, team2Code return Promises that resolve when the files are written.
This answer explains the asynchronous callbacks in Javascript.

Resources