I have a code where I need to upload bulk records like (50,000) from an CSV. From angular 6, we are making an restapi call from where we are passing the formdata to node server. In node portion we are looping that set of records(50,000) and we are uploading it in to our backend by splitting it like 1500. Records will be uploaded for every consecutive 1500.
So In our local it is working completely fine & we have tested uploading same 50000 records. But we have moved the same code to our SIT environment where we are facing the error. After uploading certain amount of record (20000 records), again its starts uploading from first record. This is some weird behaviour, which we couldnt test it in local. Can anyone please please suggest What I need to do?
var bulkUploadData = async function (jsonblob, res) {
var payloadjson = [];
var jsonResponse = {};
try {
for (let i = 0; i < jsonblob.length; i++) {
if(jsonblob[i].PN != null){
var JFB = {}
var arrayy = [];
if (jsonblob[i].Otyp != "L") {
JFB.srv = jsonblob[i].SA;
JFB.ptype = jsonblob[i].PTy;
JFB.ms = jsonblob[i].PN;
var a = JSON.stringify(JFB)
payloadjson.push(a);
}
else {
JFB.ms = jsonblob[i].PN
JFB.srv = jsonblob[i].SA;
JFB.ptype = jsonblob[i].PTy;
var a = JSON.stringify(JFB)
payloadjson.push(a);
}
if ((payloadjson.length % 1500) == 0) {
console.log("Inside first loop---------counter--------- ", i );
var result = await update.invokeSDK("fnname", payloadjson, res)
payloadjson = [];
await sleepTime(20)
console.log("-----sleeped for 20ms----- ",i)
}
if (jsonblob.length == i + 1 && payloadjson.length > 0) {
var result = await update.invokeSDK("fnname", payloadjson, res)
payloadjson = [];
}
}
console.log("FRKNG Length _________i: ",i);
}
jsonResponse = { "failedRecords": "fail" }
return jsonResponse;
} catch (err) {
console.log('error:----', err);
return err;
}
}
Related
Async and Await are not working as expected. Please correct me where I am doing wrong in code.
I am reading data (url, pagelimit, company)from excel and by using switch(), I am navigating to the service.
I have to wait till I get the response from this function cnbservice.GetcnbOpenings(url, pageLimit,company), store the response to global array and call this function mdsservice.GetMdsOpenings(url, pageLimit,company), append the results to the global array.
const readexcel = async (request, response) => {
const workbook = XLSX.readFile('file.xlsx');
const sheetnamelist = workbook.SheetNames;
var xldata = XLSX.utils.sheet_to_json(workbook.Sheets[sheetnamelist[0]]);
dataarray =[];
for (i = 0; i < xldata.length; i++) {
company = xldata[i].company;
url = xldata[i].careers_link_url;
pageLimit = xldata[i].pagelimit;
switch(company){
case process.env.cnb_company_name:
const arr = await cnbservice.GetcnbOpenings(url, pageLimit,company)
if(arr !== undefined){
dataarray.push(arr);
}
break;
case process.env.mds_company_name:
const arr1 = await mdsservice.GetMdsOpenings(url, pageLimit,company)
if(arr1 !== undefined){
dataarray.push(arr1);
}
break;
case "default":
console.log("Company Name not matching with any of the services")
}
}
}
You are running await code inside standard for loop which will not work synchronously. to run async/await inside a for loop you should use for...of loop.
for(let element of array){
//await call
}
after making following changes your code will work as expected.
const readexcel = async (request, response) => {
const workbook = XLSX.readFile('file.xlsx');
const sheetnamelist = workbook.SheetNames;
var xldata = XLSX.utils.sheet_to_json(workbook.Sheets[sheetnamelist[0]]);
dataarray = [];
for (let element of xldata) {
company = element.company;
url = element.careers_link_url;
pageLimit = element.pagelimit;
switch (company) {
case process.env.cnb_company_name:
const arr = await cnbservice.GetcnbOpenings(url, pageLimit, company)
if (arr !== undefined) {
dataarray.push(arr);
}
break;
case process.env.mds_company_name:
const arr1 = await mdsservice.GetMdsOpenings(url, pageLimit, company)
if (arr1 !== undefined) {
dataarray.push(arr1);
}
break;
case "default":
console.log("Company Name not matching with any of the services")
}
}
}
I have an issue with handling parallel requests with cloud functions.
My scenario is to select a driver from the db and update its status.
I do check for that status property before updating it, but when I send multiple requests (database on create triggers to be specific) within a second it doesn't seem to read the updated status property. And it always updates with the information of the last request. I also have noticed that sometimes the requests are processed altogether.
What can I do to fix these issues?
index.js
const db = app.database();
const TripManagementUtil = require('./utils').TripManagementUtil;
exports.triggerNotifications = functions.database.ref('/Trip/{pushId}').onCreate( (snapshot, context) =>
{
var newTrip = snapshot.val();
var tripKey = context.params.pushId;
var tripManagementUtil = new TripManagementUtil();
tripManagementUtil.searchDrivers(tripKey, newTrip, db);
});
utils.js
searchDrivers(tripKey, trip, db){
const results = [];
var lat = trip.pickupLocation.lat, long = trip.pickupLocation.lng;
var vehicleTypeID = trip.vehicleTypeID;
var alreadyAssigned = trip.alreadyAssigned;
var self = this;
if(alreadyAssigned == null || alreadyAssigned == 'undefined'){
alreadyAssigned = [];
}
const geofireQuery = new GeoFire(db.ref('vehicleLocation').child(vehicleTypeID + "")).query({
center: [lat, long],
radius: constants.searchRadius
})
.on('key_entered', (key, coords, distance) => {
if(alreadyAssigned.indexOf(key) == -1){
var result = {
driverID: key,
distance: distance
}
results.push(result);
}
});
setTimeout(() => {
geofireQuery.cancel();
if (results.length === 0) {
self.noDriversHandler(alreadyAssigned, tripKey, db);
} else {
results.sort((a, b) => a.distance - b.distance);
var driversAvailable = false;
var index = 0;
function checkDriver(){
db.ref().child("driver").child("available").child(results[index].driverID).once('value').then(function(vehicleSnap){
var vehicle = vehicleSnap.val();
if(!driversAvailable){
if(vehicle != null && vehicle.vehicleTypeID == vehicleTypeID
&& (vehicle.tripStatus != TripVehicleActionEnum.DriverConfirmed && vehicle.tripStatus != TripVehicleActionEnum.VehicleAssigned)
&& alreadyAssigned.indexOf(vehicle.driverID +"") === -1){
driversAvailable = true;
self.driverExistsHandler(trip, tripKey, alreadyAssigned, vehicle, db);
}
if(!driversAvailable && index + 1 == results.length){
self.noDriversHandler(alreadyAssigned, tripKey, db);
}
index++;
}
else{
index++;
checkDriver();
}
});
}
checkDriver();
}
}, 1500);
}
To write data to the database where the value is based on an existing value, you'll want to use Firebase Realtime Database transactions. For more on this, and examples, see save data transactionally in the Firebase documentation.
I just started writing node.js code.
I'm writing a code that extracts data from a pdf file, cleans it up and stores it in a database (using couchdb and accessing that using nano library).
The problem is that the calls are being made asynchronously... so the database get calls (i make some get calls to get a few affiliation files during the clean up) get completed only after the program runs resulting in variables being undefined. is there any way around this?
I've reproduced my code below
const fs = require('fs');
const os = require('os');
var couchDB = require('couch-db').CouchDB;
var pdf_table_extractor = require('pdf-table-extractor');
const filename = "PQ-PRI-0005-1806-01-0000_quoteSlipForLIBVIDGI1.pdf"
var nano = require('nano')('https://couchadmin:difficulttoguessmypassword#dbdev.perilwise.com');
var server = new couchDB('https://db.url.com');
server.auth("admin","admin");
var db = nano.db.use('pwfb');
var temp = [];
//New callView function
async function callView(){
try{
const doc = await view('liabilitymdm','pi');
for (var i =0; i<doc.rows.length;i++){
tmp.push(doc.rows[i]);
};
return doc;
} catch(e){
console.log(e);
};
};
function suc(result){
let ttmp = [];
console.log(result);
var pageTables = result.pageTables;
var firstPageTables = pageTables[0].tables;
ttmp = callView();
//this console log shows Promise { <pending> }
console.log(ttmp)
for (var k = 0; k < firstPageTables.length; k++) {
var temp = firstPageTables[k];
if (temp.length > 0) {
dump.push(temp);
}
}
// console.log(dump);
var insurer = filename.substr(37,8);
read_quote_slip(insurer,dump);
}
var read_quote_slip = (insurer,data) => {
console.log("read_quote_slip correctly entered");
var finOut = {};
if (insurer === "LIBVIDGI"){
finOut.insurer = insurer;
finOut.policyType = data[2][0].replace(/Quotation for/g,"");
finOut.natureOfWork = data[13][3];
let dedpos = indexGetter(data, "Deductible")[0];
finOut.deductible = data[dedpos+1][0];
let cov = indexGetter(data, "Coverage Territory and Jurisdiction")[0];
finOut.coverageTerritory = data[cov+1][0].replace(/Territory/g,"");
finOut.coverageJurisdiction = data[cov+2][0].replace(/Jurisdiction/g,"");
let ext = indexGetter(data,"Extensions")[0];
finOut.coverage = data[ext+1][0].split(/\r?\n/);
let majexc = indexGetter(data,"Major Exclusions")[0];
finOut.exclusions = data[majexc+1][0].split(/\r?\n/);
let prdtl = indexGetter(data,"Description")[0];
let prm = premiumcompute(data,prdtl,dedpos);
finOut.premium = prm;
finCleaned = libvidgi_cleaned(finOut);
// console.log(finCleaned);
}
}
var indexGetter = (words,toFind) => {
var finindex = [];
for (var i = 0; i < words.length; i++){
for (var j = 0; j < words[i].length; j++){
if(words[i][j].indexOf(toFind) >=0 ){
finindex.push(i);
}
}
}
return finindex;
}
var premiumcompute = (data, from, to) => {
let finprem = [];
let numbop = to - from - 2;
let incr = 0;
for (var i = from+2; i < to; i++){
let pr = {};
pr.option = incr+1;
pr.sumInsured = data[i][2].replace(/ /g,"");
pr.premium = data[i][data[i].length - 1].replace(/ /g,"");
finprem.push(pr);
incr +=1;
}
return finprem;
}
var libvidgi_cleaned = (finOut) => {
return finOut;
}
var fal = (result) => {
console.log(result);
console.log("there was an error");
}
var readPDFFile = function(filename){
//Decide which insurer from the filename
// console.log(filename);
console.log(filename.substr(37,8)+"Printed on line 38");
insurer = filename.substr(37,8)
pdf_table_extractor(filename, (result) => {suc(result)} , fal);
}
var libvidgi_data_extract = (data) => {
console.log(data);
let arr = data.pageTables.tables;
for (var i = 0; i <= arr.length; i++ ){
console.log(arr[i]);
}
}
readPDFFile(filename);
This answer assumes you are using Node.js > v7.6
Since db.view accepts a callback, and you wish to wait for it to finish, one solution will be to promisify it - meaning to turn it into a promise which can be awaited. You can use a library like Bluebird or you can even use Node's builtin promisify util. Then you can rewrite callViews:
const {promisify} = require('util');
const view = promisify(db.view);
async function callView() {
try {
const doc = await view('liabilitymdm', 'pi');
// the async operation is now guaranteed to be done
// (if there is an error it will be caught by the catch clause)
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
} catch (e) {
}
}
If you are not using Node.js > v7.6 (and cannot use async\await you can still utilize promises, by using their then method:
const {promisify} = require('util');
const view = promisify(db.view);
function callView() {
view('liabilitymdm', 'pi')
.then(doc => {
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
return temp;
})
.then(temp => {
console.log(temp);
})
.catch(e => {});
}
Notice how the first then is returning something which is used in a later then.
To make Node run asynchronously, you can use the keywords async and await.
They work like this:
async function doSomething () {
const formattedData = formatData();
const result = await db.postToDatabase(formattedData);
// the below will not happen until the above line is finished
doSomethingElse(result);
}
It's pretty simple in Node to get functions to execute asynchronously. Just put the async keyword at the beginning of the function definition and then put await in front of anything that you want to block execution until completed.
My script written in NodeJS connects to Firebase, checks my Firebase Database and even successfully sends notifications when results from my database return true... However, it only works when I run it from my local machine. I deploy it to Firebase and it will not work. Could someone please advise? Thank you.
I hate asking on here because I'm a newbie but I have spent hours tonight trying to find an answer...
INDEX.JS
// Firebase Functions
const functions = require('firebase-functions');
var admin = require("firebase-admin");
// Default admin firebase configuration
admin.initializeApp(functions.config().firebase);
// var serviceAccount = require("xxxxxx-80xxxxd-firebase-adminsdk- xxxxxxx.json");
var moment = require('moment');
var FCM = require('fcm-push');
var dateTime = require('node-datetime');
var serverKey = 'xxxxxxxxxxxxxpSELZBjQYwpZgmxxxxxxxxxxx';
var fcm = new FCM(serverKey);
//Initial function call:
exports.CheckDates = functions.https.onRequest((req, response) => {
// Get a database reference to our posts
var db = admin.database();
var ref = db.ref("records");
var userToken = '';
var itemExpires = '';
var itemName = '';
var reminded = '';
var itemCount = 0;
var counter = 1;
var itemFoundCount = 0;
var dt = dateTime.create();
var formatted = dt.format('m-d-Y');
ref.once("value", function (recordsSnapshot) {
recordsSnapshot.forEach(function (recordsSnapshot) {
var mainKey = recordsSnapshot.key;
recordsSnapshot.forEach(function (child) {
var key = child.key;
var value = child.val();
if (key == 'Account') {
userToken = value.userToken;
}
if (key == 'Items') {
recordsSnapshot.child("Items").forEach(function (itemsSnapshot) {
counter++;
if (itemFoundCount === 0) {
itemFoundCount = itemsSnapshot.numChildren();
}
var itemsChildkey = itemsSnapshot.key;
var itemsChildvalue = itemsSnapshot.val();
itemExpires = itemsChildvalue.itemExpires;
itemName = itemsChildvalue.itemName;
reminded = itemsChildvalue.reminded;
moment().format('YYYY-MM-DD');
var currentDate = moment();
var otherTime = moment(reminded);
if (typeof reminded !== 'undefined') {
if (currentDate.diff(otherTime, 'days') >= 30) {
if (currentDate.diff(itemExpires, 'days') <= 90) {
itemCount++;
console.log("Expire date is less than " +
currentDate + " by 90 days = " + (currentDate.diff(otherTime, 'days') <=
90));
db.ref("records/" + mainKey + "/Items/" +
itemsChildkey + '/reminded').set(formatted);
}
}
} else {
itemCount++;
db.ref("records/" + mainKey + "/Items/" + itemsChildkey +
`enter code here`'/reminded').set(formatted);
}
if (counter == itemFoundCount && itemCount > 0) {
console.log(itemFoundCount);
var message = {
to: userToken, // required fill with device token or
topics
notification: {
title: 'Item Expire Notification',
body: itemCount + ' is about to expire.'
}
};
//callback style
fcm.send(message, function (err, response) {
if (err) {
console.log("Something has gone wrong!");
} else {
console.log("Successfully sent with response: ",
response);
}
});
itemCount = 0;
itemFoundCount = 0;
counter = 1;
}
});
}
});
});
});
response.send(200, "ok");
}) // END exports.CheckDates
Obviously, I remove the Exports.CheckDates lines when I run it locally but it wasn't showing up at all without the exports on Firebase's console. It returns warnings locally and on Firebase console but it works on one and not the other.
Please disregard this question. The script adds the date that the last time a notification was sent to the item's "Reminded" key in the database... this prevents notifications from going out every day for the same thing..
It ran on my local machine, and did it's job but I couldn't figure out why it wouldn't run again... well, ha! At least we know that part works.
My source code is:
var _ = require('lodash');
var fs = require('fs');
var csv = require('fast-csv');
var topPostStream = fs.createWriteStream('top_posts.csv');
var topOtherStream = fs.createWriteStream('other_posts.csv');
var topCsvStream = csv.createWriteStream();
var otherCsvStream = csv.createWriteStream();
var dailyTops = {};
var headerRow = [];
topCsvStream.pipe(topPostStream);
otherCsvStream.pipe(topOtherStream);
console.log(process.argv);
csv
.fromPath('posts.csv')
.on('data', function(rowData) {
if(headerRow.length === 0) {
// Save header row in CSV's
headerRow = rowData;
topCsvStream.write(headerRow);
otherCsvStream.write(headerRow);
return;
}
if(rowData[2] === 'public' && rowData[5] > 10 && rowData[4] > 9000 && rowData[1].length < 40) {
// Save to the top_posts file
topCsvStream.write(rowData);
} else {
// Save to the other_posts file
otherCsvStream.write(rowData);
}
// Save to the daily tops
var postDate = new Date(rowData[6]);
postDate = new Date(postDate.getYear(), postDate.getMonth(), postDate.getDate());
if(!dailyTops[postDate] || parseInt(rowData[3]) > parseInt(dailyTops[postDate][3])) {
dailyTops[postDate] = rowData;
}
})
.on('end', finishWrite);
function finishWrite() {
topCsvStream.end();
otherCsvStream.end();
writeDailyList(dailyTops, headerRow);
}
function writeDailyList(dailyTops, headerRow) {
var daily = _.map(_.keys(dailyTops), function(key) {
return dailyTops[key];
});
var daily = [headerRow].concat(daily);
csv
.writeToPath('daily_top_posts.csv', daily, {headers: true})
}
I need to write unit tests to get as close to 100% coverage. My issues are that (a) it seems almost too simple to test and (b) how would I actually instrument the unit tests?
Posting my comment as an extended answer.
First of all, you need to extract your data handle functions into another module, e.g. data-handler.js (this is an example of your code, it can probably be enhanced to look better, but it just gives an idea)
module.exports = {
handler: (rowData, topCsvStream, otherCsvStream) => {
if(headerRow.length === 0) {
// Save header row in CSV's
headerRow = rowData;
topCsvStream.write(headerRow);
otherCsvStream.write(headerRow);
return;
}
if(rowData[2] === 'public' && rowData[5] > 10 && rowData[4] > 9000 && rowData[1].length < 40) {
// Save to the top_posts file
topCsvStream.write(rowData);
} else {
// Save to the other_posts file
otherCsvStream.write(rowData);
}
// Save to the daily tops
var postDate = new Date(rowData[6]);
postDate = new Date(postDate.getYear(), postDate.getMonth(), postDate.getDate());
if(!dailyTops[postDate] || parseInt(rowData[3]) > parseInt(dailyTops[postDate][3])) {
dailyTops[postDate] = rowData;
}
},
end: (topCsvStream, otherCsvStream) => {
topCsvStream.end();
otherCsvStream.end();
writeDailyList(dailyTops, headerRow);
}
}
In your main file, you will reference the module:
const handler = require("./data-handler");
csv
.fromPath('posts.csv')
.on("data", (data) => handler.handler(data, topCsvStream, otherCsvStream).on("end", () => handler.end(topCsvStream, otherCsvStream));
Now your handler code is not bound to streams and you can easily test it without them.