Async and Await are not working as expected. Please correct me where I am doing wrong in code.
I am reading data (url, pagelimit, company)from excel and by using switch(), I am navigating to the service.
I have to wait till I get the response from this function cnbservice.GetcnbOpenings(url, pageLimit,company), store the response to global array and call this function mdsservice.GetMdsOpenings(url, pageLimit,company), append the results to the global array.
const readexcel = async (request, response) => {
const workbook = XLSX.readFile('file.xlsx');
const sheetnamelist = workbook.SheetNames;
var xldata = XLSX.utils.sheet_to_json(workbook.Sheets[sheetnamelist[0]]);
dataarray =[];
for (i = 0; i < xldata.length; i++) {
company = xldata[i].company;
url = xldata[i].careers_link_url;
pageLimit = xldata[i].pagelimit;
switch(company){
case process.env.cnb_company_name:
const arr = await cnbservice.GetcnbOpenings(url, pageLimit,company)
if(arr !== undefined){
dataarray.push(arr);
}
break;
case process.env.mds_company_name:
const arr1 = await mdsservice.GetMdsOpenings(url, pageLimit,company)
if(arr1 !== undefined){
dataarray.push(arr1);
}
break;
case "default":
console.log("Company Name not matching with any of the services")
}
}
}
You are running await code inside standard for loop which will not work synchronously. to run async/await inside a for loop you should use for...of loop.
for(let element of array){
//await call
}
after making following changes your code will work as expected.
const readexcel = async (request, response) => {
const workbook = XLSX.readFile('file.xlsx');
const sheetnamelist = workbook.SheetNames;
var xldata = XLSX.utils.sheet_to_json(workbook.Sheets[sheetnamelist[0]]);
dataarray = [];
for (let element of xldata) {
company = element.company;
url = element.careers_link_url;
pageLimit = element.pagelimit;
switch (company) {
case process.env.cnb_company_name:
const arr = await cnbservice.GetcnbOpenings(url, pageLimit, company)
if (arr !== undefined) {
dataarray.push(arr);
}
break;
case process.env.mds_company_name:
const arr1 = await mdsservice.GetMdsOpenings(url, pageLimit, company)
if (arr1 !== undefined) {
dataarray.push(arr1);
}
break;
case "default":
console.log("Company Name not matching with any of the services")
}
}
}
Related
I am writing a node JS web crawler class, and I have encountered the following error, this.textInvertedIndex[word].push is not a function. Upon further inspection I realised that for some reason this.textInvertedIndex[word] was written as a native object, function Object({ [native code] }). For the first few iterations, by console logging this.textInvertedIndex everything seemed fine as it was an object of arrays. But then suddenly this error occurred. Is there any part of the code where I am implicitly rewriting textInvertedIndex?
Here is the relevant class:
function Crawler(queue, maxIndexSize) {
this.queue = queue;
this.maxIndexSize = maxIndexSize;
this.findChunks = () => {
let currentChunk;
let minimumDistance = Infinity;
for (i = 1; i <= this.maxIndexSize; i++) {
if (this.maxIndexSize % i === 0) {
const newDistance = Math.abs(i - 30);
if (newDistance < minimumDistance) {
minimumDistance = newDistance;
currentChunk = i;
} else {
return currentChunk
};
};
};
};
this.chunks = this.findChunks();
this.chunkSize = this.maxIndexSize / this.chunks;
this.totalWordOccurances = {};
this.imageInvertedIndex = {};
this.textInvertedIndex = {};
this.images = [];
this.sites = [];
this.seen = {};
this.write = (url, html) => {
const documentId = this.sites.length;
const website = new Website(url, html);
const title = website.title();
const content = website.content(title);
const words = content.filter(item => typeof item !== "object");
const wordsLength = words.length;
const query = new Query(words);
const individualWords = query.individualize(words);
this.seen[url] = true;
this.sites.push({
url,
title,
description: website.description()
});
for (word of individualWords) {
const normalizedTf = query.count(word) / wordsLength;
const textInvertedIndexEntry = {
documentId,
normalizedTf
};
if (this.textInvertedIndex[word]) {
this.textInvertedIndex[word].push(textInvertedIndexEntry);
} else {
this.textInvertedIndex[word] = [textInvertedIndexEntry];
};
if (this.totalWordOccurances[word]) {
this.totalWordOccurances[word] += 1;
} else {
this.totalWordOccurances[word] = 1;
};
};
for (i = 0; i < content.length; i++) {
const item = content[i];
if (typeof item === "object") {
const imageId = this.images.length;
this.images.push(item);
for (word of individualWords) {
const imageScore = getImageScore(i, word, content);
const imageInvertedIndexEntry = {
imageId,
imageScore
};
if (this.imageInvertedIndex[word]) {
this.imageInvertedIndex[word].push(imageInvertedIndexEntry);
} else {
this.imageInvertedIndex[word] = [imageInvertedIndexEntry];
};
};
};
};
};
this.crawl = async () => {
while (this.sites.length !== this.maxIndexSize) {
let nextQueue = [];
const websitesUnfiltered = await Promise.all(this.queue.map((url) => {
const website = new Website(url);
return website.request();
}));
const websitesToAdd = this.maxIndexSize - this.sites.length;
let websites = websitesUnfiltered.filter(message => message !== "Failure")
.slice(0, websitesToAdd);
for (site of websites) {
const url = site.url;
const htmlCode = site.htmlCode;
const website = new Website(url, htmlCode);
this.write(url, htmlCode);
nextQueue = nextQueue.concat(website.urls());
};
nextQueue = new Query(nextQueue.filter(url => !this.seen[url]))
.individualize();
this.queue = nextQueue;
};
};
};
Called like this
const crawler = new Crawler(["https://stanford.edu/"], 25000000);
crawler.crawl();
this.textInvertedIndex = {}; is defining an Object of which push is not a valid function. you can change it to an array by defining it as this.textInvertedIndex = []; otherwise you can add key/value entries to the object as it is defined like this: this.textInvertedIndex[key] = value;
Turns out, my key was accessing this.textInvertedIndex[word]. And word was constructor. constructor is already a built in object property so it can never be rewritten as an array with .push defined. To solve this problem, make all object keys capital, so constructor will become CONSTRUCTOR, thus making sure that already existing object properties are never called.
I am trying to get some CVS data from google sheet and store it into an Apify dataset.
const Apify = require('apify');
const request = require('request-promise');
Apify.main(async () => {
var URL = "https://docs.google.com/spreadsheets/d/1-auXklWqHQ-jj6AXymMPa7FLtP1eYGJGF3rprxuWitk/gviz/tq?tqx=out:csv";
const html = await request(URL);
console.log('My output:');
console.log(html);
await Apify.setValue('OUTPUT', html);
const namedDataset = await Apify.openDataset();
await namedDataset.pushData(html);
});
Here is error message:
2020-01-01T16:43:21.501Z My output:
2020-01-01T16:43:21.510Z "city","country"
2020-01-01T16:43:21.512Z "Berlin ","Germany"
2020-01-01T16:43:21.513Z "Los Angeles","United States"
2020-01-01T16:43:21.514Z "Melbourne","Australia"
2020-01-01T16:43:21.516Z "Sydney","Australia"
2020-01-01T16:43:21.517Z "London","United Kingdom"
2020-01-01T16:43:21.519Z "New York City","United States"
2020-01-01T16:43:21.614Z ERROR: The function passed to Apify.main() threw an exception: (error details: type=invalid-parameter)
2020-01-01T16:43:21.616Z ApifyClientError: Parameter "data" of type Array | Object must be provided
2020-01-01T16:43:21.617Z at exports.checkParamOrThrow (/usr/src/app/node_modules/apify-client/build/utils.js:222:15)
2020-01-01T16:43:21.619Z at Dataset.pushData (/usr/src/app/node_modules/apify/build/dataset.js:222:34)
2020-01-01T16:43:21.620Z at Apify.main (/usr/src/app/main.js:16:22)
2020-01-01T16:43:21.621Z at process._tickCallback (internal/process/next_tick.js:68:7)
A more elegant solution would be using our Google Sheets actor.
const Apify = require('apify');
Apify.main(async () => {
const spreadsheetId = '1-auXklWqHQ-jj6AXymMPa7FLtP1eYGJGF3rprxuWitk';
const sheetsActorInput = {
mode: 'read',
spreadsheetId,
};
const data = await Apify.call('lukaskrivka/google-sheets', sheetsActorInput);
const namedDataset = await Apify.openDataset('my-dataset');
await namedDataset.pushData(data);
});
The only disadvantage (also an advantage is some sense) is that you need to authorize in your first run but that is really simple.
I was able to use this somewhat hacky approach. I am sure their is a more modern elgagent approach:
const Apify = require('apify');
const request = require('request-promise');
function csvJSON(csv) { //https://stackoverflow.com/a/27979069/2330272
var lines = csv.split("\n");
var result = [];
// NOTE: If your columns contain commas in their values, you'll need
// to deal with those before doing the next step
// (you might convert them to &&& or something, then covert them back later)
// jsfiddle showing the issue https://jsfiddle.net/
var headers = lines[0].split(",");
for (var i = 1; i < lines.length; i++) {
var obj = {};
var currentline = lines[i].split(",");
for (var j = 0; j < headers.length; j++) {
obj[headers[j]] = currentline[j];
}
result.push(obj);
}
return JSON.stringify(result); //JSON
}
Apify.main(async () => {
var URL = "https://docs.google.com/spreadsheets/d/1-auXklWqHQ-jj6AXymMPa7FLtP1eYGJGF3rprxuWitk/gviz/tq?tqx=out:csv"; //test
const html = await request(URL);
const urls = csvJSON(html.replace(/\"/g, "")); // remove quotes from csv data
console.log('My output:');
const namedDataset = await Apify.openDataset();
await namedDataset.pushData(JSON.parse(urls));
});
I'm working on a simple function I have for a specific GET request triggered in the browser. The objective of this request is to make multiple queries to a mongodb (mongoose) database and then perform some calculation and structure formating on the results to send it back to the browser.
The only problem is that everything takes too long and it results in an error in the browser:
net::ERR_EMPTY_RESPONSE
to give an example of part of the function I'm trying to build here it goes:
async function getPriceByMake(makes, id) {
return new Promise(async (resolve, reject) => {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
}
resolve(pMakes);
});
}
In this function, if I leave the async / await out, I get an empty {} on the other end. Which is obviously not the objective.
I've been searching the web a little and was able to find an article pointing to this scheme:
Browser:
Initiates request
displays progress
Show result
WebServer:
Submit event
Checks for completion
Return result
BackEndApp:
Picks up event
Runs task
Returns results
My question is the following:
How can I do that with NodeJS and Express?
In this code:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
Your await isn't working because you're passing a callback to Listing.find(). When you do that, it does NOT return a promise and therefore the await does nothing useful. You get the empty response because the await doesn't work and thus you call resolve() before there's any actual data there.
Change the code to this:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
And, then the await will work properly.
You also should remove the return new Promise() wrapper. You don't want that. Just make the function async and use await and it will already return a promise.
Here's your function with the unnecessary promise wrapper removed:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
}
return pMakes;
}
Then, keep in mind that whatever code sends your actual response needs to use .then() or await when calling this async function in order to get the final result.
Your best bet to speed up this code would be to refactor either your queries or your database structure or both to not have to do N * M separate queries to get your final result. That's likely where your slowness is coming from. The biggest performance gains will probably come from reducing the number of queries you have to run here to far fewer.
Depending upon your database configuration and capabilities, it might speed things up to run the inner loop queries in parallel as shown here:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
await Promise.all(modelsArr.map(async currModel => {
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}));
}
return pMakes;
}
I have a lambda function in node.js to send a push notification.
In that function I need to iterate through my users sending a notification for each one prior to the callback.
Ideally I would like the iteration to perform in parallel.
What would be the best way to do this?
My code is currently as follows but it does not work as expected because the last user is not always the last to be handled:
var apnProvider = new apn.Provider(options);
var iterationComplete = false;
for (var j = 0; j < users.length; j++) {
if (j === (users.length - 1)) {
iterationComplete = true;
}
var deviceToken = users[j].user_device_token;
var deviceBadge = users[j].user_badge_count;
var notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
apnProvider.send(notification, [deviceToken]).then((response) => {
if (iterationComplete) {
context.succeed(event);
}
});
}
Use Promise.all instead - map each user's associated apnProvider.send call to a Promise in an array, and when all Promises in the array are resolved, call the callback:
const apnProvider = new apn.Provider(options);
const userPromises = users.map((user) => {
const deviceToken = user.user_device_token;
const deviceBadge = user.user_badge_count;
const notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
return apnProvider.send(notification, [deviceToken]);
})
Promise.all(userPromises)
.then(() => {
context.succeed(event);
})
.catch(() => {
// handle errors
});
I just started writing node.js code.
I'm writing a code that extracts data from a pdf file, cleans it up and stores it in a database (using couchdb and accessing that using nano library).
The problem is that the calls are being made asynchronously... so the database get calls (i make some get calls to get a few affiliation files during the clean up) get completed only after the program runs resulting in variables being undefined. is there any way around this?
I've reproduced my code below
const fs = require('fs');
const os = require('os');
var couchDB = require('couch-db').CouchDB;
var pdf_table_extractor = require('pdf-table-extractor');
const filename = "PQ-PRI-0005-1806-01-0000_quoteSlipForLIBVIDGI1.pdf"
var nano = require('nano')('https://couchadmin:difficulttoguessmypassword#dbdev.perilwise.com');
var server = new couchDB('https://db.url.com');
server.auth("admin","admin");
var db = nano.db.use('pwfb');
var temp = [];
//New callView function
async function callView(){
try{
const doc = await view('liabilitymdm','pi');
for (var i =0; i<doc.rows.length;i++){
tmp.push(doc.rows[i]);
};
return doc;
} catch(e){
console.log(e);
};
};
function suc(result){
let ttmp = [];
console.log(result);
var pageTables = result.pageTables;
var firstPageTables = pageTables[0].tables;
ttmp = callView();
//this console log shows Promise { <pending> }
console.log(ttmp)
for (var k = 0; k < firstPageTables.length; k++) {
var temp = firstPageTables[k];
if (temp.length > 0) {
dump.push(temp);
}
}
// console.log(dump);
var insurer = filename.substr(37,8);
read_quote_slip(insurer,dump);
}
var read_quote_slip = (insurer,data) => {
console.log("read_quote_slip correctly entered");
var finOut = {};
if (insurer === "LIBVIDGI"){
finOut.insurer = insurer;
finOut.policyType = data[2][0].replace(/Quotation for/g,"");
finOut.natureOfWork = data[13][3];
let dedpos = indexGetter(data, "Deductible")[0];
finOut.deductible = data[dedpos+1][0];
let cov = indexGetter(data, "Coverage Territory and Jurisdiction")[0];
finOut.coverageTerritory = data[cov+1][0].replace(/Territory/g,"");
finOut.coverageJurisdiction = data[cov+2][0].replace(/Jurisdiction/g,"");
let ext = indexGetter(data,"Extensions")[0];
finOut.coverage = data[ext+1][0].split(/\r?\n/);
let majexc = indexGetter(data,"Major Exclusions")[0];
finOut.exclusions = data[majexc+1][0].split(/\r?\n/);
let prdtl = indexGetter(data,"Description")[0];
let prm = premiumcompute(data,prdtl,dedpos);
finOut.premium = prm;
finCleaned = libvidgi_cleaned(finOut);
// console.log(finCleaned);
}
}
var indexGetter = (words,toFind) => {
var finindex = [];
for (var i = 0; i < words.length; i++){
for (var j = 0; j < words[i].length; j++){
if(words[i][j].indexOf(toFind) >=0 ){
finindex.push(i);
}
}
}
return finindex;
}
var premiumcompute = (data, from, to) => {
let finprem = [];
let numbop = to - from - 2;
let incr = 0;
for (var i = from+2; i < to; i++){
let pr = {};
pr.option = incr+1;
pr.sumInsured = data[i][2].replace(/ /g,"");
pr.premium = data[i][data[i].length - 1].replace(/ /g,"");
finprem.push(pr);
incr +=1;
}
return finprem;
}
var libvidgi_cleaned = (finOut) => {
return finOut;
}
var fal = (result) => {
console.log(result);
console.log("there was an error");
}
var readPDFFile = function(filename){
//Decide which insurer from the filename
// console.log(filename);
console.log(filename.substr(37,8)+"Printed on line 38");
insurer = filename.substr(37,8)
pdf_table_extractor(filename, (result) => {suc(result)} , fal);
}
var libvidgi_data_extract = (data) => {
console.log(data);
let arr = data.pageTables.tables;
for (var i = 0; i <= arr.length; i++ ){
console.log(arr[i]);
}
}
readPDFFile(filename);
This answer assumes you are using Node.js > v7.6
Since db.view accepts a callback, and you wish to wait for it to finish, one solution will be to promisify it - meaning to turn it into a promise which can be awaited. You can use a library like Bluebird or you can even use Node's builtin promisify util. Then you can rewrite callViews:
const {promisify} = require('util');
const view = promisify(db.view);
async function callView() {
try {
const doc = await view('liabilitymdm', 'pi');
// the async operation is now guaranteed to be done
// (if there is an error it will be caught by the catch clause)
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
} catch (e) {
}
}
If you are not using Node.js > v7.6 (and cannot use async\await you can still utilize promises, by using their then method:
const {promisify} = require('util');
const view = promisify(db.view);
function callView() {
view('liabilitymdm', 'pi')
.then(doc => {
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
return temp;
})
.then(temp => {
console.log(temp);
})
.catch(e => {});
}
Notice how the first then is returning something which is used in a later then.
To make Node run asynchronously, you can use the keywords async and await.
They work like this:
async function doSomething () {
const formattedData = formatData();
const result = await db.postToDatabase(formattedData);
// the below will not happen until the above line is finished
doSomethingElse(result);
}
It's pretty simple in Node to get functions to execute asynchronously. Just put the async keyword at the beginning of the function definition and then put await in front of anything that you want to block execution until completed.