Firebase Functions - NodeJS: Asynchronious Array handling - node.js

I´ve checked numerous posts but could not solve the issue.
I want to return the array before going on. I tried using a function with a callback but that did not work either.
My code looks as following:
exports.GetHelmets = functions.database.ref('onTrack/{userID}').onCreate(event => {
var helmets = [];
let userID = event.params.userID;
let friendRef = admin.database().ref("friends").child(userID);
friendRef.once("value").then(snapshot => {
snapshot.forEach(function(snapshot2){
let rider = snapshot2.val();
let riderID = rider.id;
let rhRef = admin.database().ref("User").child(riderID);
rhRef.once("value", function(snapshot3){
let rider2 = snapshot3.val();
let helmetID = rider2.helmet;
if (helmetID != "STANDARD"){
if(helmets.indexOf(helmetID) < 0){
helmets.push(helmetID);
};
};
});
});
return helmets;
}).then(helmets => {
//WORK WITH ARRAY
});
I hope you can help me, thanks!

You want the last then() to get all the inner data, each of which requires its own call to once(). In such a case, you'll want to use Promise.all() to wait for all the onces.
exports.GetHelmets = functions.database.ref('onTrack/{userID}').onCreate(event => {
let userID = event.params.userID;
let friendRef = admin.database().ref("friends").child(userID);
friendRef.once("value").then(snapshot => {
var promises = []l
snapshot.forEach(function(snapshot2){
let rider = snapshot2.val();
let riderID = rider.id;
let rhRef = admin.database().ref("User").child(riderID);
promises.push(rhRef.once("value");
});
});
return Promise.all(promises);
}).then(snapshots => {
var helmets = [];
snapshots.forEach((snapshot) => {
let rider2 = snapshot.val();
let helmetID = rider2.helmet;
if (helmetID != "STANDARD"){
if(helmets.indexOf(helmetID) < 0){
helmets.push(helmetID);
};
};
});
// WORK WITH helmets
});

Related

Facing issue with scooping in node JS

This code is showing empty object ( {} )
// declared at top
let mainData = {};
let trainStations = {};
let routes = {};
let trainNo = {};
data["data"].forEach(async (element) => {
const response2 = await fetch(
`https://india-rail.herokuapp.com/trains/getRoute?trainNo=${element["train_base"]["train_no"]}`
);
const data2 = await response2.json();
data2["data"].forEach((ele) => {
routes[ele["source_stn_code"]] = true;
});
trainNo[element["train_base"]["train_no"]] = routes;
});
console.log(trainNo);
if i do this then i will give response with data
data["data"].forEach(async (element) => {
const response2 = await fetch(
`https://india-rail.herokuapp.com/trains/getRoute?trainNo=${element["train_base"]["train_no"]}`
);
const data2 = await response2.json();
data2["data"].forEach((ele) => {
routes[ele["source_stn_code"]] = true;
});
trainNo[element["train_base"]["train_no"]] = routes;
console.log(trainNo);
});
maybe there is some scooping issue please kindly help me to solve this problem :)
Please refer here and also check this.
As a short note, using await inside a forEach() loop will give unexpected results. This is because the forEach() does not wait until the promise to settled (either fulfilled or rejected).
A simple solution for this could be using either the traditional for loop or the for..of loop.
for(let element of data["data"]){
const response2 = await fetch(
`https://india-rail.herokuapp.com/trains/getRoute?trainNo=${element["train_base"]["train_no"]}`
);
const data2 = await response2.json();
data2["data"].forEach((ele) => {
routes[ele["source_stn_code"]] = true;
});
trainNo[element["train_base"]["train_no"]] = routes;
}
console.log(trainNo);
NOTE: Make sure to wrap the above for..of loop inside an async function because the await keyword is allowed inside a function only when the function is defined with async keyword.

.push is not a function in web crawler

I am writing a node JS web crawler class, and I have encountered the following error, this.textInvertedIndex[word].push is not a function. Upon further inspection I realised that for some reason this.textInvertedIndex[word] was written as a native object, function Object({ [native code] }). For the first few iterations, by console logging this.textInvertedIndex everything seemed fine as it was an object of arrays. But then suddenly this error occurred. Is there any part of the code where I am implicitly rewriting textInvertedIndex?
Here is the relevant class:
function Crawler(queue, maxIndexSize) {
this.queue = queue;
this.maxIndexSize = maxIndexSize;
this.findChunks = () => {
let currentChunk;
let minimumDistance = Infinity;
for (i = 1; i <= this.maxIndexSize; i++) {
if (this.maxIndexSize % i === 0) {
const newDistance = Math.abs(i - 30);
if (newDistance < minimumDistance) {
minimumDistance = newDistance;
currentChunk = i;
} else {
return currentChunk
};
};
};
};
this.chunks = this.findChunks();
this.chunkSize = this.maxIndexSize / this.chunks;
this.totalWordOccurances = {};
this.imageInvertedIndex = {};
this.textInvertedIndex = {};
this.images = [];
this.sites = [];
this.seen = {};
this.write = (url, html) => {
const documentId = this.sites.length;
const website = new Website(url, html);
const title = website.title();
const content = website.content(title);
const words = content.filter(item => typeof item !== "object");
const wordsLength = words.length;
const query = new Query(words);
const individualWords = query.individualize(words);
this.seen[url] = true;
this.sites.push({
url,
title,
description: website.description()
});
for (word of individualWords) {
const normalizedTf = query.count(word) / wordsLength;
const textInvertedIndexEntry = {
documentId,
normalizedTf
};
if (this.textInvertedIndex[word]) {
this.textInvertedIndex[word].push(textInvertedIndexEntry);
} else {
this.textInvertedIndex[word] = [textInvertedIndexEntry];
};
if (this.totalWordOccurances[word]) {
this.totalWordOccurances[word] += 1;
} else {
this.totalWordOccurances[word] = 1;
};
};
for (i = 0; i < content.length; i++) {
const item = content[i];
if (typeof item === "object") {
const imageId = this.images.length;
this.images.push(item);
for (word of individualWords) {
const imageScore = getImageScore(i, word, content);
const imageInvertedIndexEntry = {
imageId,
imageScore
};
if (this.imageInvertedIndex[word]) {
this.imageInvertedIndex[word].push(imageInvertedIndexEntry);
} else {
this.imageInvertedIndex[word] = [imageInvertedIndexEntry];
};
};
};
};
};
this.crawl = async () => {
while (this.sites.length !== this.maxIndexSize) {
let nextQueue = [];
const websitesUnfiltered = await Promise.all(this.queue.map((url) => {
const website = new Website(url);
return website.request();
}));
const websitesToAdd = this.maxIndexSize - this.sites.length;
let websites = websitesUnfiltered.filter(message => message !== "Failure")
.slice(0, websitesToAdd);
for (site of websites) {
const url = site.url;
const htmlCode = site.htmlCode;
const website = new Website(url, htmlCode);
this.write(url, htmlCode);
nextQueue = nextQueue.concat(website.urls());
};
nextQueue = new Query(nextQueue.filter(url => !this.seen[url]))
.individualize();
this.queue = nextQueue;
};
};
};
Called like this
const crawler = new Crawler(["https://stanford.edu/"], 25000000);
crawler.crawl();
this.textInvertedIndex = {}; is defining an Object of which push is not a valid function. you can change it to an array by defining it as this.textInvertedIndex = []; otherwise you can add key/value entries to the object as it is defined like this: this.textInvertedIndex[key] = value;
Turns out, my key was accessing this.textInvertedIndex[word]. And word was constructor. constructor is already a built in object property so it can never be rewritten as an array with .push defined. To solve this problem, make all object keys capital, so constructor will become CONSTRUCTOR, thus making sure that already existing object properties are never called.

NodeJS scraper needing to increment page and re-run

I'm building a simple NodeJS web scraper, and I want to re-run the function like a 'for loop' until pageNum = totalNumberOfPages... im having a brain fart, and unable to re-run the function from inside itself, since it returns an array fragment and kills itself. Could someone help me overcome this obstacle? I'm pretty sure it's very simple.
I looked at this and this but didn't figure it out...
const cheerio = require("cheerio");
const axios = require("axios");
let pageNum = 0;
let siteUrl = "https://whatever.com?&page=" + pageNum + "&viewAll=true";
let productArray = [];
let vendor = [];
let productTitle = [];
let plantType = [];
let thcRange = [];
let cbdRange = [];
let price = [];
let totalNumberOfPages = undefined;
// called by getResults()
const fetchData = async () => {
const result = await axios.get(siteUrl);
return cheerio.load(result.data);
};
// this function is called from index.js
const getResults = async () => {
// >>>>>>>>>>>>>>>>>> HOW DO I RERUN FROM HERE <<<<<<<<<<<<<<<<<<<<<<<<<<<
const $ = await fetchData();
// first check how many total pages there are
totalNumberOfPages = parseInt($('.pagination li:nth-last-child(2)').text());
// use fetched data to grab elements (and their text) and push into arrays defined above
$('.product-tile__vendor').each((index, element) => {
vendor.push($(element).text());
});
$('.product-tile__title').each((index, element) => {
productTitle.push($(element).text());
});
$('.product-tile__plant-type').each((index, element) => {
plantType.push($(element).text());
});
$('.product-tile__properties li:nth-child(2) p').each((index, element) => {
thcRange.push($(element).text());
});
$('.product-tile__properties li:nth-child(3) p').each((index, element) => {
cbdRange.push($(element).text());
});
$('.product-tile__price').each((index, element) => {
price.push($(element).text());
});
// increment page number to get more products if the page count is less than total number of pages
if (pageNum < totalNumberOfPages) {
pageNum ++;
};
//Convert to an array so that we can sort the results.
productArray.push ({
vendors: [...vendor],
productTitle: [...productTitle],
plantType: [...plantType],
thcRange: [...thcRange],
cbdRange: [...cbdRange],
price: [...price],
pageNum
});
// >>>>>>>>>>>>>>>>>> UNTIL HERE I THINK <<<<<<<<<<<<<<<<<<<<<<<<<<<
return productArray;
};
module.exports = getResults;
you can use recursion concept in your code:
which means the function itself will call itself
so what you can do is
const getResults = async () => {
// >>>>>>>>>>>>>>>>>> HOW DO I RERUN FROM HERE <<<<<<<<<<<<<<<<<<<<<<<<<<<
const $ = await fetchData();
// first check how many total pages there are
totalNumberOfPages = parseInt($('.pagination li:nth-last-child(2)').text());
// use fetched data to grab elements (and their text) and push into arrays defined above
$('.product-tile__vendor').each((index, element) => {
vendor.push($(element).text());
});
$('.product-tile__title').each((index, element) => {
productTitle.push($(element).text());
});
$('.product-tile__plant-type').each((index, element) => {
plantType.push($(element).text());
});
$('.product-tile__properties li:nth-child(2) p').each((index, element) => {
thcRange.push($(element).text());
});
$('.product-tile__properties li:nth-child(3) p').each((index, element) => {
cbdRange.push($(element).text());
});
$('.product-tile__price').each((index, element) => {
price.push($(element).text());
});
// increment page number to get more products if the page count is less than total number of pages
if (pageNum < totalNumberOfPages) {
pageNum ++;
};
//Convert to an array so that we can sort the results.
productArray.push ({
vendors: [...vendor],
productTitle: [...productTitle],
plantType: [...plantType],
thcRange: [...thcRange],
cbdRange: [...cbdRange],
price: [...price],
pageNum
});
// >>>>>>>>>>>>>>>>>> UNTIL HERE I THINK <<<<<<<<<<<<<<<<<<<<<<<<<<<
if(pageNum >= totalNumberOfPages) getResults()
return productArray;
};

making node wait for db call to get completed

I just started writing node.js code.
I'm writing a code that extracts data from a pdf file, cleans it up and stores it in a database (using couchdb and accessing that using nano library).
The problem is that the calls are being made asynchronously... so the database get calls (i make some get calls to get a few affiliation files during the clean up) get completed only after the program runs resulting in variables being undefined. is there any way around this?
I've reproduced my code below
const fs = require('fs');
const os = require('os');
var couchDB = require('couch-db').CouchDB;
var pdf_table_extractor = require('pdf-table-extractor');
const filename = "PQ-PRI-0005-1806-01-0000_quoteSlipForLIBVIDGI1.pdf"
var nano = require('nano')('https://couchadmin:difficulttoguessmypassword#dbdev.perilwise.com');
var server = new couchDB('https://db.url.com');
server.auth("admin","admin");
var db = nano.db.use('pwfb');
var temp = [];
//New callView function
async function callView(){
try{
const doc = await view('liabilitymdm','pi');
for (var i =0; i<doc.rows.length;i++){
tmp.push(doc.rows[i]);
};
return doc;
} catch(e){
console.log(e);
};
};
function suc(result){
let ttmp = [];
console.log(result);
var pageTables = result.pageTables;
var firstPageTables = pageTables[0].tables;
ttmp = callView();
//this console log shows Promise { <pending> }
console.log(ttmp)
for (var k = 0; k < firstPageTables.length; k++) {
var temp = firstPageTables[k];
if (temp.length > 0) {
dump.push(temp);
}
}
// console.log(dump);
var insurer = filename.substr(37,8);
read_quote_slip(insurer,dump);
}
var read_quote_slip = (insurer,data) => {
console.log("read_quote_slip correctly entered");
var finOut = {};
if (insurer === "LIBVIDGI"){
finOut.insurer = insurer;
finOut.policyType = data[2][0].replace(/Quotation for/g,"");
finOut.natureOfWork = data[13][3];
let dedpos = indexGetter(data, "Deductible")[0];
finOut.deductible = data[dedpos+1][0];
let cov = indexGetter(data, "Coverage Territory and Jurisdiction")[0];
finOut.coverageTerritory = data[cov+1][0].replace(/Territory/g,"");
finOut.coverageJurisdiction = data[cov+2][0].replace(/Jurisdiction/g,"");
let ext = indexGetter(data,"Extensions")[0];
finOut.coverage = data[ext+1][0].split(/\r?\n/);
let majexc = indexGetter(data,"Major Exclusions")[0];
finOut.exclusions = data[majexc+1][0].split(/\r?\n/);
let prdtl = indexGetter(data,"Description")[0];
let prm = premiumcompute(data,prdtl,dedpos);
finOut.premium = prm;
finCleaned = libvidgi_cleaned(finOut);
// console.log(finCleaned);
}
}
var indexGetter = (words,toFind) => {
var finindex = [];
for (var i = 0; i < words.length; i++){
for (var j = 0; j < words[i].length; j++){
if(words[i][j].indexOf(toFind) >=0 ){
finindex.push(i);
}
}
}
return finindex;
}
var premiumcompute = (data, from, to) => {
let finprem = [];
let numbop = to - from - 2;
let incr = 0;
for (var i = from+2; i < to; i++){
let pr = {};
pr.option = incr+1;
pr.sumInsured = data[i][2].replace(/ /g,"");
pr.premium = data[i][data[i].length - 1].replace(/ /g,"");
finprem.push(pr);
incr +=1;
}
return finprem;
}
var libvidgi_cleaned = (finOut) => {
return finOut;
}
var fal = (result) => {
console.log(result);
console.log("there was an error");
}
var readPDFFile = function(filename){
//Decide which insurer from the filename
// console.log(filename);
console.log(filename.substr(37,8)+"Printed on line 38");
insurer = filename.substr(37,8)
pdf_table_extractor(filename, (result) => {suc(result)} , fal);
}
var libvidgi_data_extract = (data) => {
console.log(data);
let arr = data.pageTables.tables;
for (var i = 0; i <= arr.length; i++ ){
console.log(arr[i]);
}
}
readPDFFile(filename);
This answer assumes you are using Node.js > v7.6
Since db.view accepts a callback, and you wish to wait for it to finish, one solution will be to promisify it - meaning to turn it into a promise which can be awaited. You can use a library like Bluebird or you can even use Node's builtin promisify util. Then you can rewrite callViews:
const {promisify} = require('util');
const view = promisify(db.view);
async function callView() {
try {
const doc = await view('liabilitymdm', 'pi');
// the async operation is now guaranteed to be done
// (if there is an error it will be caught by the catch clause)
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
} catch (e) {
}
}
If you are not using Node.js > v7.6 (and cannot use async\await you can still utilize promises, by using their then method:
const {promisify} = require('util');
const view = promisify(db.view);
function callView() {
view('liabilitymdm', 'pi')
.then(doc => {
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
return temp;
})
.then(temp => {
console.log(temp);
})
.catch(e => {});
}
Notice how the first then is returning something which is used in a later then.
To make Node run asynchronously, you can use the keywords async and await.
They work like this:
async function doSomething () {
const formattedData = formatData();
const result = await db.postToDatabase(formattedData);
// the below will not happen until the above line is finished
doSomethingElse(result);
}
It's pretty simple in Node to get functions to execute asynchronously. Just put the async keyword at the beginning of the function definition and then put await in front of anything that you want to block execution until completed.

Using Async/Await in WATSON Nodejs SDK

I am building a chatbot with WATSON API where I use the async/await method in order to fetch the data from MongoDB and attain the result, which then I send it back to the user.
The function artpromise is the promise that collects data from mongo DB. And the function randomartist is a function that fetches 3 random document from the DB. However, the WATSON BLUEMIX Cloud service supports Nodejs SDK of 6.1.3 which does not support the async method. Is there any way to update the SDK version on Blumix or should I use a difference approach in fetching data from the server?
let getConversationResponse = (message, context) => {
let payload = {
workspace_id: process.env.WORKSPACE_ID,
context: context || {},
input: message || {}
};
payload = preProcess(payload);
return new Promise((resolved, rejected) => {
// Send the input to the conversation service
conversation.message(payload, async function(err, data) {
if (err) {
rejected(err);
}
else{
if(data.context.type == 'ask'){
let artist = data.context.name;
let result = await artpromise(artist);
console.log(result);
data.context.name = result[0].name;
data.context.nationality = result[0].nationality;
data.context.birth = result[0].years;
data.context.url = result[0].art_link;
data.output.text = data.context.name+' is a '+data.context.nationality+' artist from '+data.context.birth+'. Check out a painting at '+data.context.url;
}
else if(data.context.type == 'random_artist'){
let result = await randomArtist();
console.log(result);
data.output.text = 'Let\'s find some random artists for you! \n'+result;
}
let processed = postProcess(data);
if(processed){
// return 값이 Promise 일 경우
if(typeof processed.then === 'function'){
processed.then(data => {
resolved(data);
}).catch(err => {
rejected(err);
})
}
// return 값이 변경된 data일 경우
else{
resolved(processed);
}
}
else{
// return 값이 없을 경우
resolved(data);
}
}
});
})
}
Using Node's util.promisify() utility, you can transform a callback-style function into a Promise-based one.
Somewhere outside of your getConversationResponse-function, assign it to a local variable:
const util = require('util');
const messagePromise = util.promisify(conversation.message);
And use that function instead. Something like this should work:
const util = require('util');
const messagePromise = util.promisify(conversation.message);
let getConversationResponse = async (message, context) => {
let payload = preprocess({
workspace_id: process.env.WORKSPACE_ID,
context: context || {},
input: message || {}
});
let data = await messagePromise(payload);
if (data.context.type == 'ask') {
let artist = data.context.name;
let result = await artpromise(artist);
console.log(result)
data.context.name = result[0].name;
data.context.nationality = result[0].nationality;
data.context.birth = result[0].years;
data.context.url = result[0].art_link;
data.output.text = data.context.name+' is a '+data.context.nationality+' artist from '+data.context.birth+'. Check out a painting at '+data.context.url;
} else if (data.context.type == 'random_artist'){
let result = await randomArtist();
console.log(result);
data.output.text = 'Let\'s find some random artists for you! \n'+result;
}
return postProcess(data) || data;
};
Note that if the return value of postProcess is falsy, it will return the data variable instead. Additionally, an async function always returns a Promise, so to call this function, you'll do:
getConversationResponse(message, context).then((data) => {
// Do something with the data
}).catch((e) => {
// Handle the error!
});
or if you call it from another async function:
let data = await getConversationResponse(message, context);
or if you need to specifically catch errors in the calling async function:
try {
let data = await getConversationResponse(message, context);
} catch (e) {
// Handle error
}
Just like regular synchronous code, any error thrown in the function call chain "trickles up" to the top-most callee. If you're confused about this, I suggest reading up on error handling.
If you want to use the Watson API in an async Promise-based fashion throughout your code, it might be feasible to write a small wrapper library and use that directly instead.
A Promise-only implementation:
const util = require('util');
const messagePromise = util.promisify(conversation.message);
let getConversationResponse = (message, context) => {
let payload = preprocess({
workspace_id: process.env.WORKSPACE_ID,
context: context || {},
input: message || {}
});
return messagePromise(payload).then((data) => {
if (data.context.type == 'ask') {
let artist = data.context.name;
return artpromise(artist).then((result) => {
data.context.name = result[0].name;
data.context.nationality = result[0].nationality;
data.context.birth = result[0].years;
data.context.url = result[0].art_link;
data.output.text = data.context.name+' is a '+data.context.nationality+' artist from '+data.context.birth+'. Check out a painting at '+data.context.url;
return data;
});
} else if (data.context.type == 'random_artist') {
return randomArtist().then((result) => {
data.output.text = 'Let\'s find some random artists for you! \n' + result;
return data;
});
}
}).then((data) => {
return postProcess(data) || data;
});
};
Calling it is the exact same as the async/await implementation.

Resources