I'm trying to learn to use promise.
In this case i want to make a web scraper . I'm using request-promise inside this code.
Actually the scraper is working fine. I have missunderstand how to use the return value inside then(). I am also have googled it but no luck and still get stuck. Here is my code.
let categories = scraper.getCategories(promoUrl);
categories.then(function (val) {
let promoPerCategory = [];
for (let i = 0, len = val.length; i < len; i++) {
let json = {
category: val[i].category,
url: val[i].url,
promo: []
}
scraper.getPromoPerCategories(val[i].url)
.then(function (allPromo) {
for (let j = 0, lenResult = allPromo.length; j < lenResult; j++) {
json.promo.push({
imageUrl: allPromo[j].imageUrl,
merchantName: allPromo[j].merchantName,
promoTitle: allPromo[j].promoTitle,
validUntil: allPromo[j].validUntil,
promoUrl: allPromo[j].promoUrl
});
console.log(json.promo[j]);
}
})
promoPerCategory.push(json);
}
result.send({ "promoPerCategory": promoPerCategory });
})
the then() inside function getPromoPerCategories is working fine, i print out the value using console.log(json.promo[j])
But the problem is i want to put array from json.promo[j] into array promoPerCategory so that i can return it using result.send({ "promoPerCategory": promoPerCategory })
Can you please help me to do the right way? any help will be very appreciated.
Result is now like this
{"promoPerCategory": [{"category":"Fashion","url":"https://","promo":[]}
object promo is empty array
Your first problem is that result.send is not waiting for the getPromoPerCategories result. To make that happen, result.send needs to be in a callback of the getPromoPerCategories promise. It also looks like you want to wait for all of your results to come back before calling result.send, so you should use Promise.all and wait on all of the getPromoPerCategories promises. For example:
categories.then(function (val) {
let promoPerCategory = [],
promoPerCategoryPromises = [];
for (let i = 0, len = val.length; i < len; i++) {
let json = {
category: val[i].category,
url: val[i].url,
promo: []
}
var promise = scrapper.getPromoPerCategories(val[i].url)
.then(function (allPromo) {
for (let j = 0, lenResult = allPromo.length; j < lenResult; j++) {
json.promo.push({
imageUrl: allPromo[j].imageUrl,
merchantName: allPromo[j].merchantName,
promoTitle: allPromo[j].promoTitle,
validUntil: allPromo[j].validUntil,
promoUrl: allPromo[j].promoUrl
});
console.log(json.promo[j]);
}
promoPerCategory.push(json);
});
promoPerCategoryPromises.push(promise);
}
Promise.all(promoPerCategoryPromises).then(function() {
result.send({ "promoPerCategory": promoPerCategory });
});
})
var data = [];
scrapper.getPromoPerCategories(val[i].url)
.then(function() {
data.push({'foo': 'bar'});
});
result.send(data);
Or you can just result.send() inside the promise chain.
.then(function() {
var data = [];
data.push({'foo': 'bar'});
result.send(data);
});
Related
I'm working on a simple function I have for a specific GET request triggered in the browser. The objective of this request is to make multiple queries to a mongodb (mongoose) database and then perform some calculation and structure formating on the results to send it back to the browser.
The only problem is that everything takes too long and it results in an error in the browser:
net::ERR_EMPTY_RESPONSE
to give an example of part of the function I'm trying to build here it goes:
async function getPriceByMake(makes, id) {
return new Promise(async (resolve, reject) => {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
}
resolve(pMakes);
});
}
In this function, if I leave the async / await out, I get an empty {} on the other end. Which is obviously not the objective.
I've been searching the web a little and was able to find an article pointing to this scheme:
Browser:
Initiates request
displays progress
Show result
WebServer:
Submit event
Checks for completion
Return result
BackEndApp:
Picks up event
Runs task
Returns results
My question is the following:
How can I do that with NodeJS and Express?
In this code:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
Your await isn't working because you're passing a callback to Listing.find(). When you do that, it does NOT return a promise and therefore the await does nothing useful. You get the empty response because the await doesn't work and thus you call resolve() before there's any actual data there.
Change the code to this:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
And, then the await will work properly.
You also should remove the return new Promise() wrapper. You don't want that. Just make the function async and use await and it will already return a promise.
Here's your function with the unnecessary promise wrapper removed:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
}
return pMakes;
}
Then, keep in mind that whatever code sends your actual response needs to use .then() or await when calling this async function in order to get the final result.
Your best bet to speed up this code would be to refactor either your queries or your database structure or both to not have to do N * M separate queries to get your final result. That's likely where your slowness is coming from. The biggest performance gains will probably come from reducing the number of queries you have to run here to far fewer.
Depending upon your database configuration and capabilities, it might speed things up to run the inner loop queries in parallel as shown here:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
await Promise.all(modelsArr.map(async currModel => {
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}));
}
return pMakes;
}
I have a for loop function in NodeJS. I would like to wait until the result of Http Get request is completed in For Loop before it executes the next iteration, how do I achieve that?
for (let k=0; k<fd.length; k++) {
url = fd[k].nct_id;
HttpSearch({condition: url}).then(trials => {
//Get the result first before execute the next iteration
console.log(trials);
});
}
You should make the for-loop async:
const main = async () => {
for (let k = 0; k < fd.length; k++) {
const url = fd[k].nct_id;
const trials = await HttpSearch({ condition: url });
console.log(trials);
}
};
main().catch(console.error);
This will cause the loop to "pause" at each HttpSearch.
I will do like this
let k = 0 ;
let len = fd.length;
for (; k > len;) {
let url = fd[k].nct_id;
let subs = await HttpSearch({condition: url});
console.log(subs);
k++
}
or like this with promise
let url;
let promiseChain = Promise.resolve();
for (let i = 0; i < fd.length; i++) {
url = fd[k].nct_id;
// you need to pass the current value of `url`
// into the chain manually, to avoid having its value
// changed before the .then code accesses it.
const makeNextPromise = (url) => () => {
HttpSearch({condition: url})
.then((result) => {
// return promise here
return result
});
}
promiseChain = promiseChain.then(makeNextPromise(url))
}
This is using recursion, which calls next, once previous is finished
var limit = fd.length;
var counter = 0;
HttpSearch({condition: fd[0].nct_id;}).then(yourCallBack);
function yourCallBack(trials){
console.log(trails);
if(counter == limit)
return console.log('Done')
HttpSearch({condition: fd[counter].nct_id;}).then(yourCallBack);
counter++;
}
I am getting error in the below implementation of typescript code. I am mapping here one type to another. But vscode shows error that variable 'newData' is used before being assigned. I know it may be a silly error, but I am unable to find it.
onKeyUp(value: string) {
console.log(value);
var link = ``
link = `${API}/${value}`
this.hn.getNews().subscribe(data => {
this.loading = false;
var newData:NewsItem[];
fetch(link)
.then(function(response) {
return response.json();
})
.then(function(myJson) {
for (var i = 0; i < myJson.length; i++) {
newData.push(myJson[i])
}
})
this.news = newData;
});
}
}
You need to initialize it to empty array as follows,
var newData:NewsItem[] = [];
I wrote a script in Node that iterates over a large MongoDB collection, returning a certain number of documents at a time.
The collection has this simple format:
{
name: 'One',
data: '...'
},
{
name: 'Two',
data: '...'
},
...
I'm doing this job with the Q library, using a sequence of promises that get run one after the other:
'use strict';
var Q = require('q');
var monk = require('monk');
var CHUNK_SIZE = 100;
var LIMIT = 1000;
var collection = monk('localhost/dictionary').get('entries');
var promiseFactory = function (j) {
return function (result) {
if (undefined !== result) { // if result is undefined, we are at the first or last iteration.
if (result.length) {
for (var k = 0, max = result.length; k < max; k++) {
console.log(result[k].name); // print name
// ... do something with the document here...
}
} else { // no more documents, end of the iteration
return; // implicitely returns undefined
}
}
// returns CHUNK_SIZE documents, starting from the j-th document
return collection.find({}, { limit: CHUNK_SIZE, skip: j, sort: { name: 1 }});
};
};
var funcs = [];
for (var i = CHUNK_SIZE; i <= LIMIT; i += CHUNK_SIZE) {
funcs.push(promiseFactory(i));
}
var loop = Q.fcall(promiseFactory(0));
funcs.forEach(function (f) {
loop = loop.then(f);
});
The script works well and does achieve what it was designed to do.
However, I would like to improve it:
I'm hardcoding the number of documents in the collection (LIMIT). I would like to get rid of this variable and let the script detect when to stop.
I have a feeling that this approach may not be the most memory-efficient one. In my code, funcs.forEach() chains a lot of copies of the same function in one shot (to be exact LIMIT/CHUNK_SIZE copies). Since I'm working on a very large collection, I was wondering if there's a way to chain a new function only if there are still documents left, while running through the collection.
I think I found the solution to both problems. It is just a simple addition in promiseFactory() which I have highlighted below. Adding it here in the hope it is useful to someone:
var promiseFactory = function (j) {
return function (result) {
if (undefined !== result) { // if result is undefined, we are at the first or last iteration.
if (result.length) {
for (var k = 0, max = result.length; k < max; k++) {
console.log(result[k].en + ' - ' + result[k].le);
}
} else { // no more entries, end of the iteration
return; // implicitely returns undefined
}
}
///////////////// CHANGE HERE ////////////////////////
return entries.find({}, { limit: CHUNK_SIZE, skip: j, sort: { en: 1 }}).then(promiseFactory(j + CHUNK_SIZE));
///////////////////// END ////////////////////////////
};
};
Now I've working on NodeJS and Sequelize to query and process an database data.
I've call findAll from Table1 and I want to query each rows to apply some data to Table2 then I want to add all data to array before send output, I did like this
var last_promise;
var output_results = {};
Table1Model.findAll()
.then(function(results1)
{
for (var i = 0; i < results1.length; ++i)
{
var result1 = results1[i];
output_results[result1.id] = result1;
var add_promise = Table2Model
.create({
id_from_table1: result1.id,
data_from_table1: result1.data
});
.then(function(result2) {
output_results[result2.id_from_table1].data2 = result2;
});
if (last_promise)
{
last_promise.then(function()
{
return add_promise;
});
} else {
last_promise = add_promise;
}
}
}
}
last_promise.then(function() {
return output_results;
}
I want to know that there any better way to execute promises sequentially in a loop like this ?
It looks like you can do that with .all() method:
Table1Model
.findAll()
.then(function(results1) {
return Promise.all(results1.map(function(result) {
return Table2Model
.create({
id_from_table1: result1.id,
data_from_table1: result1.data
})
.then(function(result2) {
...
});
}));
})
.then(function(output_results) {
});