Facing issue for synchronous operation in NodeJS - node.js

Below code have 2 files and having client.js which calls to server file but didn't get the synchronous output.
I tried with promise, bluebird, async_await but didn't get success.
Expected output is a sequence of the alphabet in a console.
Do not use settimout.
Understanding of files.
server.js : server file is having NodeAPI which content only routing.
Client.js : logic which tried is in this file
You need to use two console, first run server.js and second console need to run client.js and output will be print in server.js console.
Expecting output is
a
b
c
d
e
/////////////////server.js/////////////////
var express = require('express');
var bodyParser = require('body-parser')
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
app.post('/', function (req, res) {
console.log(req.body.test)
res.status(200).send('ok');
});
var server = app.listen(3000, function () {
console.log('ok');
});
//////////////client.js///////////////////
//var Promise = require('bluebird');
var request = require('request');
console.log('server file called');
var array_data = ['a', 'b', 'c', 'd', 'e'];
// var promises = [];
// for(let i = 0; i < array_data.length; i++) {
// var promise = request.post({url: 'http://localhost:3000',form : {key:array_data[i]}});
// }
// var page = 0;
// var last_page = array_data.length;
// (function loop() {
// if (page < last_page) {
// request.post({
// url: 'http://localhost:3000',
// form: 'test=' + array_data[page]
// }, function (error, response, body) {
// page++;
// loop();
// });
// }
// }());
// async function loopAsync() {
// for (let i = 0; i < array_data.length; i++) {
// await request.post({
// url: 'http://localhost:3000',
// form: 'test=' + array_data[i]
// });
// }
// }
// loopAsync();
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
await request.post({
url: 'http://localhost:3000',
form: 'test=' + array_data[page]
});
page++;
}
}
loopAsync();
[enter image description here][1]
[1]: https://i.stack.imgur.com/WVlut.png

You can use util.promisify.
Here is sample code
const reqpost = util.promisify(request.post);
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
await reqpost({
url: 'http://localhost:3000',
form: 'test=' + array_data[page]
});
page++;
} }
loopAsync();

request.post is a function that takes request options and a callback like this.
function request(letter, callback = () => {}) {
setTimeout(() => {
console.log(letter);
callback();
}, Math.random() * 1000);
}
What you're doing is calling that function without supplying a callback:
async function clientWithRequest() {
const letters = ['a', 'b', 'c'];
for(let i = 0; i < letters.length; i++) {
await request(letters[i]);
}
}
here, the requests are all fired off at the same time and will return in an indeterminate order.
What you need to do if you want to use async is make your request return a promise. Under the hood, await is really just doing request.then(somethingElse()). So if you change your request to return a promise like:
function requestPromise(letter) {
return new Promise((resolve, reject) => {
setTimeout(() => {
console.log(letter);
resolve();
}, Math.random() * 1000);
});
}
your await code will then work as expected. Effectively it's doing.
request('a').then(() => request('b')).then(() => request('c'));

https://github.com/request/request#forms
Since you are forming the form data string yourself via Node
Is it possible that if you can just let request help you form the encoded string via their form method?
const jobs = [];
const array_data = [
'a',
'b',
'c',
'd',
'e',
];
require('request');
const request = require('request-promise');
async function loopAsync() {
for(const char of array_data) {
jobs.push(
request.post({
url: 'https://marble-scene.glitch.me/',
form: {
test: char
}
})
);
}
const output = await Promise.all(jobs);
output.forEach(char => console.log(char)); // # a b c d e
}
loopAsync();
[Edit]
Just edit the client.js code, fixed the silly syntax error inside the jobs.push block
Working prove: https://glitch.com/edit/#!/marble-scene

As an alternative to importing additional libraries. you can simply "promisify" the callback dependent request.post method.
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
await new Promise((resolve, reject) => {
request.post({
url: 'http://localhost:3000',
form: 'test=' + array_data[page]
}, function(err,httpResponse,body){
if (err) reject(err);
else resolve({httpResponse,body});
});
page++;
});
}
}
loopAsync()

As I did the R&D, request module is not returned promise so for that need to use request-promise library.
let request = require('request');
let rp = require('request-promise');
var array_data = ['a', 'b', 'c', 'd', 'e'];
//with the use of request-promise library
async function loopAsync() {
var page = 0;
var last_page = array_data.length;
while (page < last_page) {
var options = {
method: 'POST',
uri: 'http://localhost:3000',
body: {
test: array_data[page]
},
json: true // Automatically stringifies the body to JSON
};
await rp(options);
page++;
}
}
loopAsync();

Related

Nodejs request async problem for loop not work

I'm a beginner of nodejs, async bothers me.
I want my code run sequencely or it will breaks.
I have a for loop, and it simply doesn't work...
Here are all the codes:
const util = require('util');
const request = require('request');
const cheerio = require('cheerio');
var host = "http://www.nicotv.me";
var url = "http://www.nicotv.me/video/play/57838-1-%s.html";
var len = 99;
var tab = /-(\d)-/.exec(url);
tab = tab[1] // '1' not '-1-'
function getLen(url) {
//you can ignore this function, it gives len=2
request(url, function (err, response, html) {
if (err) {
console.log('url:', url);
console.log('error:', err);
console.log('statusCode:', response && response.statusCode);
}
else{
var $ = cheerio.load(html);
var cls = '.ff-playurl-dropdown-%s';
$(util.format(cls, tab)).filter(function (){
var data = $(this);
len = data.html().match(/<a href=/g).length;
console.log("episode:", len);
});
getLink(len, function(){
});
}
});
}
getLen(util.format(url, 1)); //len = 2
var getLink = function(lengths, callback){
for (let i = 1; i <= lengths; i++) {
var tmp = util.format(url, i);
try {
request(tmp, function (err, res, html){
console.log('url:', tmp);
if(err){
console.log("error:", err);
console.log("statusCode:", res && res.statusCode);
}else{
var reg = /src="(\/player.php?.{1,})"/;
var result = reg.exec(html);
console.log(result[1]);
}
});
callback();
} catch (error) {
console.log(error);
break;
}
}
}
here is my output:
episode: 2
url: http://www.nicotv.me/video/play/57838-1-2.html
/player.php?u=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1NzkxMzU2MzEyNDAwNTQ5&p=360biaofan&c=0&j=aHR0cDovL2ppZXhpLmtpbmdzbnVnLmNuLzM2MGJpYW9mYW4ucGhwP3VybD0=&x=10&y=&z=
url: http://www.nicotv.me/video/play/57838-1-2.html
/player.php?u=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1Nzg1MDQyMDYyNDAwNTgx&p=360biaofan&c=0&j=aHR0cDovL2ppZXhpLmtpbmdzbnVnLmNuLzM2MGJpYW9mYW4ucGhwP3VybD0=&x=10&y=&z=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1NzkxMzU2MzEyNDAwNTQ5
First problem is these two /player*** link are from 57838-1-1.html
And one of them are not complete.
Second problem is the url output shows 57838-1-2.html twice.
Thanks for your kindly help.
Yesterday had the same problem, so I solved with:
Using request-promise
Replace the loop method arrTitles.Each with for (const jt of arrTitles)
Here a sample:
const request = require('request-promise');
const cheerio = require('cheerio');
var getUrlData =
async function (url) {
console.log(url);
try {
return await request.get(url);
}
catch (err) {
console.error(`${err}: ${url}`);
}
return;
};
var run =
async function (pageUrl) {
var arrData =
await fn.getUrlData(pageUrl)
.then(response => readTable(response));
console.log(arrData);
};
var readTable =
function (document) {
var $;
let arrData = [];
try {
$ = cheerio.load(document);
$('table tr')
.each(
function (trN) {
$(this)
.children('td')
.each(
function (tdN) {
arrData.push($(this).text().trim());
}
)
});
}
catch { }
return arrData;
};
run();

How to return data from loop in order in node

I am creating a web scraper that scrapes all of the movies coming out for the next year from this site (https://www.imdb.com/movies-coming-soon/) and it loops through an array of links that contain all the movies for each month for the next year, its working but the only problem is that its not returning them in order due to node.js asynchronous behavior, how do i get it to loop through the array and return the data in order?
Ive tried to make a callback function but I don't know where it would go at
const request = require('request')
const cheerio = require('cheerio')
const movieArray = [ '/movies-coming-soon/2019-09/',
'/movies-coming-soon/2019-10/',
'/movies-coming-soon/2019-11/',
'/movies-coming-soon/2019-12/',
'/movies-coming-soon/2020-01/',
'/movies-coming-soon/2020-02/',
'/movies-coming-soon/2020-03/',
'/movies-coming-soon/2020-04/',
'/movies-coming-soon/2020-05/',
'/movies-coming-soon/2020-06/',
'/movies-coming-soon/2020-07/',
'/movies-coming-soon/2020-08/' ]
for (let i = 0; i < movieArray.length; i++) {
request.get('https://www.imdb.com' + movieArray[i] , (err, res, body) => {
if (!err && res.statusCode == 200) {
console.log(res.request.href)
const $ = cheerio.load(body)
//console.log(next)
$('h4').each((i, v) => {
const date = $(v).text()
console.log(date)
})
}
})
}
I'm expecting it to return the data in order instead of it being returned in a order based off how fast the data is returned due to nodes asynchronous behavior
It's a classic async issue in for loop as per explained https://lavrton.com/javascript-loops-how-to-handle-async-await-6252dd3c795/. Below would be the solution:
// const request = require('request')
const request = require('request-promise');
const cheerio = require('cheerio');
const movieArray = [
'/movies-coming-soon/2019-09/',
'/movies-coming-soon/2019-10/',
'/movies-coming-soon/2019-11/',
'/movies-coming-soon/2019-12/',
'/movies-coming-soon/2020-01/',
'/movies-coming-soon/2020-02/',
'/movies-coming-soon/2020-03/',
'/movies-coming-soon/2020-04/',
'/movies-coming-soon/2020-05/',
'/movies-coming-soon/2020-06/',
'/movies-coming-soon/2020-07/',
'/movies-coming-soon/2020-08/',
];
async function processMovieArray(array) {
for (const item of array) {
await getMovie(item);
}
console.log('Done');
}
async function getMovie(item) {
const options = {
method: `GET`,
uri: 'https://www.imdb.com' + item,
};
const response = await request(options);
const $ = cheerio.load(response.body);
$('h4').each((i, v) => {
const date = $(v).text();
console.log(date);
});
}
processMovieArray(movieArray);
The low tech way that deviates the least from your current code is to just use the index of your for loop to populate an array. Since let in the for loop will make a separate variable for i for each iteration of the for loop, we can use that index inside the async callback to reference the desired spot in a results array. Then, you also use a cntr to know when you've finished with all the results:
const request = require('request');
const cheerio = require('cheerio');
if (!Array.prototype.flat) {
Array.prototype.flat = function() {
return this.reduce((acc, val) => acc.concat(val), []);
}
}
const movieArray = [ '/movies-coming-soon/2019-09/',
'/movies-coming-soon/2019-10/',
'/movies-coming-soon/2019-11/',
'/movies-coming-soon/2019-12/',
'/movies-coming-soon/2020-01/',
'/movies-coming-soon/2020-02/',
'/movies-coming-soon/2020-03/',
'/movies-coming-soon/2020-04/',
'/movies-coming-soon/2020-05/',
'/movies-coming-soon/2020-06/',
'/movies-coming-soon/2020-07/',
'/movies-coming-soon/2020-08/' ];
let results = [];
let cntr = 0;
for (let i = 0; i < movieArray.length; i++) {
request.get('https://www.imdb.com' + movieArray[i] , (err, res, body) => {
++cntr;
if (!err && res.statusCode == 200) {
console.log(res.request.href)
const $ = cheerio.load(body)
//console.log(next)
let textArray = [];
$('h4').each((i, v) => {
console.log(date)
textArray.push($(v).text());
});
results[i] = textArray;
}
if (cntr === moveArray.length) {
// all results are done now
let allResults = results.flat();
}
})
}
A bit more elegant way is to switch over to promises and let the promise infrastructure keep everything in order for you:
const rp = require('request-promise');
const cheerio = require('cheerio');
if (!Array.prototype.flat) {
Array.prototype.flat = function() {
return this.reduce((acc, val) => acc.concat(val), []);
}
}
const movieArray = [ '/movies-coming-soon/2019-09/',
'/movies-coming-soon/2019-10/',
'/movies-coming-soon/2019-11/',
'/movies-coming-soon/2019-12/',
'/movies-coming-soon/2020-01/',
'/movies-coming-soon/2020-02/',
'/movies-coming-soon/2020-03/',
'/movies-coming-soon/2020-04/',
'/movies-coming-soon/2020-05/',
'/movies-coming-soon/2020-06/',
'/movies-coming-soon/2020-07/',
'/movies-coming-soon/2020-08/' ];
//
if (!Array.prototype.flat) {
Array.prototype.flat = function() {
return this.reduce((acc, val) => acc.concat(val), []);
}
}
Promise.all(movieArray.map(path => {
return rp('https://www.imdb.com' + path).then(body => {
const $ = cheerio.load(body);
let textArray = [];
$('h4').each((i, v) => {
// console.log($(v).text());
textArray.push($(v).text());
});
return textArray;
}).catch(err => {
// ignore errors on urls that didn't work
// so we can get the rest of the results without aborting
console.log("err");
return undefined;
});
})).then(results => {
// flatten the two level array and remove empty items
let allResults = results.flat().filter(item => !!item);
console.log(allResults);
}).catch(err => {
console.log(err);
});
FYI, I tested the 2nd version in nodejs version 10.16.0 and it works.

How to wait for a url callback before send HTTP response in koa?

I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});

Polling a URL until certain value is set in JSON response : Mocha, Integration testing

I am working on automating an End to end scenario using Mocha.
I have a url endpoint which is to be polled until a certain value is obtained in the resulting response. Is there any way to do it ?
Example with request and callback approach:
const request = require('request');
describe('example', () => {
it('polling', function (done) {
this.timeout(5000);
let attemptsLeft = 10;
const expectedValue = '42';
const delayBetweenRequest = 100;
function check() {
request('http://www.google.com', (error, response, body) => {
if (body === expectedValue) return done();
attemptsLeft -= 1;
if (!attemptsLeft) return done(new Error('All attempts used'));
setTimeout(check, delayBetweenRequest);
});
}
check();
});
});
Example with got and async/await approach:
const utils = require('util');
const got = require('got');
const wait = utils.promisify(setTimeout);
describe('example', () => {
it('polling', async function (done) {
this.timeout(5000);
const expectedValue = '42';
const delayBetweenRequest = 100;
for (let attemptsLeft = 10; attemptsLeft; attemptsLeft -= 1) {
const resp = await got.get('http://www.google.com');
if (resp.body === expectedValue) return done();
await wait(delayBetweenRequest);
}
done(new Error('All attempts used'));
});
});
This is how I was able to do it with WebdriverIO and Mocha
describe("wait for value in content of page", () => {
it("should be able to wait to value in url", () => {
var max_seconds_to_wait = 10;
var seconds_counter = 0;
var should_continue = true;
while (should_continue) {
browser.url('http://your.url.com');
var response = JSON.parse(browser.getText("body"));
console.log(response)
if (response == 'something') {
should_continue = false;
}
browser.pause(1000);
seconds_counter++;
if (seconds_counter > max_seconds_to_wait) {
throw 'Waiting for json from url timeout error';
}
}
});
});

Nodejs loop http requests in a function

I have been able to use bluebird to run an array of http get requests in parallel within a function that returns the responses of each requests after they have all finished. However, I would like to chain the requests in sequence to run one after another without effecting the asynchronous behavior of the promises in my current function shown below.
var Promise = require('bluebird');
function buildCartWithItems(cartID,requests, cookie) {
var promises = [];
for (idx in requests) {
var request = requests[idx]; // requests is an array containing the json records of each request parameter
request["id"] = cartID;
promises.push(new Promise(function(resolve, reject) {
var options = {
host: 'example.com',
path: "/example/addToCart?"+param(request),
headers: {'Cookie': cookie, 'Connection': 'keep-alive'}
};
https.get(options, function (response) {
var body = '';
response.on('data', function(d) {
body += d;
});
response.on('end', function() {
console.log("jsonservicesPostRequest response: " + body);
var parsed = JSON.parse(body);
if (parsed["STATUS"] == "success") {
resolve(parsed.RESULT.itemGroup.id);
} else
resolve("error");
});
response.on('error', function(exception) {
console.log("auth error: " + exception);
resolve(exception);
});
});
}));
}
return Promise.all(promises);
}
Use:
var cart = 12345; // cart ID
var itemsParams = [];
for (idx in products) {
var parms = {
'guid' : prod["_id"]["GUID"],
'count': prod["deficit"],
};
itemsParams.push(parms);
}
buildCartWithItems(cart,itemsParams,newcookie).then(function(results) {
// results -> array of all of the cart.id's
console.log("Build Cart Results: " + JSON.stringify(results));
});

Resources