nodejs: check until URL get's online - node.js

I need a "forever-watchdog" for checking if an URL is reachable or not. My case is that I have to see if a media stream gets online, and if, I would need to run a shell-script. If it's not reachable (404'd), I would keep trying again in x seconds.
Could anyone please guide me into the right directions, in terms on modules, or flowing? I had problems with resolving the destination with the
isReachable = require('is-reachable');
module.
Thanks for helping!

Try this, should resolve you task
'use strict';
const co = require('co');
const request = require('request');
const exec = require('child_process').exec;
let getRemoteSourceStatusPromise = function(urlForCheck){
return new Promise(function (resolve, reject) {
request(urlForCheck, function(error, response){
console.log('send request...');
if(error)
return reject(error);
let result = false;
console.log('Resource response statusCode ' + response.statusCode);
if(200 === response.statusCode){
result = true;
}
return resolve(result);
});
});
};
let callShScriptPromise = function(){
let shScriptPath = './script.sh';
return new Promise(function (resolve, reject) {
console.log('Calling sh...');
exec(shScriptPath,
function (error) {
if (error)
return reject(error);
resolve(true);
});
});
};
let sleep = function (sleepInterval) {
return new Promise(function (resolve) {
setTimeout(function () {
console.log('sleep...');
resolve();
}, sleepInterval);
});
};
let loopGenerator = function* (urlForCheck, sleepInterval){
let bShScriptStarted = false;
while(true){
let bSourceStatus = yield getRemoteSourceStatusPromise(urlForCheck);
if(bSourceStatus === true){
if(!bShScriptStarted)
bShScriptStarted = yield callShScriptPromise();
}else{
bShScriptStarted = false;
}
yield sleep(sleepInterval);
}
};
//Starting loop
co(loopGenerator('http://google.com', 30000))
.catch(function (error) {
console.log(error);
})

const rp = require('request-promise');
const checkIfOnline = (() -> {
return new Promise((resolve, reject)=>{
rp( url ).then( response => {
// this is where you run your script
resolve();
}).catch(error=>{
setTimeout( resolve(checkIfOnline()), 5000 );
});
});
})();

You can send requests to the URL repeatedly. Something like this:
//I don't know how you create nodejs server, I just leave out that part.
//I use request module. sudo npm install --save request
var request = require('request');
var URL = 'https://www.google.com';
var X = 5000; //5 seconds
var sendRequest = function(){
request(URL, function(error, response, body){
if(response.statusCode === 200){//If good
runScript(); //This is the function you write to run sh
}
setTimetout(sendRequest, X);//run the function again no matter what
});
}
sendRequest();
If you want a better forever loop, I suggest you use promise.
var rp = require('request-promise');
var options = {
uri : 'https://www.google.com',
json : true
};
var X = 5000; //5 seconds, the actual time is 5 seconds plus the time to resolve
var sendRequest_promise = function(){
//return a request promise
return rp(options).then(function(response){
return response;
});
};
var sendRequest = function(){
sendRequest_promise().then(function(response){
if(response.statusCode === 200){
runScript();
}
setTimeout(sendRequest(), X);
});
};

Related

Node.js https request stuck

I am a node.js beginner and I'm trying to check status of domains I read from a csv file. All works fine, however when the script reaches the end it simply hangs and does not quit. Below is the code I'm working with. What am i missing here? Any help would be greatly appreciated!
```
const fs = require('fs');
const request = require('request');
let https = require('https');
const input_path = 'Path_to_csv'
function FetchUrls (callback) {
fs.readFile(input_path, 'utf8', function (err, data) {
let dataArray = data.split(/\r?\n/);
console.log(`loaded ${dataArray.length} items`)
callback(dataArray)});
}
function getData (dataArray) {
let urls = dataArray
for (let url of urls) {
https.get(url, function(res) {
if(res.statusCode == 500) {
console.log("Domain Down")
} else {
console.log("Domain Up")
}
}).on('error', function(e) {
console.log(e)
process.exit()
});
}
}
FetchUrls(function(dataArray) {
getData(dataArray)
})
```
You need to add a return after the for loop. But I would wrap it in a promise like:
FetchUrls(input_path, function(dataArray) {
let promises = []
for (let domain of dataArray) {
promises.push(new Promise((resolve, reject) => {
// ... do the https.getting things, and then resolve() where you think it should end
}))
}
Promise.all(promises).then( Nothing => return Nothing)
.catch(error => return error)
}

Nodejs request async problem for loop not work

I'm a beginner of nodejs, async bothers me.
I want my code run sequencely or it will breaks.
I have a for loop, and it simply doesn't work...
Here are all the codes:
const util = require('util');
const request = require('request');
const cheerio = require('cheerio');
var host = "http://www.nicotv.me";
var url = "http://www.nicotv.me/video/play/57838-1-%s.html";
var len = 99;
var tab = /-(\d)-/.exec(url);
tab = tab[1] // '1' not '-1-'
function getLen(url) {
//you can ignore this function, it gives len=2
request(url, function (err, response, html) {
if (err) {
console.log('url:', url);
console.log('error:', err);
console.log('statusCode:', response && response.statusCode);
}
else{
var $ = cheerio.load(html);
var cls = '.ff-playurl-dropdown-%s';
$(util.format(cls, tab)).filter(function (){
var data = $(this);
len = data.html().match(/<a href=/g).length;
console.log("episode:", len);
});
getLink(len, function(){
});
}
});
}
getLen(util.format(url, 1)); //len = 2
var getLink = function(lengths, callback){
for (let i = 1; i <= lengths; i++) {
var tmp = util.format(url, i);
try {
request(tmp, function (err, res, html){
console.log('url:', tmp);
if(err){
console.log("error:", err);
console.log("statusCode:", res && res.statusCode);
}else{
var reg = /src="(\/player.php?.{1,})"/;
var result = reg.exec(html);
console.log(result[1]);
}
});
callback();
} catch (error) {
console.log(error);
break;
}
}
}
here is my output:
episode: 2
url: http://www.nicotv.me/video/play/57838-1-2.html
/player.php?u=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1NzkxMzU2MzEyNDAwNTQ5&p=360biaofan&c=0&j=aHR0cDovL2ppZXhpLmtpbmdzbnVnLmNuLzM2MGJpYW9mYW4ucGhwP3VybD0=&x=10&y=&z=
url: http://www.nicotv.me/video/play/57838-1-2.html
/player.php?u=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1Nzg1MDQyMDYyNDAwNTgx&p=360biaofan&c=0&j=aHR0cDovL2ppZXhpLmtpbmdzbnVnLmNuLzM2MGJpYW9mYW4ucGhwP3VybD0=&x=10&y=&z=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1NzkxMzU2MzEyNDAwNTQ5
First problem is these two /player*** link are from 57838-1-1.html
And one of them are not complete.
Second problem is the url output shows 57838-1-2.html twice.
Thanks for your kindly help.
Yesterday had the same problem, so I solved with:
Using request-promise
Replace the loop method arrTitles.Each with for (const jt of arrTitles)
Here a sample:
const request = require('request-promise');
const cheerio = require('cheerio');
var getUrlData =
async function (url) {
console.log(url);
try {
return await request.get(url);
}
catch (err) {
console.error(`${err}: ${url}`);
}
return;
};
var run =
async function (pageUrl) {
var arrData =
await fn.getUrlData(pageUrl)
.then(response => readTable(response));
console.log(arrData);
};
var readTable =
function (document) {
var $;
let arrData = [];
try {
$ = cheerio.load(document);
$('table tr')
.each(
function (trN) {
$(this)
.children('td')
.each(
function (tdN) {
arrData.push($(this).text().trim());
}
)
});
}
catch { }
return arrData;
};
run();

Synchronous/sequential REST calls in loop

I'm trying to call a REST API in a "for" loop, however, the results aren't what I'm expecting.
I've attempted to wrap everything in a promise, but the order of operations is still off, executing it asynchronously rather than synchronously.
var https = require('https');
var zlib = require("zlib");
var axios = require('axios');
const cheerio = require('cheerio');
var page = 1;
var hasMore = "true";
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function getLocation(page) {
// notice that we can await a function
// that returns a promise
await delay();
var apiUrl = 'https://my.api.com/search/advanced?page=' + page +
'&pagesize=5';
https.get(apiUrl, function(response) {
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function(chunk) {
jsonString += chunk;
});
gunzip.on('end', function() {
obj = JSON.parse(jsonString);
var url = obj.items[0].owner.link;
axios(url)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
//OUTPUT LOCATION
console.log($('h3.location').text().trim());
})
.catch(console.error);
});
gunzip.on('error', function(e) {
console.log(e);
});
} else {
console.log("Error");
}
});
}
async function startGetLocation() {
var page = 1;
var hasMore = "true";
do {
//OUTPUT PAGE NUMBER
console.log(page.toString());
await getLocation(page);
page = page + 1;
} while (page < 3);
}
startGetLocation();
Based on the sample code, I would have expected the below to output:
1
New York
2
However, it's outputting:
1
2
New York
The problem is that the callback function that you passed to the https.get() function gets executed asynchronously and that the getLocation function does not wait until this part resolves.
So you could simply wrap the https.get() call and the unzipping part in a promise, wait for it to resolve and then do the axios-part.
async function getLocation(page) {
await delay();
var apiUrl = 'https://my.api.com/search/advanced?page=' + page +
'&pagesize=5';
const fetchAndUnzipPromise = new Promise((resolve, reject) => {
https.get(apiUrl, function (response) {
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function (chunk) {
jsonString += chunk;
});
gunzip.on('end', function () {
obj = JSON.parse(jsonString);
var url = obj.items[0].owner.link;
resolve(url);
});
gunzip.on('error', function (e) {
reject(e);
});
} else {
reject(new Error("Statuscode not as exepcted"));
}
});
});
return fetchAndUnzipPromise.then(url => {
return axios(url)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
//OUTPUT LOCATION
console.log($('h3.location').text().trim());
})
.catch(console.error);
})
}

How could I return an array from my module to my API using promise? Async Functions and Promise - Node.js

I need to return a json object to my api. To do this I have a module that does some requests and should return the results.
My problem is grasping the promise concept and implementing it.
server.js
app.get('/users', function(req, res){
request.getUsers()
.then(function(users){
console.log(users);
res.contentType('application/json');
res.send(JSON.stringify(users));
})
.catch(function(){
console.log(users);
});
});
module.js
exports.getUsers = function(){
var params = {search_string:""};
var users = [];
return new Promise(function(resolve, reject){
var result = connection.Users.get(params, function(error,response)
{
var user = [];
for(let i = 0; i < response.data.length; i++)
{
user = response.data;
}
users.push({user});
});
if(result != null)
{
console.log(result);
resolve(result);
}
else
{
reject(new Error('Try Again'));
}
});
}
When I run the server I get the typeError: expecting a function but got [object object]
I did not really get what is wrong.
How could I return an array from my module to my API using promises?
EDIT:
app.get('/users', function(req, res){
request.getUsers()
.then(function(users){
console.log(users);
res.contentType('application/json');
res.send(JSON.stringify(users));
})
.catch(function(){
console.log("not resolved");
});
});
My problem now is actually that I am getting the .catch even before any request is made the at /users endpoint and I dont know why.
In module.js you used new Promise() constructor but the input parameter should be a function and not an object, so to fix that use:
return new Promise(function(resolve, reject) {
var result = connection.Users.get(params, function(error,response)
...
});
Notice its not new Promise({function(...) but new Promise(function(...)) ...
Read more here:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
Edit:
I have modified your code to work to fix the second problem:
exports.getUsers = function(){
var params = {search_string:""};
var users = [];
return new Promise(function(resolve, reject){
var result = connection.Users.get(params, function(error,response) {
if(error || !response)
{
// report error
reject(new Error('Try Again'));
}
else
{
//process response
var user = [];
for(let i = 0; i < response.data.length; i++)
{
user = response.data;
}
users.push({user});
// report success
resolve(users);
}
});
}
You need to call resolve or reject inside connection.Users.get(params, function(error,response) {
Modify your module.js code as below. You passed an object instead of a function.
register.getUsers = function () {
var params = { search_string: "" };
var users = [];
return new Promise(function (resolve, reject) {
var result = connection.Users.get(params, function (error, response) {
var user = [];
for (let i = 0; i < response.data.length; i++) {
user = response.data;
}
users.push({ user });
});
if (result != null) {
console.log(result);
resolve(result);
}
else {
reject(new Error('Try Again'));
}
});
};
you declared user variable as an array and inside the for loop isn't useful because the user variable is always equals to response.data
if response.data is array of JSON object you can push it to users array inside loop
for (let i = 0; i < response.data.length; i++) {
users.push(response.data[i]);
}
I guess you want to return the array of objects
also I recommend you to use bluebird module to return promises
and also you can use Promise.mapSeries instead of for loop like:
return Promise.mapSeries(response.data, item => {
users.push(item)
})

Polling a URL until certain value is set in JSON response : Mocha, Integration testing

I am working on automating an End to end scenario using Mocha.
I have a url endpoint which is to be polled until a certain value is obtained in the resulting response. Is there any way to do it ?
Example with request and callback approach:
const request = require('request');
describe('example', () => {
it('polling', function (done) {
this.timeout(5000);
let attemptsLeft = 10;
const expectedValue = '42';
const delayBetweenRequest = 100;
function check() {
request('http://www.google.com', (error, response, body) => {
if (body === expectedValue) return done();
attemptsLeft -= 1;
if (!attemptsLeft) return done(new Error('All attempts used'));
setTimeout(check, delayBetweenRequest);
});
}
check();
});
});
Example with got and async/await approach:
const utils = require('util');
const got = require('got');
const wait = utils.promisify(setTimeout);
describe('example', () => {
it('polling', async function (done) {
this.timeout(5000);
const expectedValue = '42';
const delayBetweenRequest = 100;
for (let attemptsLeft = 10; attemptsLeft; attemptsLeft -= 1) {
const resp = await got.get('http://www.google.com');
if (resp.body === expectedValue) return done();
await wait(delayBetweenRequest);
}
done(new Error('All attempts used'));
});
});
This is how I was able to do it with WebdriverIO and Mocha
describe("wait for value in content of page", () => {
it("should be able to wait to value in url", () => {
var max_seconds_to_wait = 10;
var seconds_counter = 0;
var should_continue = true;
while (should_continue) {
browser.url('http://your.url.com');
var response = JSON.parse(browser.getText("body"));
console.log(response)
if (response == 'something') {
should_continue = false;
}
browser.pause(1000);
seconds_counter++;
if (seconds_counter > max_seconds_to_wait) {
throw 'Waiting for json from url timeout error';
}
}
});
});

Resources