I'm trying to return the results of an http request from a function and thought request-promise was supposed to return after an await. I obviously don't have this right. I get "undefined" returned from the function.
var s = getUrl();
console.log('result: ' + s)
function getUrl() {
var rp = require('request-promise');
var url = 'http://myserver.com?param=xxx';
rp(url)
.then(function (data) {
return data;
})
.catch(function (err) {
console.log(err);
});
}
How do I await the call so getUrl returns the data?
EDIT: After Kevin's comment. I tried to put this into a module and call it but it's returning [object Promise].
function getUrl(url, params) {
var rp = require('request-promise');
return rp(url + '?' + params)
.then(function (data) {
return data;
})
.catch(function (err) {
console.log(err);
});
}
async function getUpdate(o) {
var url = 'http://myserver.com';
var params = 'param=xxx';
var s = await getUrl(url, params);
return s;
}
exports.askVelo = (o) => {
var sRtn = '';
console.log(o.code);
switch (o.code) {
case 'g':
sRtn = getUpdate(o);
break;
}
console.log('heres rtn: ' + sRtn); // sRtn is [object Promise]
return sRtn;
}
getUpdate is now just returning [object Promise]... Why is that not working now?
I really needed to get the data in a synchronous fashion, so I finally gave up and used sync-request and skipped all the await/promise stuff.
sr = require('sync-request');
var url = 'http://myserver';
var qstring = 'param=xxx';
var res = sr('GET', url + '?' + qstring);
var str = res.body.toString();
Related
I'm a beginner of nodejs, async bothers me.
I want my code run sequencely or it will breaks.
I have a for loop, and it simply doesn't work...
Here are all the codes:
const util = require('util');
const request = require('request');
const cheerio = require('cheerio');
var host = "http://www.nicotv.me";
var url = "http://www.nicotv.me/video/play/57838-1-%s.html";
var len = 99;
var tab = /-(\d)-/.exec(url);
tab = tab[1] // '1' not '-1-'
function getLen(url) {
//you can ignore this function, it gives len=2
request(url, function (err, response, html) {
if (err) {
console.log('url:', url);
console.log('error:', err);
console.log('statusCode:', response && response.statusCode);
}
else{
var $ = cheerio.load(html);
var cls = '.ff-playurl-dropdown-%s';
$(util.format(cls, tab)).filter(function (){
var data = $(this);
len = data.html().match(/<a href=/g).length;
console.log("episode:", len);
});
getLink(len, function(){
});
}
});
}
getLen(util.format(url, 1)); //len = 2
var getLink = function(lengths, callback){
for (let i = 1; i <= lengths; i++) {
var tmp = util.format(url, i);
try {
request(tmp, function (err, res, html){
console.log('url:', tmp);
if(err){
console.log("error:", err);
console.log("statusCode:", res && res.statusCode);
}else{
var reg = /src="(\/player.php?.{1,})"/;
var result = reg.exec(html);
console.log(result[1]);
}
});
callback();
} catch (error) {
console.log(error);
break;
}
}
}
here is my output:
episode: 2
url: http://www.nicotv.me/video/play/57838-1-2.html
/player.php?u=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1NzkxMzU2MzEyNDAwNTQ5&p=360biaofan&c=0&j=aHR0cDovL2ppZXhpLmtpbmdzbnVnLmNuLzM2MGJpYW9mYW4ucGhwP3VybD0=&x=10&y=&z=
url: http://www.nicotv.me/video/play/57838-1-2.html
/player.php?u=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1Nzg1MDQyMDYyNDAwNTgx&p=360biaofan&c=0&j=aHR0cDovL2ppZXhpLmtpbmdzbnVnLmNuLzM2MGJpYW9mYW4ucGhwP3VybD0=&x=10&y=&z=aHR0cDovL3R5angyLmtpbmdzbnVnLmNuLzM2MHl1bi0xNS5waHA/dmlkPTE1NzkxMzU2MzEyNDAwNTQ5
First problem is these two /player*** link are from 57838-1-1.html
And one of them are not complete.
Second problem is the url output shows 57838-1-2.html twice.
Thanks for your kindly help.
Yesterday had the same problem, so I solved with:
Using request-promise
Replace the loop method arrTitles.Each with for (const jt of arrTitles)
Here a sample:
const request = require('request-promise');
const cheerio = require('cheerio');
var getUrlData =
async function (url) {
console.log(url);
try {
return await request.get(url);
}
catch (err) {
console.error(`${err}: ${url}`);
}
return;
};
var run =
async function (pageUrl) {
var arrData =
await fn.getUrlData(pageUrl)
.then(response => readTable(response));
console.log(arrData);
};
var readTable =
function (document) {
var $;
let arrData = [];
try {
$ = cheerio.load(document);
$('table tr')
.each(
function (trN) {
$(this)
.children('td')
.each(
function (tdN) {
arrData.push($(this).text().trim());
}
)
});
}
catch { }
return arrData;
};
run();
I'm trying to call a REST API in a "for" loop, however, the results aren't what I'm expecting.
I've attempted to wrap everything in a promise, but the order of operations is still off, executing it asynchronously rather than synchronously.
var https = require('https');
var zlib = require("zlib");
var axios = require('axios');
const cheerio = require('cheerio');
var page = 1;
var hasMore = "true";
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function getLocation(page) {
// notice that we can await a function
// that returns a promise
await delay();
var apiUrl = 'https://my.api.com/search/advanced?page=' + page +
'&pagesize=5';
https.get(apiUrl, function(response) {
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function(chunk) {
jsonString += chunk;
});
gunzip.on('end', function() {
obj = JSON.parse(jsonString);
var url = obj.items[0].owner.link;
axios(url)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
//OUTPUT LOCATION
console.log($('h3.location').text().trim());
})
.catch(console.error);
});
gunzip.on('error', function(e) {
console.log(e);
});
} else {
console.log("Error");
}
});
}
async function startGetLocation() {
var page = 1;
var hasMore = "true";
do {
//OUTPUT PAGE NUMBER
console.log(page.toString());
await getLocation(page);
page = page + 1;
} while (page < 3);
}
startGetLocation();
Based on the sample code, I would have expected the below to output:
1
New York
2
However, it's outputting:
1
2
New York
The problem is that the callback function that you passed to the https.get() function gets executed asynchronously and that the getLocation function does not wait until this part resolves.
So you could simply wrap the https.get() call and the unzipping part in a promise, wait for it to resolve and then do the axios-part.
async function getLocation(page) {
await delay();
var apiUrl = 'https://my.api.com/search/advanced?page=' + page +
'&pagesize=5';
const fetchAndUnzipPromise = new Promise((resolve, reject) => {
https.get(apiUrl, function (response) {
console.log("headers: ", response.headers);
console.log(response.statusCode)
if (response.statusCode == 200) {
var gunzip = zlib.createGunzip();
var jsonString = '';
response.pipe(gunzip);
gunzip.on('data', function (chunk) {
jsonString += chunk;
});
gunzip.on('end', function () {
obj = JSON.parse(jsonString);
var url = obj.items[0].owner.link;
resolve(url);
});
gunzip.on('error', function (e) {
reject(e);
});
} else {
reject(new Error("Statuscode not as exepcted"));
}
});
});
return fetchAndUnzipPromise.then(url => {
return axios(url)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
//OUTPUT LOCATION
console.log($('h3.location').text().trim());
})
.catch(console.error);
})
}
I'm trying to make a webscraper, but I can't get my function to wait for the second request to fill the name key on my object. It always return undefined.
const request = require('request');
const cheerio = require('cheerio');
const base_url = 'https://www.supremenewyork.com';
const shop_url = 'https://www.supremenewyork.com/shop/';
function getItems(category) {
var items = [];
return new Promise(function(resolve, reject) {
request.get(shop_url + category, function(err, res, body) {
if(err) {
reject(err);
} else {
var $ = cheerio.load(body);
$('a', '.inner-article').each(function(i, el) {
var url = base_url + $(this).attr('href');
var isSoldout = false;
var name;
if($(this).find('div').attr('class', 'sold_out_tag').length === 1)
isSoldout = true;
request.get(url, function(err, res, html) {
var $ = cheerio.load(html);
name = $('h1', 'div').text();
})
items.push({name: name, url: url, isSoldout: isSoldout});
})
resolve(items);
}
})
})
}
I expect the name key to be fill but no, i get undefined
Use the request-promise package which wraps request in Promise. Then you can use async/await to wait for result like:
const rp = require('request-promise');
const cheerio = require('cheerio');
const base_url = 'https://www.supremenewyork.com';
const shop_url = 'https://www.supremenewyork.com/shop/';
// notice async keyword
async function getItems(category) {
var items = [];
try {
// using await to wait for promise to resolve
const body = await rp.get(shop_url + category)
var $ = cheerio.load(body);
$('a', '.inner-article').each(function(i, el) {
var url = base_url + $(this).attr('href');
var isSoldout = false;
var name;
if($(this).find('div').attr('class', 'sold_out_tag').length === 1)
isSoldout = true;
try {
const html = await rp.get(url)
var $ = cheerio.load(html);
name = $('h1', 'div').text();
items.push({name: name, url: url, isSoldout: isSoldout});
} catch (err) {
throw err;
}
})
} catch (e) {
throw e;
}
return items;
}
More about async/await at MDN
I'm very new to node, and I'm trying to pull a list of IDs from an API, iterate through that list for each ID saving the output, and ultimately rename each file generated. The code below is the closest I've come, and while it works sometimes, it frequently fails as I believe one function isn't waiting for the other to complete (e.g. tries to read before a write), but I'm sure I have other issues going on.
const apiKey = inputData.apiKey
var https = require('https');
var sync = require('sync');
var fs = require('fs');
var JSONfileloc = "./pdfs/file.json"
var queryurl = 'https://intakeq.com/api/v1/intakes/summary?startDate=2018-01-01'
var authHeaders = { 'X-Auth-Key': apiKey }
var queryOpts = { method: 'GET', headers: authHeaders}
function handleFile (error, file)
{
if (error) return console.error('Ran into a problem here', error)
}
fetch(queryurl, queryOpts)
.then
(function findAPI(res, err)
{
if( err )
{ console.log('I cant find the API '+err) }
return res.json()
{console.log('found the API!')}
}
)
.then (function itID(res, err)
{
if( err )
{ console.log('I cant iterate the API '+err) }
for(var i = 0; i < res.length; i++)
{
var intakeID=res[i].Id;
var APIoptions={ host:"intakeq.com", path:"/api/v1/intakes/"+ intakeID, headers: authHeaders };
var PDFoptions={ host:"intakeq.com", path:"/api/v1/intakes/"+ intakeID+'/pdf', headers: authHeaders };
console.log('Working on ID:'+intakeID)
var JSONrequest = https.get(APIoptions, writeJSON)
}})
//READ JSON FUNCTION
function readJSON (err, data)
{
if (err) throw err;
if(data.indexOf('New Patient Forms') >= 0)
var contents = fs.readFileSync(JSONfileloc, handleFile);
var jsonContent = JSON.parse(contents)
//pull PT Name
pName = (jsonContent.ClientName);
console.log('The Patient Name Is ' + jsonContent.ClientName)
//pull PT DOB
pDob = (jsonContent.Questions[3].Answer)
console.log('Patient DOB Is ' + jsonContent.Questions[3].Answer)
//pull Form Type
pForm = (jsonContent.QuestionnaireName)
console.log('The Form Submitted is ' + jsonContent.QuestionnaireName)
//rename and move JSON
fs.rename("./pdfs/file.json", './JSONLogs/'+pName+' '+pForm+' '+Date.now()+'.json', function(err) {
if ( err ) console.log('Problem renaming! ' + err)
else console.log('Copying & Renaming JSON File!');
})
};
//WRITE JSON FUNCTION
function writeJSON(response, err)
{
var JSONfile = fs.createWriteStream(JSONfileloc, handleFile);
if (err) throw err;
response.pipe(JSONfile);
console.log('JSON Created')
fs.readFile(JSONfileloc, readJSON)
}
The research I've done leads me to believe that async.forEach is probably the right approach here, but I've been having a hard time getting that to work properly. Thanks in advance and any suggestions are much appreciated.
const apiKey = inputData.apiKey
var https = require('https');
var sync = require('sync');
var fs = require('fs');
var JSONfileloc = "./pdfs/file.json"
var queryurl = 'https://intakeq.com/api/v1/intakes/summary?startDate=2018-01-01'
var authHeaders = {
'X-Auth-Key': apiKey
}
var queryOpts = {
method: 'GET',
headers: authHeaders
}
function handleFile(error, file) {
if (error) return console.error('Ran into a problem here', error)
}
fetch(queryurl, queryOpts)
.then(function findAPI(res) {
return res.json();
})
.then(function itID(res) {
const JSONRequests = [];
for (var i = 0; i < res.length; i++) {
var intakeID = res[i].Id;
var APIoptions = {
host: "intakeq.com",
path: "/api/v1/intakes/" + intakeID,
headers: authHeaders
};
var PDFoptions = {
host: "intakeq.com",
path: "/api/v1/intakes/" + intakeID + '/pdf',
headers: authHeaders
};
// https.get has response as a stream and not a promise
// This `httpsGet` function converts it to a promise
JSONRequests.push(httpsGet(APIoptions, i));
}
return Promise.all(JSONRequests);
})
function httpsGet(options, filename) {
return new Promise((resolve, reject) => {
https.get(options, (response) => {
// The WriteJSON function, just for brewity
// Otherwise pass resolve to the seperate writeJSON and call it in there
var JSONfile = fs.createWriteStream(filename + ".json");
response.pipe(JSONfile);
JSONfile.on('close', () => {
readJSON(filename + ".json").then(() => {
resolve();
})
})
})
})
}
//READ JSON FUNCTION
function readJSON(filename) {
// if (err) throw err;
var contents = fs.readFileSync(filename, 'utf-8'); // removed handleFile as readFileSync does not allow callbacks, added format
var jsonContent = JSON.parse(contents)
// Make your conditional checks here with the jsonContents
//pull PT Name
pName = (jsonContent.ClientName);
console.log('The Patient Name Is ' + jsonContent.ClientName)
//pull PT DOB
pDob = (jsonContent.Questions[3].Answer)
console.log('Patient DOB Is ' + jsonContent.Questions[3].Answer)
//pull Form Type
pForm = (jsonContent.QuestionnaireName)
console.log('The Form Submitted is ' + jsonContent.QuestionnaireName)
//rename and move JSON
return new Promise((resolve, reject) => {
fs.rename("./pdfs/file.json", './JSONLogs/' + pName + ' ' + pForm + ' ' + Date.now() + '.json', function (err) {
if (err) {
console.log('Problem renaming! ' + err);
reject(err);
} else {
console.log('Copying & Renaming JSON File!');
resolve();
}
})
})
};
Updated to convert https.get response stream to return a Promise which can be handled much better.
I'm working with Node, Express and Objection.js.
My file structure is:
->Project Folder
->Controllers(folder)
->agenciaController.js
->Models(folder)
->Agencia.js
->index.js
in my index.js code is:
var Knex = require('knex');
var express = require("express");
var app = express();
var knexConfig = require('./knexfile');
var Model = require('objection').Model;
var knex = Knex(knexConfig.development);
Model.knex(knex);
var reqAgenciaController = require('./Controllers/agenciaController');
var agenciaController = new reqAgenciaController();
app.get("/agencias", function(req, res) {
var respuesta = agenciaController.getAgencias();
console.log("log3: "+respuesta);
});
And my agenciaController.js is:
var Agencia = require('../Models/Agencia');
module.exports = agenciaController;
function agenciaController() {
this.getAgencias = function() {
var retValue = "retvalue";
listaAgencias = [];
Agencia.query()
.then(function(agencias) {
listaAgencias = agencias;
console.log("log1: "+ listaAgencias);
});
console.log("log2: "+listaAgencias);
return listaAgencias;
}
}
My problem is that when I run the code, I get
log2:
log3: undefined
log1: [object Object, object Object].
But I need:
log1: [object Object, object Object].
log2: [object Object, object Object].
log3: [object Object, object Object].
My question is: is it possible to return listaApuestas from the controller to index.js, with the value obtained in the promise?
agenciaController.getAgencias() should be promosified using the promise library of your choice.
I recommend using bluebird.
The trick part is that if a promise reside in another promise you'll need to change your structure a little bit.
Sample agenciaController.js
var Promise = require("bluebird");
agenciaController = {
getAgencias: function(callback) {
this.queryAsync().then(function(respuesta) {
console.log("log2: "+ respuesta);
callback(false, respuesta);
})
},
query: function(callback) {
console.log("log1", {});
callback(false, { respuesta: "respuesta" })
}
}
Promise.promisifyAll(agenciaController);
module.exports = agenciaController;
Sample app.js
var AgenciaController = require('./agenciaController');
agenciaController.getAgenciasAsync().then(function(respuesta) {
console.log("log3: " + respuesta);
});
This will output:
log1 {}
log2: [object Object]
log3: [object Object]
Thank you for your help! Finally studying more about promises and following their examples, solved the problem.
Here is the code I use to solve the problem
My agenciaController.js:
var Agencia = require('../Models/Agencia');
module.exports = agenciaController;
function agenciaController() {
this.getAgencias = function() {
return Agencia.query()
.then(function(agencias) {
return agencias;
})
.catch(function(){
console.log("error en la promesa");
});
}
}
And my index.js:
var reqAgenciaController = require('./Controllers/agenciaController');
var agenciaController = new reqAgenciaController();
app.get("/agencias", function(req, res) {
var misAgencias;
var promise = new Promise(function(resolve, reject) {
if (agenciaController.getAgencias() === undefined){
reject("Error al llamar al getAgencias");
}else {
resolve(agenciaController.getAgencias());
}
});
promise
.then( agencias => {
misAgencias = agencias;
console.log(misAgencias);
res.send(misAgencias);
})
.catch(err => console.log("Error: "+err))
});
fn: async function() {
var output = '';
await myPromise()
.then((data) => {
output = data;
})
.catch((err) => {
output = "Error: " + err;
});
return ans;
}
Notice how we await for a promise to finish, before fn returns its value