'use strict';
var http = require('http');
var request = require('request');
var body="";
exports.weatherWebhook = (req, res) => {
const host = 'http://api.openweathermap.org/';
const wwoApiKey = '7ffbb59524a81a6ac42ac3e942f68c5d';
var city = req.body.queryResult.parameters['geo-city'];
callWeatherApi(city).then((output) => {
//console.log(output);
res.json({ 'fulfillmentText': output }); // Return the results of the weather API to Dialogflow
}).catch(() => {
//console.log('ERROR');
res.json({ 'fulfillmentText':'hello'});
});
}
function callWeatherApi (city) {
return new Promise((resolve, reject) => {
var path = 'data/2.5/weather?' +'APPID=' + wwoApiKey +
'&q=' + encodeURIComponent(city)+'&units=imperial';
request(host+path,function(error,res,body)
{
/*var response = JSON.parse(body);
var forecast = response['main']['temp'];
var output='Current temperature is '+forecast;
resolve(output);*/
}).on('data', (d) => { body += d; console.log(body); })
.on('end', () => {
var response = JSON.parse(body);
var forecast = response['main']['temp'];
var output='Current temperature is ${forecast}';
resolve('abc')})
.on('error', (error) => {
console.log(`Error calling the weather API: ${error}`)
reject();
});
});
}
The output is not getting passed into webhook ,only hello is getting printed on the dialogflow console.When i remove the exports.webhook function and just hard code the city,output is getting printed on the terminal.why tis happens?
you query responds fine.
"http://api.openweathermap.org/data/2.5/weather?APPID=7ffbb59524a81a6ac42ac3e942f68c5d&q=houston&units=imperial"
so i'd check to see if your agent is sending the 'geo-city' parameters. you can see it in the dialogflow console bottom right when you say the city.
or console.log(city) to see if the params get passed. if not fix your dialogflow agent.
if it is sending the params, then check your what is in res or body after this line:
request(host+path,function(error,res,body)
Related
I am trying to build a weather app using node that takes
{
"cities": [
"toronto",
"mumbai",
"london"
]
}
as input and returns
{
"weather": {
"toronto": "24C",
"mumbai": "34C",
"london": "14C"
}
}
this as output
app.post('/getWeather',(req,res)=>{
const city = req.body.city;
city.map(city=>{
const url=`http://api.openweathermap.org/data/2.5/weather?q=${city}&appid=${process.env.WEATHER_API_KEY}`;
request(url, function(err, response, body) {
// On return, check the json data fetched
if (err) {
res.render('index', { weather: null, error: 'Error, please try again' });
} else {
let weather = JSON.parse(body);
console.log(weather);
Since the question does not have all of the code, I cannot give a definite answer, but I assume that the code either tries to sent response for the each city separately and/or does not wait for all the API calls to finish.
To fix the issue, async/await needs to be used (since the response depends on several API calls), and response must be sent after its completely assembled.
An example based on the given code:
app.post("/getWeather", async (req, res) => {
const cities = req.body.cities;
const reqs = cities.map((city) => {
const url = `http://api.openweathermap.org/data/2.5/weather?q=${city}&appid=${process.env.WEATHER_API_KEY}`;
return new Promise((resolve, reject) => {
request(url, function (err, response, body) {
if (err) {
reject(err);
} else {
let weather = JSON.parse(body);
resolve(weather);
}
});
});
});
const responses = await Promise.all(reqs);
const result = {};
for (const response of responses) {
if (response.ok) {
result[response.city] = response.temp;
}
}
res.json(result);
});
I'm trying to get an https.get request to assign data from within the request to a variable outside of the request. I'm also using axios. Within the https.get request, it returns the data I want in the res.on('end'... But I can't figure out how to get that data outside of the res.on('end'... portion of the request. Here is my code:
require('dotenv').config();
const express = require('express');
const {SERVER_PORT} = process.env;
const https = require('https');
const xml2js = require('xml2js');
const parser = new xml2js.Parser({ attrkey: "ATTR" });
const app = express();
app.use(express.json());
app.post('/api/ecb/forex/stats', async(req, res) => {
const {base_currency, base_amount, target_currency} = req.body;
let currencyInfo = https.get("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml", function(res) {
let data = '';
res.on('data', async function(stream) {
data += stream;
});
res.on('end', async function(){
parser.parseString(data, async function(error, result) {
if(error === null) {
return result['gesmes:Envelope'].Cube[0].Cube.forEach(element => {
console.log("at",element.Cube);
return element.Cube;
});;
}
else {
console.log(error);
}
});
});
});
console.log(currencyInfo);
})
const port = SERVER_PORT;
app.listen(port, () => console.log(`Port running on port ${port}`));
I want the value of 'element.Cube;' within the res.on('end"... portion of the https.get request to be assigned to the variable "currencyInfo". What am I doing wrong and how do I fix the code?
You can change your code to something like below, then you have Promise to return:
let currencyInfo = await new Promise((resolve, reject) => {
https.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml', function(res) {
let data = '';
res.on('data', async function(stream) {
data += stream;
});
return res.on('end', async function() {
return parser.parseString(data, async function(error, result) {
if(error === null) {
return result['gesmes:Envelope'].Cube[0].Cube.forEach(element => {
resolve(element.Cube);
});
}
else {
reject(error);
}
});
});
});
});
I'm working on a server side (self) project with node js (for the first time), and i ran into some difficulties.
My goal is the following:
first part - Im using "/uploads/processData" URL in my server to get URL(s) from the user request.
Now i want to access the user request URL(s) and get their HTML(s) file(s), to do so i'm using the "request" npm package (code below).
second part - I want access the body that I get back from the request package (from the first part), so I'm using cheerio npm package to do so.
Now to my problem - lets say that i'm trying to get the body of the url:
https://www.amazon.com/NIKE-Mens-Lunarconverge-Running-Shoes/dp/B06VVFGZHL?pd_rd_wg=6humg&pd_rd_r=61904ea4-c78e-43b6-8b8d-6b5ee8417541&pd_rd_w=Tue7n&ref_=pd_gw_simh&pf_rd_r=VGMA24803GJEV6DY7458&pf_rd_p=a670abbe-a1ba-52d3-b360-3badcefeb448&th=1
From some reason that i cant understand (probably because of lack of knowledge at web development), I dont always get the same body that i see when I review the above page (link) using F12, with my first part code. Hence sometimes my cheerio extraction (the second part) works as i expect and sometime does not (because some element from the full/original HTML file are missing). At first I thought it might be cache thing, so I added a middleware to set "nocache" flag.
What am I missing here? Does the way I try to operate wrong? Is there any way to ensure i get the same full/original HTML everytime?
Here is my code so far -
nocache middleware
function nocache(req, res, next) {
res.header("Cache-Control", "private, no-cache, no-store, must-revalidate");
res.header("Expires", "-1");
res.header("Pragma", "no-cache");
next();
}
EDIT
uploadRoutes.post("/processGoogleSearchData", nocache, (req, res) => {
//Assuming getting in req.body the google result JSON as "googleSearchResult"
var itemsArr = [];
var linksArr = [];
var bodysArr = [];
itemsArr = req.body.googleSearchResult.items;
if (itemsArr.length === 0) {
//return appropriate message
return res.status(400).send({ message: "No data sent to server" });
}
var linksArr = itemsArr.map(item => item.link);
//Get the needed info from every link
linksArr.forEach(link => {
request(link, (err, response, body) => {
if (!err && response.statusCode === 200) {
var $ = cheerio.load(body);
var tr = $(".a-lineitem").children();
var priceTd = tr.find(".a-span12");
var priceSpan = priceTd.find("#priceblock_ourprice");
console.log(priceSpan.text());
//when trying to build array of bodys the extraction doesnt work at all
bodysArr.push(body);
}
});
});
res.send(bodysArr);
});
I changed my code to the above, and it seems like the data extraction works more often. Can anyone explain why the extraction still sometimes doesnt work?
I tried return bodysArr for debbug purposes but when i do that the extraction does not work at all and my path response is always an empty array, why is that?
The problem is that:
res.send(bodysArr);
is executed straight after the call to
linksArr.forEach(link => {
The callbacks
(err, response, body) => {
if (!err && response.statusCode === 200) {
var $ = cheerio.load(body);
var tr = $(".a-lineitem").children();
var priceTd = tr.find(".a-span12");
var priceSpan = priceTd.find("#priceblock_ourprice");
console.log(priceSpan.text());
//when trying to build array of bodys the extraction doesnt work at all
bodysArr.push(body);
}
won't be guaranteed to have fired yet. What you want is ensure that res.send(bodysArr) runs after all the requests have happened
There are a few ways to handle this, one is with the excellent async library.
Hopefully you can get the gist of it with this example.
var array = [1,2,3]
function asyncRequest(input, callback){
//Do your fetch request here and call callback when done
setTimeout(callback, 10); //using setTiemout as an example
}
async.each(array, asyncRequest, (err) => {
if(err){
throw err;
}
console.log("All Finished");
});
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/2.6.1/async.min.js"></script>
After reviewing Sudsy explanation, I came across loops of asynchronous methods.
While playing with this subject I could not figure out whats wrong with my following code:
This works fine - so i ended up using it
async function getItemsInfo(itemsArr) {
return itemsArr.map(async item => {
try {
var body = await axios(item.link);
var $ = await cheerio.load(body.data);
var tr = await $(".a-lineitem").children();
var priceTd = await tr.find(".a-span12");
var priceSpan = await priceTd.find("#priceblock_ourprice");
return priceSpan.text();
} catch (err) {
return err.message;
}
});
}
getItemsInfo(linksArr)
.then(res => Promise.all(res))
.then(res => console.log(res))
.catch(err => console.error(err));
Can someone explain to me what's wrong with the following codes?
async function getItemsInfo(itemsArr) {
await Promise.all(
itemsArr.map(async item => {
try {
var body = await axios(item.link);
var $ = await cheerio.load(body.data);
var tr = await $(".a-lineitem").children();
var priceTd = await tr.find(".a-span12");
var priceSpan = await priceTd.find("#priceblock_ourprice");
return priceSpan.text();
} catch (err) {
throw err.message;
}
})
)
.then(resulst => {
return results;
})
.catch(err => {
throw err.message;
});
}
//the caller function
try {
getItemsInfo(linksArr).then(results => {
res.status(200).send(results);
});
} catch (err) {
res.status(400).send(err.message);
}
or
async function getItemsInfo(itemsArr) {
const promises = itemsArr.map(async item => {
try {
var body = await axios(item.link);
var $ = await cheerio.load(body.data);
var tr = await $(".a-lineitem").children();
var priceTd = await tr.find(".a-span12");
var priceSpan = await priceTd.find("#priceblock_ourprice");
return priceSpan.text();
} catch (err) {
return err.message;
}
});
var results = await Promise.all(promises)
.then(results => {
return results;
})
.catch(err => {
return err.message;
});
}
//the caller function
try {
getItemsInfo(linksArr).then(results => {
res.status(200).send(results);
});
} catch (err) {
res.status(400).send(err.message);
}
I am trying currently learning to build crawler using node + express +cheerio.
In the route I put this:
[index.js]
app.get('/api/crawler/android', crawlerController.android);
which calls into controller
[crawler-controller.js]
var androidCrawler = require('../crawlers/android')
module.exports.android = androidCrawler.androidget;
then I invoke the crawler (based on cheerio)
[crawler.js]
var request = require('request');
var cheerio = require('cheerio');
var androidget =request('https://www.developer-tech.com/categories/Android/', function (error, response, html){
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var result = {result:[]};
$('article').each(function (i, element) {
var Title = $(this).find("h2").text();
var Link = $(this).find("a").attr("href");
var Image = $(this).find("img").attr("src");
var payload = {
"Title":Title,
"Link":Link,
"Image":Image
};
result['result'].push(payload);
});
console.log("aaa", result);
console.log(typeof result);
return result;
}});
module.exports = {
getAndroid: function (androidget, res) {
res.send(JSON.stringify(result));
}
}
When I console log directly to crawler.js via terminal it return JSON object properly, but I think the way I export the function to be invoked by app.get is where I'm wrong and I can't figure it out.
Perhaps somebody could help me to properly invoke the crawler in my case?
There is no point of returning a result in a callback function, this will just do nothing.
What you can do is wrap your request in a function and call a callback that you create :
// file.js
const wrapFunction = (url, callback) => {
request(url, ((error, response, html) => {
// ...
callback(result);
})
}
and then use it :
// just an example
wrapFunction(yourUrl, (result) => {
// deal with your result
})
When you have that, you can export it and then use it in your middleware / controller :
// file.js
module.exports = wrapFunction;
// index.js
const wrapFunction = require('file.js'); // here is your function
app.get('/yourRoute', (req, res) => {
wrapFunction(yourUrl, (result) => {
res.send(JSON.stringify(result));
});
})
You can also use Promises :
const wrapFunction = (url) => {
return new Promise((resolve, reject) => {
request(url, ((error, response, html) => {
if (error) reject(error);
resolve(result);
});
});
};
And then :
wrapFunction(yourUrl).then(result => {
// deal with your result ...
}).catch(error => {
// deal with your error ...
});
Hope it helps,
Best regards
I got a problem with Multer, undefined req.files.path field
First, my Express.js route is:
routes.js
router.post('/', function(req, res, next){
// id, name, usersArray[], info, iconImg, headerImg
var dataObject = new MyMongooseDataObject();
// Receive data
dataObject.id = uuid.v4();
dataObject.name = req.body.name;
dataObject.usersArray = req.body.usersArray;
dataObject.info = req.body.info;
return someBindingWrapperToStoreTheFile.postFile(uuid.v4(), [req.files.iconImg.path.toString(), req.files.headerImg.path.toString()])
.then(function (postedFiles) {
dataObject.iconImg = postedFiles.body.payload.files[0].id;
dataObject.headerImg = postedFiles.body.payload.files[1].id;
//save dataObject after storing images and processing data
dataObject.save(function(savedDataObject){
next(success(req, 200, 'dataObject Saved ' + savedDataObject.id));
});
})
.catch(function(err){
console.log('FAILED: ', err.stack);
return next(failure(req, 500, err));
});
});
When I test my route with a separate small requestJS script, it works just fine:
HTTP rest api request test with requestJS
postDataObject.js
var request = require('request');
var fs = require('fs');
var uuid = require('uuid');
var formData = {
name: 'someName',
info: 'Some INFO and text description. ',
'usersArray[0][uuid]': uuid.v4().toString(),
'usersArray[1][uuid]': uuid.v4().toString(),
'usersArray[2][uuid]': uuid.v4().toString(),
// handle files
iconImg: fs.createReadStream('/var/tmp/img/iconImg.png'),
headerImg: fs.createReadStream('/var/tmp/img/headerImg.png')
};
request.post({url:'http://127.0.0.1:2233/api/postDataObject', formData: formData}, function (err, httpResponse, body) {
if (err) {
return console.error('failed:', err);
}
console.log('DataObject creation is successful! Server responded with:', body);
});
Now I'm writing the wrapper library for the fronted usage, when I use the same code in another context, it seems, that Multer handling req.files.headerImg.path is not working, it's undefined.
The code I use for wrapper library:
wrapper-lib.js
var request = require('request'),
URI = require('URIjs'),
fs = require('fs'),
uuid = require('uuid'),
path = require('path'),
Promise = require('bluebird'),
_ = require('lodash'),
DataObjectBindings.prototype.createDataObject = function (jsonRequestJSFormData) {
var self = this;
// this give the URL of the API to make requests to
return self.getAPIurlHelper().then(function (apiUrl) {
return new Promise(function (resolve, reject) {
request.post({url: apiUrl, form: jsonRequestJSFormData}, function (err, res, body) {
if (err) {
reject(err);
} else {
resolve({res: res, body: JSON.parse(body)});
};
}); //end of .post
}); // end of Promise
}); // end of getAPIurlHelper() function
}; // end of createDataObject function definition
And Finally I test the warpper with:
wrapper-test.js
// Instantiate Broker Client
var Wrapper = require('./wrapper-lib');
var wi; //wrapper instalnce
var ConnectApiRequestTracer = require('./connectApiRequestTracer');
var fs = require('fs');
ConnectApiRequestTracer().connect()
.then(function () {
wi = new Wrapper();
return wi.createDataObject(
uuid.v4(),
{
name: 'Some Cool Name',
info: 'Some nice description and info. ',
'usersArray[0][uuid]': uuid.v4().toString(),
'usersArray[1][uuid]': uuid.v4().toString(),
'usersArray[2][uuid]': uuid.v4().toString(),
iconImg: fs.createReadStream('/var/tmp/img/iconImg.png'),
headerImg: fs.createReadStream('/var/tmp/img/headerImg.png'),
);
})
.then(function (data) {
console.log('SUCCESS: ', data);
})
.catch(function (err) {
console.log('FAILED', err.stack);
});
When I run the test case wrapper-test.js, it throws me an error, that says the req.files.iconImg.path is undefined.
Any ideas what can be wrong?
try using :
req.file.iconImg.path
req.file.headerImg.path
instead of req.files.iconImg.path and req.files.headerImg.path