Having quite a bit of trouble getting an HTML page to download using NodeJS. Here is my code snippet:
const request = require('request');
request('http://www.google.com', { json: true }, (err, res, body) => {
if (err) {
return console.log(err);
}
console.log(body.url);
console.log(body.explanation);
});
When I step through this it executes in about half a second. I get no errors back but I'm not getting any content logged to the console...
This works for me.
const request = require('request')
request('https://google.com', (err, res, body) => console.log(err ? err : body))
With request you can pipe the response body of a request directly to a WriteableStream
const fs = require('fs')
const request = require('request')
request('https://google.com').pipe(fs.createWriteStream('./google-index.html'))
Per the comments below, the following example illustrates how to wrap this request so it can be awaited and printed to the screen or written to a file.
const {promisify} = require('util')
const fs = require('fs')
const writeFile = promisify(fs.writeFile)
const request = require('request')
const getGoogleIndexHTML = () => {
return new Promise((resolve, reject) => {
request('https://google.com', (err, res, body) => err ? reject(err) : resolve(body))
})
}
const printAndWriteGoogleIndex = async () => {
try {
let googleIndexHTML = await getGoogleIndexHTML()
console.log(googleIndexHTML)
await writeFile('./google-index.html', googleIndexHTML, 'utf8')
console.log('google-index.html written.')
} catch(err) {
console.log(err)
}
}
printAndWriteGoogleIndex()
Related
I'm trying to get an https.get request to assign data from within the request to a variable outside of the request. I'm also using axios. Within the https.get request, it returns the data I want in the res.on('end'... But I can't figure out how to get that data outside of the res.on('end'... portion of the request. Here is my code:
require('dotenv').config();
const express = require('express');
const {SERVER_PORT} = process.env;
const https = require('https');
const xml2js = require('xml2js');
const parser = new xml2js.Parser({ attrkey: "ATTR" });
const app = express();
app.use(express.json());
app.post('/api/ecb/forex/stats', async(req, res) => {
const {base_currency, base_amount, target_currency} = req.body;
let currencyInfo = https.get("https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml", function(res) {
let data = '';
res.on('data', async function(stream) {
data += stream;
});
res.on('end', async function(){
parser.parseString(data, async function(error, result) {
if(error === null) {
return result['gesmes:Envelope'].Cube[0].Cube.forEach(element => {
console.log("at",element.Cube);
return element.Cube;
});;
}
else {
console.log(error);
}
});
});
});
console.log(currencyInfo);
})
const port = SERVER_PORT;
app.listen(port, () => console.log(`Port running on port ${port}`));
I want the value of 'element.Cube;' within the res.on('end"... portion of the https.get request to be assigned to the variable "currencyInfo". What am I doing wrong and how do I fix the code?
You can change your code to something like below, then you have Promise to return:
let currencyInfo = await new Promise((resolve, reject) => {
https.get('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml', function(res) {
let data = '';
res.on('data', async function(stream) {
data += stream;
});
return res.on('end', async function() {
return parser.parseString(data, async function(error, result) {
if(error === null) {
return result['gesmes:Envelope'].Cube[0].Cube.forEach(element => {
resolve(element.Cube);
});
}
else {
reject(error);
}
});
});
});
});
I have a number of stock tickers in a JSON file that I want to use to send multiple GET requests to get the price of that stock. The problem I'm having is how to send them off in parallel and how to resolve them.
Here is the code I have tried:
const stocks = require('./stocks.json')
var request = require("request");
var stockTickers = stocks.map(x => x.stockopediaTicker)
stockTickers.forEach(ticker => {
var promises = []
var options = {
method: 'GET',
url: 'https://www.stockopedia.com/ajax/get_prices/' + ticker + '/'
};
let todaysQuote = new Promise(function (res, rej) {
request(options, function (error, response, body) {
rej(error)
res(body)
});
})
promises.push(todaysQuote)
});
Promise.all(promises)
.then((results) => {
console.log(results)
}).catch(err => console.log(err))
You were on the right path with your code.
You just need to only call rej(err) if there is actually an error
You shouldn't be attempting to share the same options object with all your requests (that might or might not cause problems)
You need to declare promises in a higher scope where it can be used with Promise.all(promises):
Here's what it would look like after fixing up those issues:
const stocks = require('./stocks.json')
const request = require("request");
let promises = [];
let stockTickers = stocks.map(x => x.stockopediaTicker);
stockTickers.forEach(ticker => {
let todaysQuote = new Promise(function (res, rej) {
let options = {
method: 'GET',
url: 'https://www.stockopedia.com/ajax/get_prices/' + ticker + '/'
};
request(options, function (error, response, body) {
if (error) {
rej(error);
} else {
res(body);
}
});
})
promises.push(todaysQuote)
});
Promise.all(promises)
.then((results) => {
console.log(results);
}).catch(err => console.log(err))
The request-promise module is simpler to use as it already includes a promise wrapper and it's probably easier to use .map() to accumulate your array of promises.
const stocks = require('./stocks.json')
const rp = require("request-promise");
Promise.all(stocks.map(x => {
let options = {
method: 'GET',
url: 'https://www.stockopedia.com/ajax/get_prices/' + x.stockopediaTicker + '/'
};
return rp(options);
})).then(results => {
console.log(results);
}).catch(err => {
console.log(err);
});
You should check if there is an error and only rej if there is an error:
let todaysQuote = new Promise(function (res, rej) {
request(options, function (error, response, body) {
if(error) {
return rej(error)
}
res(body)
});
})
Right now, you're "rejecting" every response.
I'm not sure about "request" but using "request-promise-native" you can simplify some things like this.
const stocks = require('./stocks.json');
const request = require('request-promise-native');
const parseStocks = (stocks)=>Promise.all(stocks.map(requestQuote));
const requestQuote = ({stockopediaTicker})=>{
const options = {
method: 'GET',
url: `https://www.stockopedia.com/ajax/get_prices/${stockopediaTicker}/`
};
return request(options)
}
parseStocks(stocks).then(console.log).catch(console.log)
Normally I refrain from suggesting dependency changes in an answer, but in this case, 'request-promise-native' is suggested by the 'request' docs. If you plan on using promises you may want to switch. It's usually best practice to avoid combining callbacks and promise chains together.
I am trying to make this simple api with the help of this article .The primary purpose of reading this article is to learn react native but it starts with a server in nodejs .I couldn't get it running correctly .
Here is the link to article link
Here is the code for server.js file
import express from 'express';
import http from 'http';
import giphyapi from 'giphy-api';
import base64 from 'base64-stream';
// Initialize http server
const app = express();
// Register /gif endpoint that returns base64 encoded gif
app.get('/gif', async (req, res) => {
res.json({
gif: await fetchGif(),
});
});
// Launch the server on port 3000
const server = app.listen(3000, () => {
const { address, port } = server.address();
console.log(`Listening at http://${address}:${port}`);
});
// Fetch random GIF url with Giphy API, download and Base64 encode it
export const fetchGif = async () => {
const item = await giphyapi().random('cat');
return await encode(await download(item.data.image_url));
};
// File download helper
const download = async (url) => {
return new Promise((resolve, reject) => {
let req = http.get(url.replace('https', 'http'));
req.on('response', res => {
resolve(res);
});
req.on('error', err => {
reject(err);
});
});
};
// Base64 encode helper
const encode = async (content) => {
let output = 'data:image/gif;base64,';
const stream = content.pipe(base64.encode());
return new Promise((resolve, reject) => {
stream.on('readable', () => {
let read = stream.read();
if (read) {
output += read.toString();
}
else {
resolve(output);
}
});
stream.on('error', (err) => {
reject(err);
});
});
};
the error is occurring because the third-party library "base64-stream" does not have the "encoded" function, it may exist in previous versions.
To solve this problem, you need to change some lines of your code so that your server looks like the code below.
const express = require('express'); // instead of "import express from 'express';"
const http = require('http'); // instead of "import http from 'http';"
const giphyapi = require('giphy-api'); // instead of "import http from 'http';"
const { Base64Encode } = require('base64-stream'); // instead of "import base64 from 'base64-stream';"
// Initialize http server
const app = express();
// Register /gif endpoint that returns base64 encoded gif
app.get('/gif', async (req, res) => {
try {
const gif = await fetchGif();
res.json({ gif });
} catch (error) {
res.status(500).send({ error });
}
});
// Base64 encode helper
const encode = (content) => {
let output = 'data:image/gif;base64,';
const stream = content.pipe(new Base64Encode()); // instead of "const stream = content.pipe(base64.encode());"
return new Promise((resolve, reject) => {
stream.on('readable', () => {
let read = stream.read();
if (read) {
output += read.toString();
}
else {
resolve(output);
}
});
stream.on('error', (err) => {
reject(err);
});
});
};
// Launch the server on port 3000
const server = app.listen(3000, () => {
const { address, port } = server.address();
console.log(`Listening at http://${address}:${port}`);
});
// Fetch random GIF url with Giphy API, download and Base64 encode it
const fetchGif = async () => {
try {
const item = await giphyapi().random('cat');
const image = await download(item.data.image_url);
return await encode(image);
} catch (error) {
console.log('fetchGif', error);
}
};
// File download helper
const download = (url) => {
return new Promise((resolve, reject) => {
let req = http.get(url.replace('https', 'http'));
req.on('response', res => {
resolve(res);
});
req.on('error', err => {
reject(err);
});
});
};
base64-stream doesn't have any function called encode(..).
There are two Classes Base64Encode, Base64Decode which are used to stream.
'use strict';
var http = require('http');
var request = require('request');
var body="";
exports.weatherWebhook = (req, res) => {
const host = 'http://api.openweathermap.org/';
const wwoApiKey = '7ffbb59524a81a6ac42ac3e942f68c5d';
var city = req.body.queryResult.parameters['geo-city'];
callWeatherApi(city).then((output) => {
//console.log(output);
res.json({ 'fulfillmentText': output }); // Return the results of the weather API to Dialogflow
}).catch(() => {
//console.log('ERROR');
res.json({ 'fulfillmentText':'hello'});
});
}
function callWeatherApi (city) {
return new Promise((resolve, reject) => {
var path = 'data/2.5/weather?' +'APPID=' + wwoApiKey +
'&q=' + encodeURIComponent(city)+'&units=imperial';
request(host+path,function(error,res,body)
{
/*var response = JSON.parse(body);
var forecast = response['main']['temp'];
var output='Current temperature is '+forecast;
resolve(output);*/
}).on('data', (d) => { body += d; console.log(body); })
.on('end', () => {
var response = JSON.parse(body);
var forecast = response['main']['temp'];
var output='Current temperature is ${forecast}';
resolve('abc')})
.on('error', (error) => {
console.log(`Error calling the weather API: ${error}`)
reject();
});
});
}
The output is not getting passed into webhook ,only hello is getting printed on the dialogflow console.When i remove the exports.webhook function and just hard code the city,output is getting printed on the terminal.why tis happens?
you query responds fine.
"http://api.openweathermap.org/data/2.5/weather?APPID=7ffbb59524a81a6ac42ac3e942f68c5d&q=houston&units=imperial"
so i'd check to see if your agent is sending the 'geo-city' parameters. you can see it in the dialogflow console bottom right when you say the city.
or console.log(city) to see if the params get passed. if not fix your dialogflow agent.
if it is sending the params, then check your what is in res or body after this line:
request(host+path,function(error,res,body)
I am trying currently learning to build crawler using node + express +cheerio.
In the route I put this:
[index.js]
app.get('/api/crawler/android', crawlerController.android);
which calls into controller
[crawler-controller.js]
var androidCrawler = require('../crawlers/android')
module.exports.android = androidCrawler.androidget;
then I invoke the crawler (based on cheerio)
[crawler.js]
var request = require('request');
var cheerio = require('cheerio');
var androidget =request('https://www.developer-tech.com/categories/Android/', function (error, response, html){
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var result = {result:[]};
$('article').each(function (i, element) {
var Title = $(this).find("h2").text();
var Link = $(this).find("a").attr("href");
var Image = $(this).find("img").attr("src");
var payload = {
"Title":Title,
"Link":Link,
"Image":Image
};
result['result'].push(payload);
});
console.log("aaa", result);
console.log(typeof result);
return result;
}});
module.exports = {
getAndroid: function (androidget, res) {
res.send(JSON.stringify(result));
}
}
When I console log directly to crawler.js via terminal it return JSON object properly, but I think the way I export the function to be invoked by app.get is where I'm wrong and I can't figure it out.
Perhaps somebody could help me to properly invoke the crawler in my case?
There is no point of returning a result in a callback function, this will just do nothing.
What you can do is wrap your request in a function and call a callback that you create :
// file.js
const wrapFunction = (url, callback) => {
request(url, ((error, response, html) => {
// ...
callback(result);
})
}
and then use it :
// just an example
wrapFunction(yourUrl, (result) => {
// deal with your result
})
When you have that, you can export it and then use it in your middleware / controller :
// file.js
module.exports = wrapFunction;
// index.js
const wrapFunction = require('file.js'); // here is your function
app.get('/yourRoute', (req, res) => {
wrapFunction(yourUrl, (result) => {
res.send(JSON.stringify(result));
});
})
You can also use Promises :
const wrapFunction = (url) => {
return new Promise((resolve, reject) => {
request(url, ((error, response, html) => {
if (error) reject(error);
resolve(result);
});
});
};
And then :
wrapFunction(yourUrl).then(result => {
// deal with your result ...
}).catch(error => {
// deal with your error ...
});
Hope it helps,
Best regards