I've been working on a discord/roblox bot. I've gotten almost finished, but ran into a problem. I'm a noob at node, so please dont harp too much if the code is bad. What this does is I click a button in a roblox game, it sends a POST request to a heroku app, and then with the info provided it sends a message to a channel in my discord server.
Relevant bits of code:
const Discord = require('discord.js');
const request = require('request');
var express = require('express');
app = express();
bodyParser = require('body-parser');
path = require('path');
var bodyParser = require('body-parser');
var PORT = process.env.PORT || 5000;
const bot = new Discord.Client();
function GetData(id){
let url = "https://economy.roblox.com/v1/assets/"+id+"/resale-data";
let url2 = "https://thumbnails.roblox.com/v1/assets?assetIds="+id+"&size=150x150&format=Png";
let url3 = "http://api.roblox.com/marketplace/productinfo?assetId="+id
let options = {json: true};
var response = {};
request(url, options, (error, res, body) => {
if (error) {
return console.log(error)
};
if (!error && res.statusCode == 200) {
response.rap = body.recentAveragePrice
console.log(response)
};
});
request(url2, options, (error, res, body) => {
if (error) {
return console.log(error)
};
if(!error && res.statusCode == 200) {
response.thumbnail = body.data[0].imageUrl
console.log(response)
}
});
request(url3, options, (error, res, body) => {
if (error) {
return console.log(error)
};
if(!error && res.statusCode == 200) {
response.name = body.Name
console.log(response)
}
});
return response
};
app.post('/exec', function(req, res) {
res.setHeader('Content-Type', 'application/json')
res.send(JSON.stringify({
success: true
}));
console.log("working");
console.log(req.body)
var data = GetData(req.body.id)
if (req.body.state === true){
var state = "New Projected"
} else {
var state = "No longer projected"
}
const msg = new Discord.RichEmbed()
.setDescription(data.name)
.setAuthor(state)
.setColor(0x1c90d9)
.addField("RAP:", data.rap)
.setImage(data.thumbnail)
bot.channels.get("655196831835226133").send(msg);
});
However, this is the message that is sent in the discord server:
everything is undefined and there is no thumbnail
Help is greatly appreciated. Thanks a lot.
Some reading here: https://www.promisejs.org/
Asynchronous functions, like making a request to a server, take time to return and do so in a callback. You have to keep track of the paths of the callbacks to make sure you do not do the next step before a previous step has completed otherwise you will skip over it entirely. It is not like calling a regular function that does something in order.
When you do return response at the end of GetData, there is nothing in the object because you need to wait for the three responses to complete within their respective callbacks. You are seeing things logged to the console because they are happening after the fact in the background. If you were to put timestamps and additional logging, you would see that it is not all happening in the order of how you wrote it, but rather in order of when the requests themselves complete.
There are many different ways to write and use promises, but a very simple one here would be to nest the requests so they happen one after the other, and then finally resolve the promise at the very end when your response object has all the data. With Promises you resolve which is the same as returning, and reject when you want to raise an error.
Without going too deep into it, here is an example of what you can do. This is untested as I am not running your application or trying to rewrite it, that is not was StackOverflow is for, but hopefully it can send you in the right direction. Note that I call reject on the errors to tell the promise to stop running.
function GetData(id){
// declare the Promise
return new Promise((resolve, reject) => {
let url = "https://economy.roblox.com/v1/assets/"+id+"/resale-data";
let url2 = "https://thumbnails.roblox.com/v1/assets?assetIds="+id+"&size=150x150&format=Png";
let url3 = "http://api.roblox.com/marketplace/productinfo?assetId="+id
let options = {json: true};
var response = {};
// make request 1
request(url, options, (error1, res1, body1) => {
if (error1) {
console.log(error1)
// reject
reject(error1);
};
if (!error1 && res1.statusCode == 200) {
// all good, continue
response.rap = body1.recentAveragePrice
console.log(response)
// make request 2
request(url2, options, (error2, res2, body2) => {
if (error2) {
console.log(error2)
// reject
reject(error2);
};
if(!error2 && res2.statusCode == 200) {
// all good, continue
response.thumbnail = body2.data[0].imageUrl
console.log(response)
// make request 3
request(url3, options, (error3, res3, body3) => {
if (error3) {
console.log(error3)
// reject
reject(error3);
};
if(!error3 && res3.statusCode == 200) {
response.name = body3.Name
console.log(response)
// finally, resolve the final response object
resolve(response);
}
});
}
});
};
});
});
};
Now to the calling function. You still call GetData() but you use the then() function to get what was resolved, or catch() to catch any errors. I would also suggest putting your res.send() functions at the end, unless you absolutely need to return right away as your originally wrote it.
app.post('/exec', function(req, res) {
res.setHeader('Content-Type', 'application/json')
console.log("working");
console.log(req.body)
// get all the data as a Promise
GetData(req.body.id)
.then((data) => {
// now you have your data!
if (req.body.state === true){
var state = "New Projected"
} else {
var state = "No longer projected"
}
const msg = new Discord.RichEmbed()
.setDescription(data.name)
.setAuthor(state)
.setColor(0x1c90d9)
.addField("RAP:", data.rap)
.setImage(data.thumbnail)
bot.channels.get("655196831835226133").send(msg);
res.send(JSON.stringify({
success: true
}));
})
.catch((err) => {
// you can handle errors here that were rejected in the promise, if necessary
res.send(JSON.stringify({
success: false
}));
})
});
Related
I am trying to use node-http-proxy inside an AdonisJS controller, but I get the error
The "url" argument must be of type string. Received type function
The line causing the error is the proxy.web(request, response, { target: urlToProxy });
async proxy({ request, response }){
var resource = await Resource.query().where('uri', request.url()).with('service.user').with('userSecurity').first()
resource = resource.toJSON()
if(!resource.service.active){
return response.status(404).send(`Parent service '${resource.service.title}' is disabled`)
}
if(!resource.active){
return response.status(404).send(`Resource is disabled`)
}
if(resource.userSecurity.length <= 0) {
return response.status(403).send(`You are not permitted to access that resource. Contact ${resource.service.user.first_name} ${resource.service.user.last_name} (${resource.service.user.email})`)
}
var urlToProxy = url.resolve(resource.service.basepath, request.originalUrl())
var proxy = httpProxy.createProxyServer()
proxy.web(request, response, { target: urlToProxy });
}
In the end I got a bit closer but not a full fix. The getting close bit was to realise the http-proxy was delivering data via buffer so I had to do something like this
proxy.web(req, res, { target: data.service.basepath})
proxy.on('error', function(err, req, res){
console.log("There was an error", err)
})
proxy.on('proxyRes', async (proxyRes, request, response) =>{
var body = new Buffer('')
proxyRes.on('data', (data)=>{
body = Buffer.concat([body, data])
})
proxyRes.on('end', ()=>{
body = body.toString()
try{
res.send(body)
}catch(err){
}
})
});
However, I still could not get it to work as the controller was returning before http-proxy had completed the request.
In the end and probably for the best, I wrote a stand alone proxy app and used the main app just to validate JWT tokens before they go through the Proxy.
You were so close, I wanted to do something similar and wrapped the proxy in a promise so we can wait for the proxy to return before responding with our response:
const proxy = httpProxy.createProxyServer();
const prom = new Promise((resolve, reject) => {
proxy.web(request.request, response.response, {
target: urlToTarget
}, (e) => {
reject(e);
});
proxy.on('proxyRes', function (proxyRes, req, res) {
let body = [];
proxyRes.on('data', function (chunk) {
body.push(chunk);
});
proxyRes.on('end', function () {
body = Buffer.concat(body).toString();
resolve(body);
});
});
});
const result = await prom;
response.body(result);
return response;
I thought I'd give you a complete answer for anyone else that comes across this.
I'm working on an Actions on Google chatbot using Dialogflow and Cloud Functions. The runtime is Node.js 6.
Why does this function return the empty string?
function getJSON(url) {
var json = "";
var request = https.get(url, function(response) {
var body = "";
json = 'response';
response.on("data", function(chunk) {
body += chunk;
json = 'chunk';
});
response.on("end", function() {
if (response.statusCode == 200) {
try {
json = 'JSON.parse(body)';
return json;
} catch (error) {
json = 'Error1';
return json;
}
} else {
json = 'Error2';
return json;
}
});
});
return json;
}
This is the intent in which I want to access the json data:
app.intent('test', (conv) => {
conv.user.storage.test = 'no change';
const rp = require("request-promise-native");
var options = {
uri: 'https://teamtreehouse.com/joshtimonen.json',
headers: {
'User-Agent': 'Request-Promise'
},
json: true // Automatically parses the JSON string in the response
};
rp(options)
.then(function (user) {
conv.user.storage.test = user.name;
})
.catch(function (err) {
conv.user.storage.test = 'fail';
});
conv.ask(conv.user.storage.test);
});
You can try to use the request module for node.js, I tried my self reproducing your use case and worked fine. The code should be something similar to this:
const request = require('request');
request(url, {json: true}, (err, res, body) => {
if (err) { res.send(JSON.stringify({ 'fulfillmentText': "Some error"})); }
console.log(body);
});
Also, you need to add "request": "2.81.0" in the dependencies section inside your package.json file.
The function returns the empty string because https sets up a callback function, but the program flow continues to the return statement before the callback is called.
In general, when working with Dialogflow Intent Handlers, you should be returning a Promise instead of using callbacks or events. Look into using request-promise-native instead.
Two clarifying points:
You must return the Promise. Otherwise Dialogflow will assume the Handler has completed. If you return the Promise, it will wait for the Promise to finish.
Everything you want to send back must be done inside the then() block. This includes setting any response. The then() block runs after the asynchronous operation (the web API call) completes. So this will have the results of the call, and you can return these results in your call to conv.ask().
So it might look something like this:
return rp(options)
.then(function (user) {
conv.add('your name is '+user.name);
})
.catch(function (err) {
conv.add('something went wrong '+err);
});
You can use Axios as well
To install Axios : npm install axios
OR
You can add it in package.json as a dependency
"dependencies": {
"axios": "^0.27.2",
}
index.js
const axios = require('axios').default;
exports.makeRequest = async (req, res) => {
axios.get('https://jsonplaceholder.typicode.com/todos/1')// Dummy URL
.then(function (response) {
// handle success
res.status(200).json({
success:true,
result:response.data
})
})
.catch(function (error) {
// handle error
console.log(error);
})
};
OR
const Axios = require("axios");
exports.makeRequest = async (req, res) => {
const { data } = await
Axios.get('https://jsonplaceholder.typicode.com/todos/1')
res.status(200).send({data})
};
sorry for the generic title. I'm pretty new to nodejs as well as the idea of async/await.
So I have an express app, which makes an HTTP get request as a callback function. The callback function gets the body object, and returns it to getBody function. But when I try to assign getBody to a variable, it returns undefined.
Yes I know. The getBody function returns body before body gets filled up with data, but I just don't know how to write a getter method for this body object. So my question is, how can I run the get request and access body object in the global scope at the same time, so all functions depending on body object can run without further problems.
async function getBody (req, res, next) {
let body = await makeRequest(req, res);
return body; // THIS RETURNS UNDEFINED
}
router.get('/', getBody);
function makeRequest (req, res){
let uri;
let options = {
uri: uri,
};
request(options, function (error, response, body) {
if (error){
console.log('error:', error);
} else {
console.log('Success! statusCode:', response && response.statusCode);
let jsonObject = JSON.parse(body);
return jsonObject;
}
});
}
I did my research, but I just could not find a useful resource. Thanks in advance.
await and async should be used with a promise, these kind of method cannot return data. return is used to return a value from a synchronous method.
So you may return a promise from your makeRequest method like this,
async function getBody(req, res, next) {
let body = await makeRequest(req, res);
return body; // NOW BODY IS NOT UNDEFINED, call next() or send response here
}
router.get('/', getBody);
function makeRequest(req, res) {
return new Promise((resolve, reject) => {
let uri;
let options = {
uri: uri,
};
request(options, function (error, response, body) {
if (error) {
console.log('error:', error);
return reject(error);
} else {
console.log('Success! statusCode:', response && response.statusCode);
let jsonObject = JSON.parse(body);
return resolve(jsonObject);
}
});
})
}
FYI,
let body = await makeRequest(req, next)
is equals to
makeRequest(req, next).then(body => { /* YOUR CODE HERE */ })
and if you didn't knew, you have to process the body and send the response, return body won't send the response to the client.
OK, #JanithKasun did a great job of answering your original question fully. This answer is intended to expand on that a little to get a the problem you're having conceptually that isn't explicitly asked in your question.
As I understand your code, you're trying to fetch some data from a third party resource based on information in the request coming to your app's handler. Ideally, you want to separate your code in a way to make it more reusable/maintainable. I note that the code in question doesn't actually use the request or response object at all, but I'm going to assume you would have some kind of parameter to the getBody function that helps you construct the URI it's requesting. So, to that end:
// ./lib/get-body.js
const BASE_URI = 'https://example.com/search?q='
async function getBody (query) {
let body = await makeRequest(query);
return body;
}
function makeRequest(query) {
return new Promise((resolve, reject) => {
let uri = `${BASE_URI}{query}`; // results in something like 'https://example.com/search?q=cats'
let options = {
uri: uri,
};
// Note: removed console statements here to centralize error handling
request(options, function (error, response, body) {
if (error) {
return reject(error);
} else {
let jsonObject = JSON.parse(body);
return resolve(jsonObject);
}
});
})
}
// export the top level function for reuse
module.exports = getBody;
Now, in your routing code:
// ./index.js or wherever
const express = require('express');
const getBody = require('./lib/get-body');
//...whatever other setup...
app.get('/', async (req, res, next) => {
const query = req.query.terms; // or whatever
try {
const body = await getBody(query);
return res.send(body);
} catch (e) {
return next(e); // if you don't do this, process hangs forever on error
}
});
// handle the errors. obviously you can do something smart like
// figure out the error code and send back something other than a 500 if appropriate.
app.use((err, req, res, next) => {
console.error(err);
res.status(500).send('I am Bender; please insert girder.');
});
Hope that helps!
I'm working on a server side (self) project with node js (for the first time), and i ran into some difficulties.
My goal is the following:
first part - Im using "/uploads/processData" URL in my server to get URL(s) from the user request.
Now i want to access the user request URL(s) and get their HTML(s) file(s), to do so i'm using the "request" npm package (code below).
second part - I want access the body that I get back from the request package (from the first part), so I'm using cheerio npm package to do so.
Now to my problem - lets say that i'm trying to get the body of the url:
https://www.amazon.com/NIKE-Mens-Lunarconverge-Running-Shoes/dp/B06VVFGZHL?pd_rd_wg=6humg&pd_rd_r=61904ea4-c78e-43b6-8b8d-6b5ee8417541&pd_rd_w=Tue7n&ref_=pd_gw_simh&pf_rd_r=VGMA24803GJEV6DY7458&pf_rd_p=a670abbe-a1ba-52d3-b360-3badcefeb448&th=1
From some reason that i cant understand (probably because of lack of knowledge at web development), I dont always get the same body that i see when I review the above page (link) using F12, with my first part code. Hence sometimes my cheerio extraction (the second part) works as i expect and sometime does not (because some element from the full/original HTML file are missing). At first I thought it might be cache thing, so I added a middleware to set "nocache" flag.
What am I missing here? Does the way I try to operate wrong? Is there any way to ensure i get the same full/original HTML everytime?
Here is my code so far -
nocache middleware
function nocache(req, res, next) {
res.header("Cache-Control", "private, no-cache, no-store, must-revalidate");
res.header("Expires", "-1");
res.header("Pragma", "no-cache");
next();
}
EDIT
uploadRoutes.post("/processGoogleSearchData", nocache, (req, res) => {
//Assuming getting in req.body the google result JSON as "googleSearchResult"
var itemsArr = [];
var linksArr = [];
var bodysArr = [];
itemsArr = req.body.googleSearchResult.items;
if (itemsArr.length === 0) {
//return appropriate message
return res.status(400).send({ message: "No data sent to server" });
}
var linksArr = itemsArr.map(item => item.link);
//Get the needed info from every link
linksArr.forEach(link => {
request(link, (err, response, body) => {
if (!err && response.statusCode === 200) {
var $ = cheerio.load(body);
var tr = $(".a-lineitem").children();
var priceTd = tr.find(".a-span12");
var priceSpan = priceTd.find("#priceblock_ourprice");
console.log(priceSpan.text());
//when trying to build array of bodys the extraction doesnt work at all
bodysArr.push(body);
}
});
});
res.send(bodysArr);
});
I changed my code to the above, and it seems like the data extraction works more often. Can anyone explain why the extraction still sometimes doesnt work?
I tried return bodysArr for debbug purposes but when i do that the extraction does not work at all and my path response is always an empty array, why is that?
The problem is that:
res.send(bodysArr);
is executed straight after the call to
linksArr.forEach(link => {
The callbacks
(err, response, body) => {
if (!err && response.statusCode === 200) {
var $ = cheerio.load(body);
var tr = $(".a-lineitem").children();
var priceTd = tr.find(".a-span12");
var priceSpan = priceTd.find("#priceblock_ourprice");
console.log(priceSpan.text());
//when trying to build array of bodys the extraction doesnt work at all
bodysArr.push(body);
}
won't be guaranteed to have fired yet. What you want is ensure that res.send(bodysArr) runs after all the requests have happened
There are a few ways to handle this, one is with the excellent async library.
Hopefully you can get the gist of it with this example.
var array = [1,2,3]
function asyncRequest(input, callback){
//Do your fetch request here and call callback when done
setTimeout(callback, 10); //using setTiemout as an example
}
async.each(array, asyncRequest, (err) => {
if(err){
throw err;
}
console.log("All Finished");
});
<script src="https://cdnjs.cloudflare.com/ajax/libs/async/2.6.1/async.min.js"></script>
After reviewing Sudsy explanation, I came across loops of asynchronous methods.
While playing with this subject I could not figure out whats wrong with my following code:
This works fine - so i ended up using it
async function getItemsInfo(itemsArr) {
return itemsArr.map(async item => {
try {
var body = await axios(item.link);
var $ = await cheerio.load(body.data);
var tr = await $(".a-lineitem").children();
var priceTd = await tr.find(".a-span12");
var priceSpan = await priceTd.find("#priceblock_ourprice");
return priceSpan.text();
} catch (err) {
return err.message;
}
});
}
getItemsInfo(linksArr)
.then(res => Promise.all(res))
.then(res => console.log(res))
.catch(err => console.error(err));
Can someone explain to me what's wrong with the following codes?
async function getItemsInfo(itemsArr) {
await Promise.all(
itemsArr.map(async item => {
try {
var body = await axios(item.link);
var $ = await cheerio.load(body.data);
var tr = await $(".a-lineitem").children();
var priceTd = await tr.find(".a-span12");
var priceSpan = await priceTd.find("#priceblock_ourprice");
return priceSpan.text();
} catch (err) {
throw err.message;
}
})
)
.then(resulst => {
return results;
})
.catch(err => {
throw err.message;
});
}
//the caller function
try {
getItemsInfo(linksArr).then(results => {
res.status(200).send(results);
});
} catch (err) {
res.status(400).send(err.message);
}
or
async function getItemsInfo(itemsArr) {
const promises = itemsArr.map(async item => {
try {
var body = await axios(item.link);
var $ = await cheerio.load(body.data);
var tr = await $(".a-lineitem").children();
var priceTd = await tr.find(".a-span12");
var priceSpan = await priceTd.find("#priceblock_ourprice");
return priceSpan.text();
} catch (err) {
return err.message;
}
});
var results = await Promise.all(promises)
.then(results => {
return results;
})
.catch(err => {
return err.message;
});
}
//the caller function
try {
getItemsInfo(linksArr).then(results => {
res.status(200).send(results);
});
} catch (err) {
res.status(400).send(err.message);
}
I am trying currently learning to build crawler using node + express +cheerio.
In the route I put this:
[index.js]
app.get('/api/crawler/android', crawlerController.android);
which calls into controller
[crawler-controller.js]
var androidCrawler = require('../crawlers/android')
module.exports.android = androidCrawler.androidget;
then I invoke the crawler (based on cheerio)
[crawler.js]
var request = require('request');
var cheerio = require('cheerio');
var androidget =request('https://www.developer-tech.com/categories/Android/', function (error, response, html){
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html);
var result = {result:[]};
$('article').each(function (i, element) {
var Title = $(this).find("h2").text();
var Link = $(this).find("a").attr("href");
var Image = $(this).find("img").attr("src");
var payload = {
"Title":Title,
"Link":Link,
"Image":Image
};
result['result'].push(payload);
});
console.log("aaa", result);
console.log(typeof result);
return result;
}});
module.exports = {
getAndroid: function (androidget, res) {
res.send(JSON.stringify(result));
}
}
When I console log directly to crawler.js via terminal it return JSON object properly, but I think the way I export the function to be invoked by app.get is where I'm wrong and I can't figure it out.
Perhaps somebody could help me to properly invoke the crawler in my case?
There is no point of returning a result in a callback function, this will just do nothing.
What you can do is wrap your request in a function and call a callback that you create :
// file.js
const wrapFunction = (url, callback) => {
request(url, ((error, response, html) => {
// ...
callback(result);
})
}
and then use it :
// just an example
wrapFunction(yourUrl, (result) => {
// deal with your result
})
When you have that, you can export it and then use it in your middleware / controller :
// file.js
module.exports = wrapFunction;
// index.js
const wrapFunction = require('file.js'); // here is your function
app.get('/yourRoute', (req, res) => {
wrapFunction(yourUrl, (result) => {
res.send(JSON.stringify(result));
});
})
You can also use Promises :
const wrapFunction = (url) => {
return new Promise((resolve, reject) => {
request(url, ((error, response, html) => {
if (error) reject(error);
resolve(result);
});
});
};
And then :
wrapFunction(yourUrl).then(result => {
// deal with your result ...
}).catch(error => {
// deal with your error ...
});
Hope it helps,
Best regards