I am trying to make a URL shortener. I need to take a given URL as a parameter and send a request to that URL just to get the status code. If status = 200, I know I've got a functioning URL, and I'll go ahead and add it to the DB and shorten it.
Problem is, when I make that request, the connection times out.
const express = require('express')
const mongoose = require('mongoose')
const cors = require('cors')
const nofavicon = require('express-no-favicons')
const Shortener = require('./shortener')
const app = express()
app.disable('x-powered-by')
app.use(cors())
app.use(nofavicon())
app.use(express.static(__dirname + '/static'))
mongoose.connect(
process.env.MONGODB_URI || 'mongodb://heroku_x7hcc5zd:39c8i70697o7qrpjn4rd6kslch#ds123371.mlab.com:23371/heroku_x7hcc5zd'
)
app.get('/url/:urlParam(*)', (request, response) => {
let urlParam = request.params.urlParam
let urlRegEx = /[A-Za-z]+[://]+[A-Za-z0-9-_]+\.[A-Za-z0-9-_:%&;?#/.=]+/g
if (urlRegEx.test(urlParam)) {
let shortRandomNum = Math.floor(Math.random() * 10000).toString()
// Shortener here refers to a mongoose Schema in external file
let lmao = new Shortener({
url: urlParam,
urlmao: 'localhost:8080/lol/' + shortRandomNum,
})
// Request header from passed URL to verify legitimacy
// Check statusCode and end request.
app.head(urlParam, (req, res) => {
let end = res.end
// Override standard res.end function with custom function
res.end = () => {
if (res.statusCode == 200) {
lmao.save((error) => {
if (error) {
response.send('Unable to write to collection')
}
})
console.log('pass')
response.json({lmao})
}
}
res.end = end
res.end()
})
} else {
// If passed URL does not satisfy regEx, return error message.
urlParam = 'unfunny url. http(s):// prefix required. check url and retry.'
console.log('invalid url')
response.json({
url: urlParam,
})
}
})
app.listen(process.env.PORT || 8080, () => {
console.log('live connection')
})
Most baffingly, the code shown here worked on Friday. Tested it last night, no dice. Any insight would be greatly, greatly appreciated.
app.head(urlParam, [Function]) doesn't make a request to the url, it defines a new route on your application so that it responds to HEADrequests on that url.
To check if the URL is alive you need to use another package to make requests. One of my favourites is Request. To use it simply replace app.head with request and add the require('request') to the top of your file.
Related
I am following course for learning node. In process of API creation I have got strange error that doesn't break my code but yet I am not able to get the required output. Please check the codebelow. I am using Express V4 as the trainer recommended to use the same version as he was using. Now my codeis exact same but outputis not same. I am using Postman for this purpose, and the image of Postman is also attached.
The actual output was supposed to postthe data into my tours api but its adding "id": "[object Object]1" Please someone explain me why is that error and how to resolve it. Thanks in advance.
const express = require('express');
const fs = require('fs');
const app = express();
const port = 3000;
app.use(express.json());
const tours = JSON.parse(fs.readFileSync(`${__dirname}/dev-data/data/tours-simple.json`));
app.get('/api/v1/tours', (req, res) => {
res.status(200).json({
stauts: 'success',
results: tours.length,
data: {
tours // this is same as tours: tours
}
})
});
app.post('/api/v1/tours', (req, res) => {
// console.log(req.body);
const newId = tours[tours.length - 1].id + 1;
const newTour = Object.assign({id: newId}, req.body);
tours.push(newTour);
fs.writeFile(`${__dirname}/dev-data/data/tours-simple.json`, JSON.stringify(tours), err => {
res.status(201).json({
status: 'success',
data: {
tour: newTour
}
});
})
// res.send('Done');
})
app.listen(port, () => {
console.log(`Server is listening at port no ${port}`);
});
Postman
tours-simple.json file data
I tried simple post to my api but my data wasn't posted instead "id": "[object Object]1" was added.
In Postman change the request body format from body -> raw -> Javascript to body -> raw -> JSON, this will allow your server to parse the request data.
For the id, I'm not really sure what you are trying to do, tours[tours.length-1] is an object, if you would like to get the id of the last element in tours and increment it by 1 you should change it to this:
const newId = tours[tours.length - 1].id + 1;
I'm trying to create an app in Node.JS that receives POST requests. At the moment, I have them tunneled through ngrok, and I can get the headers without a problem, but I can't get the request body. Here's my code:
const http = require("http");
const crypto = require("crypto");
const qs = require("qs");
const bodyParser = require("body-parser");
const slackSigningSecret = process.env.SLACK_SIGNING_SECRET;
http.createServer(function(request) {
let x_slack_signature = request.headers["x-slack-signature"]; // Get the Slack signature
let raw_body = request.body; // Get the request body
let x_slack_timestamp = request.headers["x-slack-request-timestamp"]; // Get the timestamp of the request
(...)
}).listen(3000);
But when I make a console log with the variables I defined, raw_body returns either empty or undefined.
I have tried to qs.stringify since I was reading that seemed to work. Also with body parser; I get this error body-parser deprecated undefined extended: provide extended option. If I write the extended option then it outputs this:
function urlencodedParser (req, res, next) {
if (req._body) {
debug('body already parsed')
next()
return
}
req.body = req.body || {}
// skip requests without bodies
if (!typeis.hasBody(req)) {
debug('skip empty body')
next()
return
}
debug('content-type %j', req.headers['content-type'])
// determine if request should be parsed
if (!shouldParse(req)) {
debug('skip parsing')
next()
return
}
// assert charset
var charset = getCharset(req) || 'utf-8'
if (charset !== 'utf-8') {
debug('invalid charset')
next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', {
charset: charset,
type: 'charset.unsupported'
}))
return
}
// read
read(req, res, next, parse, debug, {
debug: debug,
encoding: charset,
inflate: inflate,
limit: limit,
verify: verify
})
}
What am I doing wrong?
I am trying to get the full body first, then to divide it into something like an array of some sorts
Thanks in advance
I have an angular app and a nodejs backend server. I want to get data from my backend but when I try to connect to it with Angular HTTPClient, it says: POST http://localhost:3000/login/aa/aa 404 (Not Found).However, when I put the link manually into the browser, it works perfectly fine. Here is some code:
service.ts
addUser(user: IUser): Observable<IUser> {
return this.httpClient.post<IUser>(`http://localhost:3000/login/${user.email}/${user.passwort}`, user, {
headers: new HttpHeaders({
'Content-Type': 'application/json'
})
})
.pipe(catchError(this.handleError));
}
index.js
var mysql = require('mysql');
var express = require('express');
var app = express();
const port = process.env.PORT || 3000;
[...]
app.get('/login/:email/:pw',function(req,res) {
res.setHeader('Content-Type', 'application/json');
var passwort = new Passwort(''+req.params.pw);
passwort.comparePasswort();
con.query("SELECT u.Email, u.Hash FROM User u WHERE u.Email LIKE "+ "'" + req.params.email+ "'", function(err, result ){
if(err) throw err;
console.log(result)
res.send("test")
})
});
Thanks for every answer and for your time!
Your route in your backend is set as a get request and not a post request.
You should either convert your request to a get in your service with this.httpClient.get... or convert to a post request in your backend with app.post.
The reason it works in your browser is that the browser performs a GET request when acessing something using the address bar.
In backed you declared a get method and from frontend you are calling post. your code in service should be :-
addUser(user: IUser): Observable<IUser> {
return this.httpClient.get<IUser>(`http://localhost:3000/login/${user.email}/${user.passwort}`, {
headers: new HttpHeaders({
'Content-Type': 'application/json'
})
})
.pipe(catchError(this.handleError));
}
before using /:email you need to subscribe this particular element
const mongoose = require("mongoose");
const User = mongoose.model("User");
const userParams = (req, res, next, email) => {
User.findOne({email:email})
.then((user)=> {
if (!user) {
return res.sendStatus(404);
}
req.user = user;
return next();
})
.catch(next);
};
module.exports = userParams;
then use that in express router by typing
router.param("email", userParams);
this way your router will get to know what the params you are trying to send
In your index.js file, you are creating a handler for a GET request (which is the default request sent by your browser while accessing your webpage)
But in your service.ts file you are trying to send a post request to the server which is not handled, so the simple solution would be to replace the line
return this.httpClient.post<IUser> `http://localhost:3000/login/${user.email}/${user.passwort}`, user, {
with:
return this.httpClient.get<IUser> `http://localhost:3000/login/${user.email}/${user.passwort}`, user, {
For more info you can read this: https://angular.io/guide/http
I have written a programmable SMS feature using Twilio in nodejs. I have a message that has options to select and when user sends back any response I want to send an automated response using twilio.
I have completed all except after processing the response from user my automated response is not being delivered to user.
I keep getting above thing from my twilio dashboard.
Here is my response handler code..
app.post('/ui/sms',function(req, res) {
//req.headers['Content-type'] = 'text/xml';
//req.headers['Accept'] = 'text/xml';
try {
console.log('Processing Response', req.headers);
const MessagingResponse = require('twilio').twiml.MessagingResponse;
const twiml = new MessagingResponse();
const fromTwilio = isFromTwilio(req);
console.log('isFromTwilio: ', fromTwilio);
if (fromTwilio) {
let msg = req.body.Body||'';
if (msg.indexOf('1')>-1) {
twiml.message('Thanks for confirming your appointment.');
} else if (msg.indexOf('2')>-1) {
twiml.message('Please call 408-xxx-xxxx to reschedule.');
} else if (msg.indexOf('3')>-1) {
twiml.message('We will call you to follow up.');
} else {
twiml.message(
'Unknown option, please call 408-xxx-xxxx to talk with us.'
);
}
res.writeHead(200, { 'Content-Type': 'text/xml' });
res.end(twiml.toString());
}
else {
// we don't expect these
res.status(500).json({ error: 'Cannot process your request.' });
}
/*processSMSResponse(req, function(response) {
res.json(response);
});*/
} catch(e) {
res.json(e);
}
});
function isFromTwilio(req) {
console.log('REQ HEADER:::::::::\n', req);
// Get twilio-node from twilio.com/docs/libraries/node
const client = require('twilio');
// Your Auth Token from twilio.com/console
const authToken = 'xxxxxxxxxxxxxxxxx';
// The Twilio request URL
//const url = 'https://mycompany.com/myapp.php?foo=1&bar=2';
const url = 'https://xxxx.com/ui/sms';
var reqUrl = 'https://xxxx.com/ui/sms'
// The post variables in Twilio's request
//const params = {
//CallSid: 'CA1234567890ABCDE',
//Caller: '+14158675310',
//Digits: '1234',
//From: '+14158675310',
//To: '+18005551212',
//};
const params = req.body;
console.log('post params: ', params);
// The X-Twilio-Signature header attached to the request
try{
Object.keys(params).sort().forEach(function(key) {
reqUrl = reqUrl + key + params[key];
});
var twilioSignature = crypto.createHmac('sha1', authToken).update(Buffer.from(reqUrl, 'utf-8')).digest('base64');
//const twilioSignature = req.header('HTTP_X_TWILIO_SIGNATURE');
console.log('twilioSignature: ', twilioSignature);
} catch(e){
console.log(e);
}
return client.validateRequest(
authToken,
twilioSignature,
url,
params
);
}
I have explicitly tried setting headers but no use. I'm clue less on what twilio expects from me or how to modify headers.
{
"status": "Error",
"error": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Response><Message>Please call 408-xxx-xxxx to reschedule.</Message></Response>"
}
I see this as body in Twilio console and it has the response that I need but could not send as a message..
Twilio Developer Evangelist here.
From looking at your code and trying it myself I can't see anything generally wrong with it. There are two potential failing points here though:
1) I can't see your isFromTwilio function. If that one fails it might cause an error and then return JSON instead of XML on your error handler. I don't know why it would reply with the TwiML in that JSON though.
2) The other behavior I could reproduce (except that the TwiML is not being sent in the response body) is when I don't include body-parser in the middleware chain. This will cause req.body to be undefined and therefore req.body.Body will throw an error that is then being caught and JSON is being returned.
Do you have body-parser included and properly included as a middleware? You can either do it this way:
const { urlencoded } = require('body-parser');
app.use(urlencoded({ extended: false });
or if you only want to use it for this one endpoint you can add urlencoded({ extended: false }) as an argument before your request handler:
app.post('/ui/sms', urlencoded({ extended: false }), function(req, res) {
I hope this helps.
Cheers,
Dominik
I am trying to verify the hmac code sent from a shopify webhook on a dev environment. However shopify will not send a post request for a webhook to a non live endpoint, so I am using requestbin to capture the request and then use postman to send it to my local webserver.
From shopify documentation, I seem to be doing everything right and have also tried applying the method used in node-shopify-auth verifyWebhookHMAC function. But none of this has worked so far.
The codes are never a match.
What am I doing wrong here?
My code to verify the webhook:
function verifyWebHook(req, res, next) {
var message = JSON.stringify(req.body);
//Shopify seems to be escaping forward slashes when the build the HMAC
// so we need to do the same otherwise it will fail validation
// Shopify also seems to replace '&' with \u0026 ...
//message = message.replace('/', '\\/');
message = message.split('/').join('\\/');
message = message.split('&').join('\\u0026');
var signature = crypto.createHmac('sha256', shopifyConfig.secret).update(message).digest('base64');
var reqHeaderHmac = req.headers['x-shopify-hmac-sha256'];
var truthCondition = signature === reqHeaderHmac;
winston.info('sha256 signature: ' + signature);
winston.info('x-shopify-hmac-sha256 from header: ' + reqHeaderHmac);
winston.info(req.body);
if (truthCondition) {
winston.info('webhook verified');
req.body = JSON.parse(req.body.toString());
res.sendStatus(200);
res.end();
next();
} else {
winston.info('Failed to verify web-hook');
res.writeHead(401);
res.end('Unverified webhook');
}
}
My route which receives the request:
router.post('/update-product', useBodyParserJson, verifyWebHook, function (req, res) {
var shopName = req.headers['x-shopify-shop-domain'].slice(0, -14);
var itemId = req.headers['x-shopify-product-id'];
winston.info('Shopname from webhook is: ' + shopName + ' For item: ' + itemId);
});
I do it a little differently -- Not sure where I saw the recommendation but I do the verify in the body parser. IIRC one reason being that I get access to the raw body before any other handlers are likely to have touched it:
app.use( bodyParser.json({verify: function(req, res, buf, encoding) {
var shopHMAC = req.get('x-shopify-hmac-sha256');
if(!shopHMAC) return;
if(req.get('x-kotn-webhook-verified')) throw "Unexpected webhook verified header";
var sharedSecret = process.env.API_SECRET;
var digest = crypto.createHmac('SHA256', sharedSecret).update(buf).digest('base64');
if(digest == req.get('x-shopify-hmac-sha256')){
req.headers['x-kotn-webhook-verified']= '200';
}
}}));
and then any web hooks just deal with the verified header:
if('200' != req.get('x-kotn-webhook-verified')){
console.log('invalid signature for uninstall');
res.status(204).send();
return;
}
var shop = req.get('x-shopify-shop-domain');
if(!shop){
console.log('missing shop header for uninstall');
res.status(400).send('missing shop');
return;
}
Short Answer
The body parser in express does not handle BigInt well, and things like order number which are passed as integer get corrupted. Apart from that certain values are edited such as URLs are originally sent as "https://...", which OP also found out from the other code.
To solve this, do not parse the data using body parser and instead get it as raw string, later on you can parse it with json-bigint to ensure none of it has been corrupted.
Long Answer
Although the answer by #bknights works perfectly fine, it's important to find out why this was happening in the first place.
For a webhook I made on the "order_created" event from Shopify I found out that the id of the request being passed to the body was different than what I was sending from my test data, this turned out to be an issue with body-parser in express which did not play nice with big integers.
Ultimately I was deploying something to Google cloud functions and the req already had raw body which I could use, but in my test environment in Node I implemented the following as a separate body parser as using the same body parser twice overwrote the raw body with JSON
var rawBodySaver = function (req, res, buf, encoding) {
if (buf && buf.length) {
req.rawBody = buf.toString(encoding || 'utf8');
}
}
app.use(bodyParser.json({verify: rawBodySaver, extended: true}));
Based on this answer
I later on parse the rawBody using json-bigint for use in code elsewhere as otherwise some of the numbers were corrupted.
// Change the way body-parser is used
const bodyParser = require('body-parser');
var rawBodySaver = function (req, res, buf, encoding) {
if (buf && buf.length) {
req.rawBody = buf.toString(encoding || 'utf8');
}
}
app.use(bodyParser.json({ verify: rawBodySaver, extended: true }));
// Now we can access raw-body any where in out application as follows
// request.rawBody in routes;
// verify webhook middleware
const verifyWebhook = function (req, res, next) {
console.log('Hey!!! we got a webhook to verify!');
const hmac_header = req.get('X-Shopify-Hmac-Sha256');
const body = req.rawBody;
const calculated_hmac = crypto.createHmac('SHA256', secretKey)
.update(body,'utf8', 'hex')
.digest('base64');
console.log('calculated_hmac', calculated_hmac);
console.log('hmac_header', hmac_header);
if (calculated_hmac == hmac_header) {
console.log('Phew, it came from Shopify!');
res.status(200).send('ok');
next();
}else {
console.log('Danger! Not from Shopify!')
res.status(403).send('invalid');
}
}
Had the same issue. Using request.rawBody instead of request.body helped:
import Router from "koa-router";
import koaBodyParser from "koa-bodyparser";
import crypto from "crypto";
...
koaServer.use(koaBodyParser());
...
koaRouter.post(
"/webhooks/<yourwebhook>",
verifyShopifyWebhooks,
async (ctx) => {
try {
ctx.res.statusCode = 200;
} catch (error) {
console.log(`Failed to process webhook: ${error}`);
}
}
);
...
async function verifyShopifyWebhooks(ctx, next) {
const generateHash = crypto
.createHmac("sha256", process.env.SHOPIFY_WEBHOOKS_KEY) // that's not your Shopify API secret key, but the key under Webhooks section in your admin panel (<yourstore>.myshopify.com/admin/settings/notifications) where it says "All your webhooks will be signed with [SHOPIFY_WEBHOOKS_KEY] so you can verify their integrity
.update(ctx.request.rawBody, "utf-8")
.digest("base64");
if (generateHash !== shopifyHmac) {
ctx.throw(401, "Couldn't verify Shopify webhook HMAC");
} else {
console.log("Successfully verified Shopify webhook HMAC");
}
await next();
}