I'm trying to create an app in Node.JS that receives POST requests. At the moment, I have them tunneled through ngrok, and I can get the headers without a problem, but I can't get the request body. Here's my code:
const http = require("http");
const crypto = require("crypto");
const qs = require("qs");
const bodyParser = require("body-parser");
const slackSigningSecret = process.env.SLACK_SIGNING_SECRET;
http.createServer(function(request) {
let x_slack_signature = request.headers["x-slack-signature"]; // Get the Slack signature
let raw_body = request.body; // Get the request body
let x_slack_timestamp = request.headers["x-slack-request-timestamp"]; // Get the timestamp of the request
(...)
}).listen(3000);
But when I make a console log with the variables I defined, raw_body returns either empty or undefined.
I have tried to qs.stringify since I was reading that seemed to work. Also with body parser; I get this error body-parser deprecated undefined extended: provide extended option. If I write the extended option then it outputs this:
function urlencodedParser (req, res, next) {
if (req._body) {
debug('body already parsed')
next()
return
}
req.body = req.body || {}
// skip requests without bodies
if (!typeis.hasBody(req)) {
debug('skip empty body')
next()
return
}
debug('content-type %j', req.headers['content-type'])
// determine if request should be parsed
if (!shouldParse(req)) {
debug('skip parsing')
next()
return
}
// assert charset
var charset = getCharset(req) || 'utf-8'
if (charset !== 'utf-8') {
debug('invalid charset')
next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', {
charset: charset,
type: 'charset.unsupported'
}))
return
}
// read
read(req, res, next, parse, debug, {
debug: debug,
encoding: charset,
inflate: inflate,
limit: limit,
verify: verify
})
}
What am I doing wrong?
I am trying to get the full body first, then to divide it into something like an array of some sorts
Thanks in advance
Related
I'm sending a POST request to a HTTP Firebase Cloud Function endpoint which contains the following body:
{
"securityContext": "edbsne0f17660e9ab49ad9bc3ddd4656230e0a9304bd15916f595b793da9109e1f7aa61e76c4afa91319011a7259b24ed583302334e22c5544c4e00506cf2fed93b28434e4088b22dda02712adf63bb6a370f",
"event": "onInstall",
"orgId": "7001935574",
"dc": "AU"
}
But when I try to access any of those properties it shows undefined. The entire body is also undefined.
This is what my onRequest HTTP Cloud Function endpoint looks like. It also shows my other failed attempts of getting the body data, which I have commented out:
export const getZohoDeskCallBack = functions.https.onRequest((req, res) => {
const body = req.body;
functions.logger.info('body', body);
const rawBody = req.body;
functions.logger.info('rawBody', rawBody);
// Other attempt 1:
// const bodySecurityContext = req.body.securityContext;
// functions.logger.info('bodySecurityContext', bodySecurityContext);
// Other attempt 2:
// const rawBodySecurityContext = req.rawBody.securityContext;
// functions.logger.info('rawBodySecurityContext', rawBodySecurityContext);
// Other attempt 3:
// const reqBodyToJSON = req.body.toJSON();
// functions.logger.info('req.body.toJSON()', reqBodyToJSON);
// Other attempt 4:
// const reqRawBodyToJSON = req.rawBody.toJSON();
// functions.logger.info('req.rawBody.toJSON()', reqRawBodyToJSON);
// Other attempt 5:
// const reqBodyToJSONparse = JSON.parse(req.body);
// functions.logger.info('reqBodyToJSONparse', reqBodyToJSONparse);
// Other attempt 6:
// const reqRawBodyToJSONparse = JSON.parse(req.rawBody);
// functions.logger.info('reqRawBodyToJSONparse', reqRawBodyToJSONparse);
// Other attempt 7:
// const reqBodyToJSONparse = req.body.toString();
// functions.logger.info('reqBodyToJSONparse', reqBodyToJSONparse);
// Other attempt 8:
// const reqRawBodyToJSONparse = req.rawBody.toString();
// functions.logger.info('reqRawBodyToJSONparse', reqRawBodyToJSONparse);
// Other attempt 9:
// const reqBodyToJSONparse = req.body.toString();
// const securityContext = reqBodyToJSONparse.securityContext;
// functions.logger.info('securityContext', securityContext);
res.end();
});
You can see a test of the POST request here and here.
Apparently, if a request has a content-type of application/json Firebase Cloud Functions will automatically parse the JSON and put it into the body property.
But as you can see from those tests linked above the header content-type is empty or missing. Also I am unable to change the POST request because I have no control over that.
Maybe that could be the issue? If so, I thought I could access it from the rawBody property, but that also doesn't work.
I have been pulling my hair out trying to solve this. Any help would be much appreciated.
Apparently, if a request has a content-type of application/json Firebase Cloud Functions will automatically parse the JSON and put it into the body property.
That's because you do need to add the Content-Type header and Firebase use the appropriate parser for that request. For more information on the libraries that Firebase uses, checkout the documentation.
You can use the body parser middleware and force parse the body as JSON as shown below:
import * as functions from "firebase-functions/v1";
import * as express from "express";
import * as bodyParser from "body-parser";
const app = express();
const parseJsonMiddleware = bodyParser.json({
type(req) {
return true;
},
});
app.post("/getZohoDeskCallBack", parseJsonMiddleware, (req, res) => {
const body = req.body;
console.log("body", body);
const rawBody = req.body;
console.log("rawBody", rawBody);
res.end();
});
export const api = functions.https.onRequest(app);
You'll have to update your webhook URL as this will add /api in the URL like this:
POST https://[FUNCTIONS_URL]/api/getZohoDeskCallBack
I have written a programmable SMS feature using Twilio in nodejs. I have a message that has options to select and when user sends back any response I want to send an automated response using twilio.
I have completed all except after processing the response from user my automated response is not being delivered to user.
I keep getting above thing from my twilio dashboard.
Here is my response handler code..
app.post('/ui/sms',function(req, res) {
//req.headers['Content-type'] = 'text/xml';
//req.headers['Accept'] = 'text/xml';
try {
console.log('Processing Response', req.headers);
const MessagingResponse = require('twilio').twiml.MessagingResponse;
const twiml = new MessagingResponse();
const fromTwilio = isFromTwilio(req);
console.log('isFromTwilio: ', fromTwilio);
if (fromTwilio) {
let msg = req.body.Body||'';
if (msg.indexOf('1')>-1) {
twiml.message('Thanks for confirming your appointment.');
} else if (msg.indexOf('2')>-1) {
twiml.message('Please call 408-xxx-xxxx to reschedule.');
} else if (msg.indexOf('3')>-1) {
twiml.message('We will call you to follow up.');
} else {
twiml.message(
'Unknown option, please call 408-xxx-xxxx to talk with us.'
);
}
res.writeHead(200, { 'Content-Type': 'text/xml' });
res.end(twiml.toString());
}
else {
// we don't expect these
res.status(500).json({ error: 'Cannot process your request.' });
}
/*processSMSResponse(req, function(response) {
res.json(response);
});*/
} catch(e) {
res.json(e);
}
});
function isFromTwilio(req) {
console.log('REQ HEADER:::::::::\n', req);
// Get twilio-node from twilio.com/docs/libraries/node
const client = require('twilio');
// Your Auth Token from twilio.com/console
const authToken = 'xxxxxxxxxxxxxxxxx';
// The Twilio request URL
//const url = 'https://mycompany.com/myapp.php?foo=1&bar=2';
const url = 'https://xxxx.com/ui/sms';
var reqUrl = 'https://xxxx.com/ui/sms'
// The post variables in Twilio's request
//const params = {
//CallSid: 'CA1234567890ABCDE',
//Caller: '+14158675310',
//Digits: '1234',
//From: '+14158675310',
//To: '+18005551212',
//};
const params = req.body;
console.log('post params: ', params);
// The X-Twilio-Signature header attached to the request
try{
Object.keys(params).sort().forEach(function(key) {
reqUrl = reqUrl + key + params[key];
});
var twilioSignature = crypto.createHmac('sha1', authToken).update(Buffer.from(reqUrl, 'utf-8')).digest('base64');
//const twilioSignature = req.header('HTTP_X_TWILIO_SIGNATURE');
console.log('twilioSignature: ', twilioSignature);
} catch(e){
console.log(e);
}
return client.validateRequest(
authToken,
twilioSignature,
url,
params
);
}
I have explicitly tried setting headers but no use. I'm clue less on what twilio expects from me or how to modify headers.
{
"status": "Error",
"error": "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Response><Message>Please call 408-xxx-xxxx to reschedule.</Message></Response>"
}
I see this as body in Twilio console and it has the response that I need but could not send as a message..
Twilio Developer Evangelist here.
From looking at your code and trying it myself I can't see anything generally wrong with it. There are two potential failing points here though:
1) I can't see your isFromTwilio function. If that one fails it might cause an error and then return JSON instead of XML on your error handler. I don't know why it would reply with the TwiML in that JSON though.
2) The other behavior I could reproduce (except that the TwiML is not being sent in the response body) is when I don't include body-parser in the middleware chain. This will cause req.body to be undefined and therefore req.body.Body will throw an error that is then being caught and JSON is being returned.
Do you have body-parser included and properly included as a middleware? You can either do it this way:
const { urlencoded } = require('body-parser');
app.use(urlencoded({ extended: false });
or if you only want to use it for this one endpoint you can add urlencoded({ extended: false }) as an argument before your request handler:
app.post('/ui/sms', urlencoded({ extended: false }), function(req, res) {
I hope this helps.
Cheers,
Dominik
I am trying to make a URL shortener. I need to take a given URL as a parameter and send a request to that URL just to get the status code. If status = 200, I know I've got a functioning URL, and I'll go ahead and add it to the DB and shorten it.
Problem is, when I make that request, the connection times out.
const express = require('express')
const mongoose = require('mongoose')
const cors = require('cors')
const nofavicon = require('express-no-favicons')
const Shortener = require('./shortener')
const app = express()
app.disable('x-powered-by')
app.use(cors())
app.use(nofavicon())
app.use(express.static(__dirname + '/static'))
mongoose.connect(
process.env.MONGODB_URI || 'mongodb://heroku_x7hcc5zd:39c8i70697o7qrpjn4rd6kslch#ds123371.mlab.com:23371/heroku_x7hcc5zd'
)
app.get('/url/:urlParam(*)', (request, response) => {
let urlParam = request.params.urlParam
let urlRegEx = /[A-Za-z]+[://]+[A-Za-z0-9-_]+\.[A-Za-z0-9-_:%&;?#/.=]+/g
if (urlRegEx.test(urlParam)) {
let shortRandomNum = Math.floor(Math.random() * 10000).toString()
// Shortener here refers to a mongoose Schema in external file
let lmao = new Shortener({
url: urlParam,
urlmao: 'localhost:8080/lol/' + shortRandomNum,
})
// Request header from passed URL to verify legitimacy
// Check statusCode and end request.
app.head(urlParam, (req, res) => {
let end = res.end
// Override standard res.end function with custom function
res.end = () => {
if (res.statusCode == 200) {
lmao.save((error) => {
if (error) {
response.send('Unable to write to collection')
}
})
console.log('pass')
response.json({lmao})
}
}
res.end = end
res.end()
})
} else {
// If passed URL does not satisfy regEx, return error message.
urlParam = 'unfunny url. http(s):// prefix required. check url and retry.'
console.log('invalid url')
response.json({
url: urlParam,
})
}
})
app.listen(process.env.PORT || 8080, () => {
console.log('live connection')
})
Most baffingly, the code shown here worked on Friday. Tested it last night, no dice. Any insight would be greatly, greatly appreciated.
app.head(urlParam, [Function]) doesn't make a request to the url, it defines a new route on your application so that it responds to HEADrequests on that url.
To check if the URL is alive you need to use another package to make requests. One of my favourites is Request. To use it simply replace app.head with request and add the require('request') to the top of your file.
How to accept request content-type of application/vnd.api+json and reject anything else?
Also how can I access the x-api-key value using Koa.js ?
Thanks in advance
To get the header: ctx.get('x-api-key');
This is my attempt to the first part of the question, content negotiation:
const Koa = require('koa');
const Router = require('koa-router');
const app = new Koa();
const router = new Router();
//const dataAPI = require('../models/traffic');
router.get('/locations/:geohash/traffic/last-hour', (ctx, next) => {
// some code for validating geohash goes here ...
if (ctx.request.type=='application/vnd.api+json') {
//ctx.body = dataAPI.getTrafficData(ctx.params.geohash, 'hours', 1);
ctx.body = { status: "success" };
ctx.type = "application/vnd.api+json";
next();
}
else {
ctx.throw(406, 'unsupported content-type');
// actual error will be in JSON API 1.0 format
}
});
I am getting status 406 Not Acceptable and unsupported content-type in Postman when I submit the value for Content-Type in Postman anything that is not application/vnd.api+json. Otherwise, I get station 200 OK and { "status": "success" in the body.
Edited
Haven't found a better to this but below is a quick and dirty way to extract the value of x-api-key. It works for my purpose:
var key = ctx.request.headers['x-api-key']
I am trying to verify the hmac code sent from a shopify webhook on a dev environment. However shopify will not send a post request for a webhook to a non live endpoint, so I am using requestbin to capture the request and then use postman to send it to my local webserver.
From shopify documentation, I seem to be doing everything right and have also tried applying the method used in node-shopify-auth verifyWebhookHMAC function. But none of this has worked so far.
The codes are never a match.
What am I doing wrong here?
My code to verify the webhook:
function verifyWebHook(req, res, next) {
var message = JSON.stringify(req.body);
//Shopify seems to be escaping forward slashes when the build the HMAC
// so we need to do the same otherwise it will fail validation
// Shopify also seems to replace '&' with \u0026 ...
//message = message.replace('/', '\\/');
message = message.split('/').join('\\/');
message = message.split('&').join('\\u0026');
var signature = crypto.createHmac('sha256', shopifyConfig.secret).update(message).digest('base64');
var reqHeaderHmac = req.headers['x-shopify-hmac-sha256'];
var truthCondition = signature === reqHeaderHmac;
winston.info('sha256 signature: ' + signature);
winston.info('x-shopify-hmac-sha256 from header: ' + reqHeaderHmac);
winston.info(req.body);
if (truthCondition) {
winston.info('webhook verified');
req.body = JSON.parse(req.body.toString());
res.sendStatus(200);
res.end();
next();
} else {
winston.info('Failed to verify web-hook');
res.writeHead(401);
res.end('Unverified webhook');
}
}
My route which receives the request:
router.post('/update-product', useBodyParserJson, verifyWebHook, function (req, res) {
var shopName = req.headers['x-shopify-shop-domain'].slice(0, -14);
var itemId = req.headers['x-shopify-product-id'];
winston.info('Shopname from webhook is: ' + shopName + ' For item: ' + itemId);
});
I do it a little differently -- Not sure where I saw the recommendation but I do the verify in the body parser. IIRC one reason being that I get access to the raw body before any other handlers are likely to have touched it:
app.use( bodyParser.json({verify: function(req, res, buf, encoding) {
var shopHMAC = req.get('x-shopify-hmac-sha256');
if(!shopHMAC) return;
if(req.get('x-kotn-webhook-verified')) throw "Unexpected webhook verified header";
var sharedSecret = process.env.API_SECRET;
var digest = crypto.createHmac('SHA256', sharedSecret).update(buf).digest('base64');
if(digest == req.get('x-shopify-hmac-sha256')){
req.headers['x-kotn-webhook-verified']= '200';
}
}}));
and then any web hooks just deal with the verified header:
if('200' != req.get('x-kotn-webhook-verified')){
console.log('invalid signature for uninstall');
res.status(204).send();
return;
}
var shop = req.get('x-shopify-shop-domain');
if(!shop){
console.log('missing shop header for uninstall');
res.status(400).send('missing shop');
return;
}
Short Answer
The body parser in express does not handle BigInt well, and things like order number which are passed as integer get corrupted. Apart from that certain values are edited such as URLs are originally sent as "https://...", which OP also found out from the other code.
To solve this, do not parse the data using body parser and instead get it as raw string, later on you can parse it with json-bigint to ensure none of it has been corrupted.
Long Answer
Although the answer by #bknights works perfectly fine, it's important to find out why this was happening in the first place.
For a webhook I made on the "order_created" event from Shopify I found out that the id of the request being passed to the body was different than what I was sending from my test data, this turned out to be an issue with body-parser in express which did not play nice with big integers.
Ultimately I was deploying something to Google cloud functions and the req already had raw body which I could use, but in my test environment in Node I implemented the following as a separate body parser as using the same body parser twice overwrote the raw body with JSON
var rawBodySaver = function (req, res, buf, encoding) {
if (buf && buf.length) {
req.rawBody = buf.toString(encoding || 'utf8');
}
}
app.use(bodyParser.json({verify: rawBodySaver, extended: true}));
Based on this answer
I later on parse the rawBody using json-bigint for use in code elsewhere as otherwise some of the numbers were corrupted.
// Change the way body-parser is used
const bodyParser = require('body-parser');
var rawBodySaver = function (req, res, buf, encoding) {
if (buf && buf.length) {
req.rawBody = buf.toString(encoding || 'utf8');
}
}
app.use(bodyParser.json({ verify: rawBodySaver, extended: true }));
// Now we can access raw-body any where in out application as follows
// request.rawBody in routes;
// verify webhook middleware
const verifyWebhook = function (req, res, next) {
console.log('Hey!!! we got a webhook to verify!');
const hmac_header = req.get('X-Shopify-Hmac-Sha256');
const body = req.rawBody;
const calculated_hmac = crypto.createHmac('SHA256', secretKey)
.update(body,'utf8', 'hex')
.digest('base64');
console.log('calculated_hmac', calculated_hmac);
console.log('hmac_header', hmac_header);
if (calculated_hmac == hmac_header) {
console.log('Phew, it came from Shopify!');
res.status(200).send('ok');
next();
}else {
console.log('Danger! Not from Shopify!')
res.status(403).send('invalid');
}
}
Had the same issue. Using request.rawBody instead of request.body helped:
import Router from "koa-router";
import koaBodyParser from "koa-bodyparser";
import crypto from "crypto";
...
koaServer.use(koaBodyParser());
...
koaRouter.post(
"/webhooks/<yourwebhook>",
verifyShopifyWebhooks,
async (ctx) => {
try {
ctx.res.statusCode = 200;
} catch (error) {
console.log(`Failed to process webhook: ${error}`);
}
}
);
...
async function verifyShopifyWebhooks(ctx, next) {
const generateHash = crypto
.createHmac("sha256", process.env.SHOPIFY_WEBHOOKS_KEY) // that's not your Shopify API secret key, but the key under Webhooks section in your admin panel (<yourstore>.myshopify.com/admin/settings/notifications) where it says "All your webhooks will be signed with [SHOPIFY_WEBHOOKS_KEY] so you can verify their integrity
.update(ctx.request.rawBody, "utf-8")
.digest("base64");
if (generateHash !== shopifyHmac) {
ctx.throw(401, "Couldn't verify Shopify webhook HMAC");
} else {
console.log("Successfully verified Shopify webhook HMAC");
}
await next();
}