Node.js: how to disable chunked transfer-encoding? - node.js

I'm missing a content-length header on my response from a Node server that I'm piping a .zip file from another location. I've injected a content-length header via the code below, but still it seems the transfer-encoding: chunked is overwriting it somehow.
Response Headers
HTTP/1.1 200 OK
access-control-allow-origin: *
connection: close
content-type: application/zip
date: Mon, 14 Jul 2014 03:47:00 GMT
etag: "\"eb939974703e14ee9f578642972ed984\""
last-modified: Sat, 12 Jul 2014 02:15:52 GMT
server: Apache-Coyote/1.1
set-cookie: rememberMe=deleteMe; Path=/; Max-Age=0; Expires=Sun, 13-Jul-2014 03:47:00 GMT
transfer-encoding: chunked
X-Powered-By: Express
Code
var request = require('request');
var express = require('express');
var async = require('async');
var app = express();
app.get('/:bundle_id?', function(req, res) {
var bundle_id = req.params.bundle_id;
bundle_id = bundle_id.replace(/\.zip$/, '');
var url = "https://url....../bundles/" + bundle_id;
async.waterfall([
function(callback) {
request.get(url, function(req, res, data) {
callback(null, JSON.parse(data).entities[0]['file-metadata']['content-length']);
});
}
], function(err, contentLength) {
request.get({
url: url,
headers: {
"Accept": "application/zip"
}
}).pipe(res);
res.oldWriteHead = res.writeHead;
res.writeHead = function(statusCode, reasonPhrase, headers) {
res.header('Content-Length', contentLength);
res.oldWriteHead(statusCode, reasonPhrase, headers);
}
});
});
app.listen(9000);

Turns out this was actually a rather simple fix: setting the transfer-encoding header to an empty string in the response solved the problem:
...
res.oldWriteHead = res.writeHead;
res.writeHead = function(statusCode, reasonPhrase, headers) {
res.header('Content-Length', contentLength);
res.header('transfer-encoding', ''); // <-- add this line
res.oldWriteHead(statusCode, reasonPhrase, headers);
}
...
The reason this works, is because after doing some digging, it appears the transfer-encoding header replaces content-length (since both can't co-exist). It just so happens that the clients I was using to test were choosing chunked transfer encoding over content length.

If you define a Content-Length, Transfer-Encoding will no longer be sent to "chunked".

Related

Unable to set cookie from node server -> app

I'm trying to set an httpOnly cookie from my node.js api (localhost:3001) to work with my react client app (localhost:3000), everything I've tried so far results in no cookie being set in my browser. Some key factors about my setup:
Backend is node, running fastify, fastify-cookie & cors
// CORS
server.use(
require('cors')({
origin: ['https://localhost:3000'],
optionsSuccessStatus: 200,
credentials: true
})
)
// Cookies
server.register(require('fastify-cookie'), {
secret: process.env.JWT_SECRET
})
// Sending the cookie
reply
.setCookie('token', token, {
domain: 'localhost',
path: '/',
secure: true,
sameSite: 'lax',
httpOnly: true
})
.send({ user })
Client is running https localhost in chrome, making api calls using fetch.
const fetchUsers = async () => {
const req = await fetch(`${process.env.USERS_API_BASE}/users`, { credentials: 'include' })
const res = await req.json()
console.log(res)
}
Result
No cookie is ever set in my chrome application inspector, but it is sent to the browser from the server and looks correct.
set-cookie: token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1aWQiOjEsImVtYWlsIjoiaGVsbG9Ac2hhbi5kaWdpdGFsIiwiaWF0IjoxNjIwNDI1ODI0LCJleHAiOjE2MjA0Mjk0MjR9.S8eOQMtSBY85wlenuxjIGYNuk3Ec5cKQ87pAhmCvQ9w.nfRxGzq3IMFimC%2FSJeUH9Xl7bH%2FyXVprwK1NBYfur4k; Domain=localhost; Path=/; HttpOnly; Secure; SameSite=Lax
request.cookies on the sever always returns a blank object {}. Any suggestions?
What you are facing is a CORS error OR at least it is categorized as one..
you see the server seems to think you're making a cross-domain request..
If you log the responce Headers this is typically what you would see
HTTP/1.1 200 OK
Date: Sun, 20 May 2018 20:43:05 GMT
Server: Apache
Set-Cookie: name=value; expires=Sun, 20-May-2018 21:43:05 GMT; Max-Age=3600; path=/; domain=.localHost
Cache-Control: no-cache, private
Access-Control-Allow-Origin: http://localHost:8080
Vary: Origin
X-RateLimit-Limit: 60
X-RateLimit-Remaining: 59
Content-Length: 2
Keep-Alive: timeout=10, max=100
Connection: Keep-Alive
Content-Type: text/html; charset=UTF-8
but when you are making a request you kinda send it like this
const res = await axios({ method: 'POST', url: 'http://127.0.0.1:3000/api/v1/users/login', data: { email, password } });
Do you see the problem 127.0.0.1 != http://localhost:8000 and that is the solution to your problem
In short Check the Key=value Pair of Access-Control-Allow-Origin on your response and Request the domain names should match else the cookie won't be set on the browser...
Here is a GitHub Issue Link for this same problem

Overwrite all response headers with Express

I'm trying to build a Node/Express server that essentially acts as a middleman server and logs any requests that come in, then forwards the request to the destination server, and forwards back any responses that come from the destination server. The goal is to be as "transparent as possible, making it seem as if there is no middleman server at all.
The problem I'm having is that my express server seems to be dumping in a bunch of unnecessary headers in the response from the destination server.
In my app.js I have some (a lot of) middlewares that are useful for my app in general but seems to inject headers in the response:
app.use(rateLimiter);
app.use(helmet());
app.use(xss());
app.use(mongoSanitize());
app.use(nocache());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
Then I have my middleware endpoint that receives the request, forwards it, and returns the response:
exports.createLink = async (req, res, next) => {
try {
const url = 'https://destination-endpoint.com';
const options = {
url: url,
};
request( options, function(err, remoteResponse, remoteBody) {
res.writeHead(remoteResponse.statusCode, {...remoteResponse.headers});
return res.end(remoteBody);
});
} catch (error) {
console.log(error);
next(error);
}
};
If I hit my middleman I get the response:
HTTP/1.1 200 OK
Access-Control-Allow-Origin: http://localhost:3000
Vary: Accept-Encoding
Access-Control-Allow-Credentials: true
X-RateLimit-Limit: 100
X-RateLimit-Remaining: 99
date: Sun, 24 Jan 2021 05:23:07 GMT
X-RateLimit-Reset: 1611465796
Content-Security-Policy: default-src 'self';base-uri 'self';block-all-mixed-content;font-src 'self' https: data:;frame-ancestors 'self';img-src 'self' data:;object-src 'none';script-src 'self';script-src-attr 'none';style-src 'self' https: 'unsafe-inline';upgrade-insecure-requests
X-DNS-Prefetch-Control: off
Expect-CT: max-age=0
X-Frame-Options: SAMEORIGIN
Strict-Transport-Security: max-age=15552000; includeSubDomains
X-Download-Options: noopen
X-Content-Type-Options: nosniff
X-Permitted-Cross-Domain-Policies: none
Referrer-Policy: no-referrer
X-XSS-Protection: 0
Surrogate-Control: no-store
cache-control: no-cache, private
Pragma: no-cache
Expires: 0
server: nginx/1.14.2
content-type: text/plain; charset=UTF-8
transfer-encoding: chunked
connection: close
x-request-id: 864b8443-5fe2-498e-8e88-662035afe6c7
x-token-id: d0cb94e2-9c87-4d6e-b32a-11fcc698ad2c
set-cookie: laravel_session=tBlSCeel0OFIR5pL9C6f02JfXGqoyg3SN6BH6jjG; expires=Sun, 24-Jan-2021 07:23:07 GMT; Max-Age=7200; path=/; httponly
{
"foo": "bar"
}%
While if I hit the endpoint directly I just get this:
HTTP/1.1 200 OK
Server: nginx/1.14.2
Content-Type: text/plain; charset=UTF-8
Transfer-Encoding: chunked
Vary: Accept-Encoding
X-Request-Id: 1f033b57-4a2f-48a1-82b3-41bce6e2d748
X-Token-Id: d0cb94e2-9c87-4d6e-b32a-11fcc698ad2c
Cache-Control: no-cache, private
Date: Sun, 24 Jan 2021 05:24:28 GMT
Set-Cookie: laravel_session=MdOGJ18iDU28XLmNYXf5T2RxSa25KxqFisxzCBzR; expires=Sun, 24-Jan-2021 07:24:28 GMT; Max-Age=7200; path=/; httponly
{
"foo": "bar"
}%
As you can see there is a lot of extra stuff added to the header. Some of it may be solvable by removing middlewares/etc. but I think the bigger issue is that I want to return the response from the destination server AS IS no matter what. I found that I can delete the current headers before sending it like this:
request( options, function(err, remoteResponse, remoteBody) {
const headers = res.getHeaders();
for (const head in headers){
res.removeHeader(head);
}
res.writeHead(remoteResponse.statusCode, {...remoteResponse.headers});
return res.end(remoteBody);
});
But that seems very heavy handed, there has to be an easier way to overwrite/set all response headers to exactly what I need. So my overall question is:
How do I return the response from the request to the destination server EXACTLY as is?

Why does JWT Cookie get removed on page refresh

Why does the JWT Cookie disappear on page refresh?
I'm using Fetch to make a request
it initially sets the JWT cookie which I verify on Chrome Inspector
on page refresh, the cookie disappears. No specific errors are being logged.
Here's the frontend request:
$(form).submit(function(e) {
e.preventDefault();
var form_email = $('#email').val();
var form_password = $('#password').val();
var formData = {
email:form_email,
password:form_password,
}
const url = $(form).attr('action');
const options = {
method: 'POST',
body: JSON.stringify({
email:form_email,
password:form_password
}),
headers:{
'Content-Type':'application/json'
},
credentials:'include'
}
fetch(url,options)
.then(function(response) {
console.log(response);
})
});
On the server-side, I already have enabled CORS, set credentials to being true, and the origin.
const express = require("express");
const connectDB = require("./config/db");
const cors = require('cors');
const app = express();
connectDB();
app.use(cors({credentials: true, origin: 'https://zkarimi.com'}));
app.use(express.json({ extended: false }));
Here's how I handle the JWT signing and cookie response:
jwt.sign(
payload,
config.get("jwtSecret"),
{ expiresIn: 36000 },
(err, token) => {
if (err) throw err;
res.cookie('jwt',token, { httpOnly: true ,secure: true, sameSite:'None',maxAge: 3600000 })
res.json({user_email:email });
}
);
Lastly here's the response from the initial request:
HTTP/1.1 200 OK
Server: Cowboy
Connection: keep-alive
X-Powered-By: Express
Access-Control-Allow-Origin: https://zkarimi.com
Vary: Origin
Access-Control-Allow-Credentials: true
Set-Cookie: jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyIjp7ImlkIjoiNWZjODA1Zjk2MDRlNjEwMDE3NzQwZWUxIn0sImlhdCI6MTYwNzAyMjg5MSwiZXhwIjoxNjA3MDU4ODkxfQ.oXuA6DfdZbO2XW5HtLR44pVgGUWsXMyaV-l0iVeF7to; Max-Age=3600; Path=/; Expires=Thu, 03 Dec 2020 20:14:51 GMT; HttpOnly; Secure; SameSite=None
Content-Type: application/json; charset=utf-8
Content-Length: 30
Etag: W/"1e-vu/QJUeZrcotU6SR/PRLtu/c6Jw"
Date: Thu, 03 Dec 2020 19:14:51 GMT
Via: 1.1 vegur
After this request ^, refreshing the page removes the cookie from Application > Storage > Cookies and subsequent authentication requests indicate there's no JWT cookie.

Node.js - Certain headers are not being set

As i stated in the title, I tried setting headers in node.js, but some of them just would not stick/would get overwritten.
Here is content of my server.js file:
const next = require("next");
const http = require("http");
const url = require("url");
const path = require("path");
const port = process.env.PORT || 3000;
const dev = process.env.NODE_ENV !== "production";
const app = next({ dev });
const handle = app.getRequestHandler();
app.prepare().then(() => {
http
.createServer((req, res) => {
const parsedUrl = url.parse(req.url, true);
res.setHeader("X-Content-Type-Options", "nosniff");
res.setHeader("X-Frame-Options", "DENY");
res.setHeader("X-XSS-Protection", "1; mode=block");
res.setHeader(
"Strict-Transport-Security",
"max-age=31536000; includeSubDomains; preload"
);
res.setHeader("Cache-Control", "public, max-age=31557600");
handle(req, res, parsedUrl);
})
.listen(port, () => {
console.log(`listening on PORT ${port}`);
});
});
Every header except Cache-control and x-powered-by (its not in code but i tried) is getting set.
Weird thing is that when i log response, my headers are loged out:
'x-content-type-options': [ 'X-Content-Type-Options', 'nosniff' ],
'x-frame-options': [ 'X-Frame-Options', 'DENY' ],
'x-xss-protection': [ 'X-XSS-Protection', '1; mode=block' ],
'strict-transport-security': [
'Strict-Transport-Security',
'max-age=31536000; includeSubDomains; preload'
],
'cache-control': [ 'Cache-Control', 'public, max-age=31557600' ]
There are the headers that i have in my browser:
HTTP/1.1 200 OK
X-Content-Type-Options: nosniff
X-Frame-Options: DENY
X-XSS-Protection: 1; mode=block
Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
Cache-Control: no-store, must-revalidate
X-Powered-By: Next.js
ETag: "1373f-5S13UfVtDhxl5s8GCDKvO1iq/oY"
Content-Type: text/html; charset=utf-8
Vary: Accept-Encoding
Content-Encoding: gzip
Date: Thu, 06 Feb 2020 17:13:04 GMT
Connection: keep-alive
Transfer-Encoding: chunked
Any idea whats happening here? Some default setting overwriting my custom?
What you are experiencing is specific to Next.js framework.
Regarding 'X-Powered-By', this is expected, as per the Next.js docs
By default Next.js will add x-powered-by to the request headers. To opt-out of it, open next.config.js and disable the poweredByHeader config:
Regarding 'Cache-Control', it looks like it is overwritten by Next.js in development mode as you can see in the source code
sendHTML(req: IncomingMessage, res: ServerResponse, html: string) {
// In dev, we should not cache pages for any reason.
res.setHeader('Cache-Control', 'no-store, must-revalidate')
return super.sendHTML(req, res, html)
}

"invalid_request ... Missing grant type" when calling certain OAuth2 from NodeJs but not from curl with exact same header

I am very confused why I am getting "Missing grant type" error from Spring Security OAuth2 while calling it from NodeJs node-rest-client.
I checked via "sniffer" and I can see I am posting exact the same header and body. Well, the only difference I noted it is "Content-Type:application/x-www-form-urlencoded" not settup while calling from NodeJs but, as far as I could see, it is not allowed for node-rest-client. I am not expecting this to impact since I can see "grant_type=password&username=a&password=a" in exact same format as the one working from curl.
As far as I understand, in curl, "-u greencard-trusted-client:greencard-secret" means I am passing it throw header and "-d "grant_type=password&username=a&password=a"" as body, so, I understand I am using node-rest-client in the same way I am using curl bellow.
Any suggestion will be appreciated.
curl straight to the Spring OAuth2 service
curl -u myapp-trusted-client:myapp-secret -k -d "grant_type=password&username=a&password=a" -H "Content-Type:application/x-www-form-urlencoded" http://localhost:9080/myclient/oauth/token
>sudo ngrep -Wbyline -d lo port 9080
interface: lo (127.0.0.0/255.0.0.0)
filter: (ip or ip6) and ( port 9080 )
####
T 127.0.0.1:38606 -> 127.0.0.1:9080 [AP]
POST /myclient/oauth/token HTTP/1.1.
Host: localhost:9080.
Authorization: Basic Z3JlZW5jYXJkLXRydXN0ZWQtY2xpZW50OmdyZWVuY2FyZC1zZWNyZXQ=.
User-Agent: curl/7.47.0.
Accept: */*.
Content-Type:application/x-www-form-urlencoded.
Content-Length: 41.
.
grant_type=password&username=a&password=a
##
T 127.0.0.1:9080 -> 127.0.0.1:38606 [AP]
HTTP/1.1 200 OK.
X-Powered-By: Servlet/3.1.
Cache-Control: no-store.
Pragma: no-cache.
Content-Type: application/xml;charset=UTF-8.
X-Content-Type-Options: nosniff.
X-XSS-Protection: 1; mode=block.
X-Frame-Options: DENY.
Content-Language: en-US.
Transfer-Encoding: chunked.
Date: Thu, 09 Mar 2017 20:10:54 GMT.
.
105.
<OAuth2AccessToken><access_token>78048b70-f84c-476c-ba4f-6eecca1c5f77</access_token><token_type>bearer</token_type><refresh_token>78410631-e3a3-4c75-b8f5-7373bbcd4fd1</refresh_token><expires_in>119</expires_in><scope>read write trust</scope></OAuth2AccessToken>.
##
T 127.0.0.1:9080 -> 127.0.0.1:38606 [AP]
exact same service consumed by node-rest-client
ngrep -Wbyline -d lo port 9080
interface: lo (127.0.0.0/255.0.0.0)
filter: (ip or ip6) and ( port 9080 )
####
T 127.0.0.1:38750 -> 127.0.0.1:9080 [AP]
POST /myclient/oauth/token HTTP/1.1.
Authorization: Basic Z3JlZW5jYXJkLXRydXN0ZWQtY2xpZW50OmdyZWVuY2FyZC1zZWNyZXQ=.
Content-Length: 41.
Host: 127.0.0.1:9080.
Connection: close.
.
grant_type=password&username=a&password=a
##
T 127.0.0.1:9080 -> 127.0.0.1:38750 [AP]
HTTP/1.1 400 Bad Request.
X-Powered-By: Servlet/3.1.
Cache-Control: no-store.
Pragma: no-cache.
Content-Type: application/xml;charset=UTF-8.
X-Content-Type-Options: nosniff.
X-XSS-Protection: 1; mode=block.
X-Frame-Options: DENY.
Content-Language: en-US.
Transfer-Encoding: chunked.
Connection: Close.
Date: Thu, 09 Mar 2017 20:15:09 GMT.
.
7a.
<OAuth2Exception><error>invalid_request</error><error_description>Missing grant type</error_description></OAuth2Exception>.
##
T 127.0.0.1:9080 -> 127.0.0.1:38750 [AP]
0.
nodejs calling the Spring OAuth2 Rest Service
var express = require('express');
var bodyParser = require('body-parser');
var Client = require('node-rest-client').Client;
var client = new Client();
client.registerMethod("postMethod", "http://127.0.0.1:9080/myclient/oauth/token", "POST");
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
var port = process.env.PORT || 3000;
var router = express.Router();
var tokenRoute = router.route('/token');
tokenRoute.post(function (req, res) {
var username = 'myapp-trusted-client';
var password = 'myapp-secret';
var auth = 'Basic ' + new Buffer(username + ':' + password).toString('base64');
var args = {
//data: req.body,
data: 'grant_type=password&username=a&password=a',
headers: {'Authorization': auth }
//headers: { "Content-Type": "application/json" }
};
client.methods.postMethod(args, function (data, response) {
res.writeHead(200, { "Content-Type": "application/json" });
var json = JSON.stringify({
tokenBackEnd: data
});
res.end(json);
});
});
app.use('/myclient', router);
app.listen(port);

Resources