Express Compress not working in Node website - node.js

I've my site built on node + express, and trying to enable gzip compression using this tutorial but it didn't worked as shown in the tutorial. I can't see Content-Encoding in response header.
Here is my code.
const compression = require('compression');
const express = require("express");
const path = require("path");
var redirects = require('express-seo-redirects');
var proxy = require('express-http-proxy');
require('dotenv').config()
/**
* App Variables
*/
const app = express();
app.use(compression({ filter: shouldCompress, threshold: 0 }));
//app.use(compression()) - //I've also tried this.
function shouldCompress (req, res) {
if (req.headers['x-no-compression']) {
// don't compress responses with this request header
return false
}
// fallback to standard filter function
return compression.filter(req, res)
}
let setCache = function (req, res, next) {
// here you can define period in second, this one is 1 day
const period = 1 * 24 * 60 * 60 * 1000
// you only want to cache for GET requests
if (req.method == 'GET') {
res.set('Cache-control', `public, max-age=${period}`)
} else {
// for the other requests set strict no caching parameters
res.set('Cache-control', `no-store`)
}
// res.set('Content-Encoding', 'gzip')
// remember to call next() to pass on the request
next()
}
app.use(setCache)
And when I use res.set('Content-Encoding', 'gzip') inside app.get("/", (req, res) => { then it shows in response header but website stop working (not showing any error other than blank screen).
Bellow images are of my rest code.

Gzip compression was showing only for resource files (css, js etc..). So I resolved it by adding res.contentType('text/html'); in setCache function.
const exceptions = ['.js', '.css', '.ico', '.jpg', '.jpeg', '.png', '.gif', '.tiff', '.tif', '.bmp', '.svg', '.ttf', '.eot', '.woff', '.php', '.xml', '.xsl'];
let setCache = function (req, res, next) {
// here you can define period in second, this one is 5 minutes
const period = 1 * 24 * 60 * 60 * 1000;
if(!exceptions.some(v => req.url.includes(v))){
res.contentType('text/html');
}
// you only want to cache for GET requests
if (req.method == 'GET') {
res.set('Cache-control', `public, max-age=${period}`)
} else {
// for the other requests set strict no caching parameters
res.set('Cache-control', `no-store`)
}
// res.set('Content-Encoding', 'gzip')
// remember to call next() to pass on the request
next()
}
app.use(setCache)

Related

There is a problem with multiple guest requests

Next.js/server.js project
My server.ts file:
import * as express from 'express';
import * as next from 'next';
...
import * as rateLimit from 'express-rate-limit';
import * as config from '../config';
// const apiLimiter = rateLimit({
// windowMs: 1 * 60 * 1000, // 1 minutes
// max: 200,
// message: "Too many requests from this IP"
// });
const routes = require('./nextRoutes');
const app = next({ dev: config.dev });
app.prepare().then(() => {
const server = express();
// server.use(apiLimiter);
...
server.get('*', (req, res) => {
const handle = routes.getRequestHandler(app);
handle(req, res, req.url);
});
server.listen(config.port, (err) => {
if (err) { throw err; }
});
})
question 1: I tried limiting requests to 200 per minute. But this limitation worked for all requests. How can I limit a specific IP address, since it will not work to limit the sessionID, they are all different come? I want to restrict by IP
question 2: Is there any way to get the IP address exceeding the limit, some callback inside the given middleware? I looked through the documentation and understand that there is such a way, but I did not understand how to implement it.
Thanks for any answers or ideas!!! Plus in karma. And have a nice day!

node, express - restrict access to downloadable file

I am using express as server for micro-services rest api. Endpoints are built from directory structure. There are few downloadable pdf files which are currently at client side. And it can be downloadable (with the href URL) even if user is not logged into the portal. So, I put all the pdf files to server.
Directory structure on server:
pdf files are inside docs directory. Please find below the code of server:
/* global __dirname */
import morgan from 'morgan';
import logger, { webStream } from './services/logger';
import { socket } from './services';
// set env variables before all else
import { GATEWAY_PORT, CORS_ORIGINS } from './config';
const express = require('express');
const bodyParser = require('body-parser');
const cookieParser = require('cookie-parser')();
const version = require('./services/utils').version();
const authentication = require('./services/authentication');
const utils = require('./services/utils');
// set up app and middleware
const app = express();
app.use(morgan('User::req[user-id] Correlation::req[x-correlation-id] Method::method URL::url Status::status :res[content-length] - :response-time ms', { stream: webStream }));
logger.info('Starting...');
app.use(cookieParser);
app.use(bodyParser.json({ limit: '50mb' }));
app.disable('x-powered-by');
// CORS headers to allow running client/server on different ports
app.use((req, res, next) => {
// Check if the origin is whitelisted in the env vars
const actual = req.headers.origin || '';
if (utils.matchCors(actual, CORS_ORIGINS.split(','))) {
res.set({ 'Access-Control-Allow-Origin': actual });
}
res.set({
// standard CORS headers
'Access-Control-Allow-Headers': 'Content-Type, Authorization, Accept, Accept-Language',
'Access-Control-Allow-Credentials': true,
'Access-Control-Allow-Methods': 'PATCH,POST,GET,DELETE',
// addresses security issues identified by automated pen testing
'X-Frame-Options': 'DENY',
'X-Content-Type-Options': 'nosniff',
'X-XSS-Protection': 1,
});
next();
});
// set the user property of the request object
app.use((req, res, next) => {
const token = req.cookies[authentication.cookieName];
if (!token) {
req.user = false;
} else {
req.user = authentication.decodeJWT(token);
authentication.setCookie(res, token, req.user);
}
utils.setCorrelationId(req.headers['x-correlation-id']);
req.correlationId = req.headers['x-correlation-id'];
next();
});
// helper function returning middleware to reject unauthorised users
function requiredRoles(roles, abcOnly) {
return function requireRolesHandler(req, res, next) {
if (
!req.user
|| (abcOnly && !req.user.isabc)
|| !authentication.hasRole(req.user, roles)) {
const error = new Error('UNAUTHORISED');
error.status = 403;
next(error);
} else {
next();
}
};
}
// Add the endpoints to express.
// Reversed to get literal routes before # capture groups.
utils.parseDirectory(`${__dirname}/rest`, [], true).reverse().forEach((endpoint) => {
const { auth, functions } = endpoint.handler;
if (auth) {
functions.unshift(requiredRoles(auth.roles, auth.abcOnly));
}
app[endpoint.method](
endpoint.url,
functions,
);
});
// setup server
const server = app.listen(GATEWAY_PORT, () => {
logger.info(`Allowed CORS: ${CORS_ORIGINS}`);
logger.info(`Started ${version.name} (${version.number}) listening on ${GATEWAY_PORT}`);
});
socket.createServer(server);
How do I serve pdf files from server to client only to authorized user when user clicks on link on a page ?
Have a route to download file e.g. GET /api/download?file=abc.pdf
Now in the middleware,
Check if the req.user exists or not.
Check if the user has sufficient rights to download the file or
not
If 1 and 2 satisfy, then serve the file
Code would look more or less like this:
app.get('/api/download', (req, res, next) => {
// Check if the request had valid token or not
if(!req.user) {
const error = new Error('UNAUTHORISED');
error.status = 403;
return next(error);
}
const { user } = req;
const { file } = req.query;
// If you want to have some additional logic wherein
// you want to restrict the download of the file,
// you can put that logic in this function
const isAllowed = canDownload(user, file);
if(isAllowed) {
return res.sendFile(path.join(__dirname, 'docs', path.sep, file));
}
const error = new Error('UNAUTHORISED');
error.status = 403;
return next(error);
})
You might need to require path, implement canDownload or solve no such file or directory errors because of __dirname usage. All of those are trivial. If you need help for those as well, let me know in the comments.
Here is the reference to response.sendFile()
And this might be helpful too.

node.js session values aren't accessible in second api call

i am facing the following issue:
(I am using node-client-sessions module)
I send an ajax request, through the browser, to my API : /api/request1
In my api.js I handle that request, calculate some stuff and write some results into the session like this.
router.post('/request1', function(req, response, next) {
// some wield calculations
req.session.calcData = { // some content }; 
// some other calculations
console.log(req.session.calcData); // logs the correct object
response.send('success');
}
After receiving the success on client side I send another api call like this for example: /api/request2
router.post('/request2', function(req, response, next) {
// here i want to use some of the results from the previous calculations which i stored in the calcData session.
console.log(req.session.calcData); // this logs undefined
}
Shouldn't req.session.calcData be available in both functions?
Enviroment Info
Express Framework 4.x
app.js :
...
var app = express();
...
var session = require('client-sessions');
...
app.use(session({
cookieName: 'session',
secret: 'random_string_goes_here',
duration: 30 * 60 * 9999999,
activeDuration: 5 * 60 * 1000,
}));
...
app.use('/api', api);
According to example at client-session, you must use req.csession and req.csflush();
var express = require('express');
var path = require('path')
var cs = require('client-session');
var clientSession = cs('mysecretkey');
var app = express();
app.use(clientSession.connect());
app.get('/', function(req, res){
var count = req.csession['count'];
if(!count) count = 1;
else count++;
req.csession['count'] = count;
//sync to cookie session equal to res.csflush(),make sure to call it before response
req.csflush();
res.send(count.toString());
});
app.listen(8124);
console.log('Server running at http://127.0.0.1:8124/');

Express: SetHeader 'Cache-Control' for a single file after setting maxAge

node newbie here. The below code results in all files receiving the response header "Cache-Control public, max-age=31536000". I would like the index to have a max-age of 0. I have tried several different methods but the root of it seems to be a more fundamental misunderstanding on my part.
Why does the below res.setHeader line not take effect (so that I can understand my misconception) and what would be the correct method to ensure Index.html is not cached?
var express = require('express'),
gzipStatic = require('connect-gzip-static'),
app = express(),
port= process.env.PORT || 3030,
cacheAge = 365 * 24 * 60 * 60 * 1000;
app.use(gzipStatic(__dirname + '/public', { maxAge: cacheAge }));
app.get('/', function(req, res) {
res.setHeader('Cache-Control', 'max-age=0');
res.sendFile('index.html');
});
Try to set header in app.use():
app.use(function (req, res, next) {
......
res.setHeader('Cache-Control', 'public, max-age=31557600'); // 1 year
......
})

Node.js HTML5 Boilerplate Server Config Integration

I'm setting up the structure of a new project which will be built using Node.js and Express. I 'm using HTML5 Boilerplate for an optimal starting point. It comes with configuration files for multiple types of servers: Apache, Nginx, Node.js, etc. The following is the Node.js server configuration file provided by the HTML5 Boilerplate team:
/* h5bp server-configs project
*
* maintainer: #xonecas
* contributors: #niftylettuce
*
* NOTES:
* compression: use the compress middleware provided by connect 2.x to enable gzip/deflate compression
* http://www.senchalabs.org/connect/compress.html
*
* concatenation: use on of the following middlewares to enable automatic concatenation of static assets
* - https://github.com/mape/connect-assetmanager
* - https://github.com/TrevorBurnham/connect-assets
*/
var h5bp = module.exports,
_http = require('http'),
_parse = require('url').parse;
// send the IE=Edge and chrome=1 headers for IE browsers
// on html/htm requests.
h5bp.ieEdgeChromeFrameHeader = function () {
return function (req, res, next) {
var url = req.url,
ua = req.headers['user-agent'];
if (ua && ua.indexOf('MSIE') > -1 && /html?($|\?|#)/.test(url)) {
res.setHeader('X-UA-Compatible', 'IE=Edge,chrome=1');
}
next();
};
};
// block access to hidden files and directories.
h5bp.protectDotfiles = function () {
return function (req, res, next) {
var error;
if (/(^|\/)\./.test(req.url)) {
error = new Error(_http.STATUS_CODES[405]); // 405, not allowed
error.status = 405;
}
next(error);
};
};
// block access to backup and source files
h5bp.blockBackupFiles = function () {
return function (req, res, next) {
var error;
if (/\.(bak|config|sql|fla|psd|ini|log|sh|inc|swp|dist)|~/.test(req.url)) {
error = new Error(_http.STATUS_CODES[405]); // 405, not allowed
error.status = 405;
}
next(error);
};
};
// Do we want to advertise what kind of server we're running?
h5bp.removePoweredBy = function () {
return function (req, res, next) {
res.removeHeader('X-Powered-By');
next();
};
};
// Enable CORS cross domain rules, more info at http://enble-cors.org/
h5bp.crossDomainRules = function () {
return function (req, res, next) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With');
next();
};
};
// Suppress or force 'www' in the urls
// #param suppress = boolean
h5bp.suppressWww = function (suppress) {
return function (req, res, next) {
var url = req.url;
if (suppress && /^www\./.test(url)) {
res.statusCode = 302;
res.setHeader('Location', url.replace(/^www\./,''));
}
if (!suppress && !/^www\./.test(url)) {
res.statusCode = 302;
res.setHeader('Location', "www."+url);
}
next();
};
};
// Far expire headers
// use this when not using connect.static for your own expires/etag control
h5bp.expireHeaders = function (maxAge) {
return function (req, res, next) {
res.setHeader('Cache-Control', 'public, max-age='+ (maxAge));
next();
};
};
// Etag removal
// only use this is you are setting far expires for your files
// ** WARNING ** connect.static overrides this.
h5bp.removeEtag = function () {
return function (req, res, next) {
res.removeHeader('Last-Modified');
res.removeHeader('ETag');
next();
};
};
// set proper content type
// #param mime = reference to the mime module (https://github.com/bentomas/node-mime)
h5bp.setContentType = function (mime) {
return function (req, res, next) {
// I'm handling the dependency by having it passed as an argument
// we depend on the mime module to determine proper content types
// connect also has the same dependency for the static provider
// ** #TODO ** maybe connect/express expose this module somehow?
var path = _parse(req.url).pathname,
type = mime.lookup(path);
res.setHeader('Content-Type', type);
next();
};
};
// return a express/connect server with the default middlewares.
// #param serverConstructor = express/connect server instance
// #param options = {
// root: 'path/to/public/files',
// maxAge: integer, time in miliseconds ex: 1000 * 60 * 60 * 24 * 30 = 30 days,
// mime: reference to the mime module ex: require('mime')
// }
// Depends:
// express or connect server
// mime module [optional]
h5bp.server = function (serverConstructor, options) {
var server = serverConstructor.createServer(),
stack = [
this.suppressWww(true),
this.protectDotfiles(),
this.blockBackupFiles(),
this.crossDomainRules(),
this.ieEdgeChromeFrameHeader()
//,this.expireHeaders(options.maxAge),
// this.removeEtag(),
// this.setContentType(require('mime'))
];
// express/connect
if (server.use) {
stack.unshift(serverConstructor.logger('dev'));
stack.push(
//serverConstructor.compress(), // express doesn't seem to expose this middleware
serverConstructor['static'](options.root, { maxAge: options.maxAge }), // static is a reserved
serverConstructor.favicon(options.root, { maxAge: options.maxAge }),
serverConstructor.errorHandler({
stack: true,
message: true,
dump: true
})
);
for (var i = 0, len = stack.length; i < len; ++i) server.use(stack[i]);
} else {
server.on('request', function (req, res) {
var newStack = stack,
func;
(function next (err) {
if (err) {
throw err;
return;
} else {
func = newStack.shift();
if (func) func(req, res, next);
return;
}
})();
});
}
return server;
};
My question is this: how exactly do I go about integrating this with Express? The section of code that specifically confuses me is the bottom portion:
// return a express/connect server with the default middlewares.
// #param serverConstructor = express/connect server instance
// #param options = {
// root: 'path/to/public/files',
// maxAge: integer, time in miliseconds ex: 1000 * 60 * 60 * 24 * 30 = 30 days,
// mime: reference to the mime module ex: require('mime')
// }
// Depends:
// express or connect server
// mime module [optional]
h5bp.server = function (serverConstructor, options) {
var server = serverConstructor.createServer(),
stack = [
this.suppressWww(true),
this.protectDotfiles(),
this.blockBackupFiles(),
this.crossDomainRules(),
this.ieEdgeChromeFrameHeader()
//,this.expireHeaders(options.maxAge),
// this.removeEtag(),
// this.setContentType(require('mime'))
];
// express/connect
if (server.use) {
stack.unshift(serverConstructor.logger('dev'));
stack.push(
//serverConstructor.compress(), // express doesn't seem to expose this middleware
serverConstructor['static'](options.root, { maxAge: options.maxAge }), // static is a reserved
serverConstructor.favicon(options.root, { maxAge: options.maxAge }),
serverConstructor.errorHandler({
stack: true,
message: true,
dump: true
})
);
for (var i = 0, len = stack.length; i < len; ++i) server.use(stack[i]);
} else {
server.on('request', function (req, res) {
var newStack = stack,
func;
(function next (err) {
if (err) {
throw err;
return;
} else {
func = newStack.shift();
if (func) func(req, res, next);
return;
}
})();
});
}
return server;
};
My JavaScript isn't exactly at a beginners level but I wouldn't say I'm advanced either. This code is beyond me. Any pointers as to what I can read, watch, or do, to learn what I'm obviously missing here would be appreciated.
Most of the file is made up of a series of functions that generate middleware for frameworks, like Express, that conform to Connect's middleware specification. The second code listing is designed to create an HTTP server that uses all these functions. From what I can tell, it looks like you're supposed to pass in whatever you would normally call createServer on, and h5bp will do the creation and setup for you. For example, if you would normally do:
var express = require('express');
var server = express.createServer();
You would instead pass express to h5bp.server, which calls createServer on whatever you pass in right off the bat:
var express = require('express');
var server = h5bp.server(express, options);
After a bit of setup, it checks to see if the server has a function called use (the line is if (server.use)), and if so uses it to inject all the middleware it set up into the server. If it doesn't, then it assumes you're using a raw Node.js HTTP server, and sets up the necessary code to pass the request through each of the items in stack manually (this is what Connect/Express does for you).
It's worth noting that, in Express 3 (currently in release candidate stage), the applications created by Express no longer inherit from Node's HTTP server, so you don't call createServer on express; instead, you should just call express() and then pass the results of that into http.createServer. (See "Application function" at Migrating from 2.x to 3.x on the Express wiki for more information.) This means that this script is not compatible with the latest version of Express.
[update]
If you take a look at the test directory on GitHub, you can see an example app:
var express = require('express'),
h5bp = require('../node.js'),
server = h5bp.server(express, {
root: __dirname,
maxAge: 1000 * 60 * 60 * 30
});
server.listen(8080);
console.log('ok');
There was a major update of h5bp for node.js.
You can use it as an express middleware now.
The repository has moved here : https://github.com/h5bp/node-server-config.
From the documentation:
var express = require('express'),
h5bp = require('h5bp');
var app = express();
// ...
app.use(h5bp({ root: __dirname + '/public' }));
app.use(express.compress());
app.use(express.static(__dirname + '/public'));
// ...
app.listen(3000);

Resources