Node.js: Unable to compress response using express - node.js

I am using express version 4.13.4 and the following code for my app.js. I have tried changing the location of app.use(compression()) which did not show any effect. when I run the application I saw no evidence of compression in viewing the chrome dev tools response headers i.e it doesn't have the gzip content-encoding header.
I am new to node js.I want to gzip compress my response to browser. Please help me fix this issue.
var compression = require('compression')
var express = require('express');
var http = require('http');
var app = express();
app.use(compression());
var settings = {
UiServerPort: 8080,
ApiServerHost: "localhost",
ApiServerPort: 12121
};
app.use('/ui', express.static('ui'));
app.all('/api/*', function (req, res) {
var options = {
host: settings.ApiServerHost,
port: settings.ApiServerPort,
path: req.url.substring(4),
method: 'POST'
};
var requestData = '';
req.on('data', function (data) { requestData += data; });
req.on('end', function () {
var request = http.request(options, function (response) {
var responseData = '';
res.flush();
response.on('data', function (data) { responseData += data; });
response.on('end', function () {
res.statusCode = response.statusCode;
res.write(responseData);
res.end();
});
});
request.write(requestData);
request.end();
});
});
app.listen(settings.UiServerPort)

you saw " Vary Accept-Encoding " ?? if you don't use compression , this won't show. and I paste your code ,but It can't run.

Instead of
app.use(compression())
You should add this piece of code :
app.use(compression({filter: shouldCompress}))
function shouldCompress (req, res) {
if (req.headers['x-no-compression']) {
// don't compress responses with this request header
return false
}
// fallback to standard filter function
return compression.filter(req, res)
}
PS : it works for me.

Related

how to receive gzip data ? Node.js

I'm trying to retrieve data from KEEPA about Amazon's products.
I'm straggling to receive the data in proper JSON format, as KEEPA sending the data as gzip.
I tried to used 'decompressResponse' module which helped to get the data as JSON but it was received multiple times on each call.
As the code appears below I'm just getting a huge Gibberish to my console.
Let me know what am I missing here, or if you have a better suggestion please let me know.
Thanks
const express = require("express");
const https = require("https");
const bodyParser = require("body-parser");
const app = express();
app.use(bodyParser.urlencoded({extended: true}));
app.get("/", function(req, res) {
res.sendFile(__dirname + "/index.html");
});
app.post("/", function(req, res) {
const query = req.body.asinId;
const apiKey = "MY_API_KEY";
const url = "https://api.keepa.com/product?key=" + apiKey + "&domain=1&asin=" + query;
const options = {
methode: "GET",
headers: {
"Content-Type": "application/json;charset=UTF-8",
"Accept-Encoding":"gzip"
}
}
https.get(url,options,function(res) {
console.log(res.statusCode);
console.log(res.headers);
var data;
res.on("data", function(chunk){
if(data){
data = chunk;
} else {
data += chunk;
}
console.log(data);
});
});
res.send("server is running");
});
app.listen(3000, function() {
console.log("server is running on port 3000");
});
Have you tried using the built-in zlib module with gunzip()?
zlib.gunzip(data, (error, buff) => {
if (error != null) {
// An error occured while unzipping the .gz file.
} else {
// Use the buff which contains the unzipped JSON.
console.log(buff)
}
});
Full example with your code: https://www.napkin.io/n/7c6bc48d989b4727
well the output function was wrong .. the correct one below
https.get(url,options,function(response) {
response = decompressResponse(response);
console.log(res.statusCode);
console.log(res.headers);
let data = '';
response.on("data", function(chunk){
data += chunk;
});
response.on("end",function(){
console.log(data);
});
});

Why does NodeJs server return the same data 2 times?

I am currently learning NodeJs, thus I am building a simple NodeJs server which is called server.js
const http = require('http');
const port = 3000;
const fs = require('fs');
const sourceFile = './client/src/account.json';
var data = []
const service = http.createServer(
(req, res) => {
var receive = "";
var result = "";
req.on('data', (chunk)=>{ receive += chunk })
req.on('end', () =>{
data = fs.readFileSync(sourceFile, 'UTF8');
result = JSON.stringify(data);
var data_receive = receive;
console.log(data)
res.setHeader('Access-Control-Allow-Origin', '*');
res.write(data);
res.end()
})
})
service.listen(port)
Why does every time I request to the server, the console.log returns the data 2 times. It seems like it is looped somewhere.
This is my json file account.json
[
{
"id":"account_1",
"pass":"abc123",
"name":"Account 1"
}
]
Thank you for you help!
When performing the request from the browser, you will get an extra request for favicon.ico.
To avoid those double requests, handle the favicon.
if (req.url === '/favicon.ico') {
res.writeHead(200, { 'Content-Type': 'image/x-icon' });
console.log('favicon...');
return res.end();
}
/* The rest of your code */

Node.js Application Performance

I currently have a node.js script sitting on Azure that gets a file (via a download URL link to that file) and base64 encodes it, and then sends this base64 encoded file back to the request source. The problem I am running into is performance based. The script below, in some instances, is timing out a separate application by having a run time over 30 seconds. The file in question on one of these timeouts was under a MB in size. Any ideas?
The script:
const express = require('express');
const bodyParser = require('body-parser');
const https = require('https');
const fs = require('fs');
const request = require('request');
const util = require('util');
const port = process.env.PORT || 3000;
var app = express();
app.use(bodyParser.json());
app.post('/base64file', (req, res) => {
var fileURL = req.body.fileURL;
var listenerToken = req.body.listenerToken;
var testingData = {
fileURL: fileURL,
listenerToken: listenerToken
};
/*
Make sure correct token is used to access endpoint..
*/
if( listenerToken !== <removedforprivacy> ) {
res.status(401);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ error: 'You are not authorized'}));
} else if ( !fileURL ){
res.status(400);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ error: 'The request could not be understood by the server due to malformed syntax.'}));
} else {
https.get(fileURL, function(response) {
var data = [];
response.on('data', function(chunk) {
data.push(chunk);
}).on('end', function() {
//build the base64 endoded file
var buffer = Buffer.concat(data).toString('base64');
//data to return
var returnData = {
base64File: buffer
};
if( buffer.length > 0 ) {
res.setHeader('Content-Type', 'application/json');
res.status(200);
res.send(JSON.stringify(returnData));
} else {
res.setHeader('Content-Type', 'application/json');
res.status(404);
res.send(JSON.stringify({ error: 'File URL not found.'}));
}
});
});
}
});
app.listen(port, () => {
console.log('Server is up and running ' + port);
});
One idea: you are missing error handling.
If you get an error on the https.get(), you will just never send a response and the original request will timeout.

how to get data from an api on the server side and pass the data retrieved to pugjs file using expressjs?

sample code:
// index.pug
p #{polls}
// apiendpoint
http://localhost:8080/api/polls
// routes file (index.js):
Here, how do I make get request to the api, and pass the retrieved result from api(locals) to polls variable while rendering the profile.pug
app.route('/profile')
.get(isLoggedIn, function (req, res) {
res.render('profile', {'polls': passvaluehere});
});
});
You can also use **http** module like this
var http = require('http');
var options = {
host: 'localhost',
path: '/api/polls',
port: '80',
method: 'GET'
};
var req = http.request(options, response);
var str = ''
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
console.log(str);
res.render('profile', {'polls': str});
});
req.end();

http request does not emit 'end' after proxy

I need to be able to use the http request body in my request proxy application and then again in the actual web service. I am using restreamer to 'reset' the stream (and even wrote the middleware myself with no change). The web service receives the body just fine, but because end is never emitted, I cannot continue with the request.
Testing with postman, sending a raw body, with content type set. Any suggestions would be greatly appreciated. Thanks!
var express = require('express')
, bodyParser = require('body-parser')
, http = require('http')
, restreamer = require('connect-restreamer')
, httpProxy = require('http-proxy')
var app = express();
app.use(function (req, res, next) {
var body = '';
req.on('data', function (chunk) {
body += chunk.toString('utf8');
});
req.on('end', function (chunk) {
req.body = JSON.parse(body)
next();
});
});
app.use(restreamer());
var proxy = httpProxy.createServer();
app.all('*', function (req, res) {
proxy.web(req, res, {
target: 'http://localhost:8001'
});
});
http.createServer(app).listen(8000);
app2 = express();
app2.use(function (req, res, next) {
var body = '';
req.on('data', function (chunk) {
body += chunk.toString('utf8');
});
req.on('end', function (chunk) {
req.body = JSON.parse(body)
next();
});
});
app2.all('*', function (req, res) {
res.send(req.body)
});
http.createServer(app2).listen(8001);
Using the request library in my application, it worked:
var request = require('request')
request.post({
url: 'http://localhost:8000',
json: {content: 123, type: "greeting", access_token: "here i am"}
},function(err, res,data){
console.log('return:' ,err, data)
});
But using curl with a file containing the same message, it would not work:
curl -X POST -d #data.json http://localhost:8000 --header "Content-Type:application/json"
I compared the request objects against each other and found a few differences and when I got to the request header for content-length, I found that editing it the "correct" length would end the steam (and the web server would send a response).
I will make the modifications needed and commit to connect-restreamer module.

Resources