node.js: browser image caching with correct headers - node.js

I'm developing a web application that manages a large amount of images, stores and resizes them.
the request of an image is something like:
domain:port/image_id/size
The server takes the image_id and if there isn't yet an image of such size it creates it and stores it on filesystem.
So everything is ok and the server is running but I need to cache those images in browser for at least one day to reduce the server bandwidth consumption.
I did several tests but nothing seems to work.
Here is the code I use to make the response header:
response.writeHead(304, {
"Pragma": "public",
"Cache-Control": "max-age=86400",
"Expires": new Date(Date.now() + 86400000).toUTCString(),
"Content-Type": contentType});
response.write(data);
response.end();
I also tried with response status 200.
contentType is always a mime type like "image/jpg" or "image/png"
data is the bytes buffer of the image.
Any advice?
Thanks a lot.
live long and prosper,
d.

I did a lot of tests and I came out with a solution that seems pretty good to manage this caching problem.
Basically what I do is getting the request and check for the request header named "if-modified-since".
If I find it and the value (it is a date) is the same as the modified date of the file, the response will be a 304 status with no content.
If I don't find this value or it's different from the modified date of the file, I send the complete response with status 200 and the header parameter for further access by the browser.
Here is the complete code of the working test I did:
with "working" I mean that the first request get the file from the server while the next requests get a 304 response and don't send content to the browser, that load it from local cache.
var http = require("http");
var url = require("url");
var fs = require('fs');
function onRequest(request, response) {
var pathName = url.parse(request.url).pathname;
if (pathName!="/favicon.ico") {
responseAction(pathName, request, response);
} else {
response.end();
}
}
function responseAction(pathName, request, response) {
console.log(pathName);
//Get the image from filesystem
var img = fs.readFileSync("/var/www/radar.jpg");
//Get some info about the file
var stats = fs.statSync("/var/www/radar.jpg");
var mtime = stats.mtime;
var size = stats.size;
//Get the if-modified-since header from the request
var reqModDate = request.headers["if-modified-since"];
//check if if-modified-since header is the same as the mtime of the file
if (reqModDate!=null) {
reqModDate = new Date(reqModDate);
if(reqModDate.getTime()==mtime.getTime()) {
//Yes: then send a 304 header without image data (will be loaded by cache)
console.log("load from cache");
response.writeHead(304, {
"Last-Modified": mtime.toUTCString()
});
response.end();
return true;
}
} else {
//NO: then send the headers and the image
console.log("no cache");
response.writeHead(200, {
"Content-Type": "image/jpg",
"Last-Modified": mtime.toUTCString(),
"Content-Length": size
});
response.write(img);
response.end();
return true;
}
//IF WE ARE HERE, THERE IS A PROBLEM...
response.writeHead(200, {
"Content-Type": "text/plain",
});
response.write("ERROR");
response.end();
return false;
}
http.createServer(onRequest).listen(8889);
console.log("Server has started.");
Of course, I don't want to reinvent the wheel, this is a benchmark for a more complex server previously developed in php and this script is a sort of "porting" of this PHP code:
http://us.php.net/manual/en/function.header.php#61903
I hope this will help!
Please, if you find any errors or anything that could be improved let me know!
Thanks a lot,
Daniele

Related

SSE events are not reaching client until res.end() is called

The same code used to work before but i dont know why it is not working now.Please help.
My problem is that when i use SSE for real time data sharing.The data object which should be sent on res.write(data:${JSON.stringify(dataObject)}\n\n) is not being sent until res.end() is called and all the data is event streams are sent at once.
response.writeHead(200, {
Connection: "keep-alive",
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
"Access-Control-Allow-Origin": '*',
'Content-Encoding':'identity' // this with and without this
});
let syncher= setInterval(() => {
if(response.finished){ // If response ended the interval is cleared
clearInterval(syncher);
return;
}else{
let dataToSend = getEventData(user,event);
if(! dataToSend ){
response.write('data:{close:true}');
clearInterval(syncher);
return;
}
response.write(`data:${JSON.stringify(dataToSend)}\n\n`);
response.flushHeaders(); // Also tried with response.flush()
if(dataToSend.close){
delEventData(user,event);
response.end();
}
}
}, 500);
The above code is in server side this also has on close listener to close the connection
const ev = new EventSource(conf.apiUrl+'/getStatus/'+ (userData.id || '') );
let data = '';
ev.onmessage = eventData=>{
data = JSON.parse(eventData.data);
if(!data){
setState('progress '+data.completedSoFar)
return;
}
if(!data.close){
}else{
if(data.success){
console.log('Done Successfully')
ev.close();
}
}
This is my client side code
I don't know why the event listener is not getting data stream while i searched the internet about this issue i only found that when compression middleware is used this issue occurs .I don't use any compression middleware in my app. I am using nodejs v11.4.0. I am guessing that when i am making eventsource request chrome is adding gzip encoding by default and node is using that to set response encoding header as gzip I tried to delete and replace it but did'nt work which is causing this issue??
Here is my request and response headers for eventSource request
Sorry for my grammar if i made any mistakes.
Thanks for help. Cheers!
Nextjs is basically compressing your data to make it transmit faster. Unfortunately, this makes it so we can't see the data until we flush the cache (my guess is render behavior has changed because compression has changed the content). You can disable compression entirely by including compress: false in your next.config.
I found here that including the following header sidesteps the compression for a specific endpoint:
res.setHeader("Cache-Control", "no-cache, no-transform");
Caution: This will increase bandwidth/resource usage! HTTP compression can reduce the size of your data by 70%.
After lot of debugging and research. The problem turned to be in the webpack-dev-server which compressed my responses. For more info refer https://github.com/facebook/create-react-app/issues/966

NodeJS http.request fails when header is too large

My code is as follows
var http = require('http');
var host=...
var postData=({
//some fun stuff
})
var postOptions ={
host: host,
path: '/api/dostuff',
method: 'POST',
headers:{
AppID:"some stuff",
Authorization: "OAuth token",
"Content-Type":"application/json"
},
};
var req = http.request(postOptions, function(res){
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
//sanitize data stuff here
console.log("DATA HERE: "+ data);
return data;
});
});
req.write(JSON.stringify(postData));
req.end();
It's a basic HTTP post to a C# server. The important stuff is in the headers. I send the app ID (which is ~50 characters) and the OAuth token (which can be thousands of characters). Right now, the server isn't set up to do anything with the Authorization header. It doesn't even check if its there.
My problem is that when I populate the Authorization header (or any header) with a few random characters as a test, the post succeeds. When I tried it again with a full valid Authorization token (which, to reiterate, is very long) it fails. No matter which part of the header i fill, once it gets too full it returns an error. The error I receive is "Processing of the HTTP request resulted in an exception. Please see the HTTP response returned by the 'Response' property of this exception for details". I was somewhat certain this is a server issue, but when I tried running the exact same body and headers in Postman, I got a valid response.
Does anyone have any idea what is causing this?
There's a compiled constant that's defined to be 80k for Node HTTP headers. Are you running into that? I'd recommend seeing how big the header is with your OAuth token. It shouldn't exceed 80k though, and FWIW, even a kilobyte is huge for OAuth... But regardless... Try dumping the size of the headers (in bytes).

Response not ending and browser keeps loading - nodejs and graphicsmagick

I am new to nodejs. I am using graphicsmagick to resize the image before sending it to the browser.
My code looks like this (res is the response to be sent from function(req,res){...}) -
imageURLPromise
.then(function(response) {
var body = response.body;
if (body) {
console.log("Found From: " + response.request.uri.href);
//set response headers here
setResponseData(res, response);
var buf = new Buffer(body, "binary");
console.log(buf);
//gm module
gm(buf).resize(400,300).toBuffer(function(err,buffer) {
console.log("buffer here");
res.end(buffer, "binary");
});
}
}, function(error) {
console.log(error);
});
I get the image in the browser, I get the log "buffer here" but the browser stays in the "loading" state.
I have tried using .stream() with gm and pipe the stdout to response but it has the same problem.
If I do away with gm and directly write body to the response like this
res.end(body, 'binary');
then it works correctly.
Can someone tell what I am doing wrong here?
I figured out the problem.
The problem was not with node or gm but with the HTTP response headers.
When GM returns a buffer and we write that to the HTTP response then it sets the Transfer-Encoding header to "chunked". In that case the Content-Length header should never be set.
You can read more about it here
http://en.wikipedia.org/wiki/Chunked_transfer_encoding
Since I was setting both the browser kept waiting for content even after the image had been sent.
The code is exactly the same as I posted, except for the fact that in the setResponseData() function (which basically used to set headers) I am not setting the content-length header now.

check on server side if youtube video exist

How to check if youtube video exists on node.js app server side:
var youtubeId = "adase268_";
// pseudo code
youtubeVideoExist = function (youtubeId){
return true; // if youtube video exists
}
You don't need to use the youtube API per-se, you can look for the thumbnail image:
Valid video = 200 - OK:
http://img.youtube.com/vi/gC4j-V585Ug/0.jpg
Invalid video = 404 - Not found:
http://img.youtube.com/vi/gC4j-V58xxx/0.jpg
I thought I could make this work from the browser since you can load images from a third-party site without security problems. But testing it, it's failing to report the 404 as an error, probably because the content body is still a valid image. Since you're using node, you should be able to look at the HTTP response code directly.
I can't think of an approach that doesn't involve making a separate HTTP request to the video link to see if it exists or not unless you know beforehand of a set of video IDs that are inactive,dead, or wrong.
Here's an example of something that might work for you. I can't readily tell if you're using this as a standalone script or as part of a web server. The example below assumes the latter, assuming you call a web server on /video?123videoId and have it respond or do something depending on whether or not the video with that ID exists. It uses Node's request library, which you can install with npm install request:
var request = require('request');
// Your route here. Example on what route may look like if called on /video?id=123videoId
app.get('/video', function(req, response, callback){
var videoId = 'adase268_'; // Could change to something like request.params['id']
request.get('https://www.youtube.com/watch?v='+videoId, function(error, response, body){
if(response.statusCode === 404){
// Video doesn't exist. Do what you need to do here.
}
else{
// Video exists.
// Can handle other HTTP response codes here if you like.
}
});
});
// You could refactor the above to take out the 'request.get()', wrap it in a function
// that takes a callback and re-use in multiple routes, depending on your problem.
#rodrigomartell is on the right track, in that your check function will need to make an HTTP call; however, just checking the youtube.com URL won't work in most cases. You'll get back a 404 if the videoID is a malformed ID (i.e. less than 11 characters or using characters not valid in their scheme), but if it's a properly formed videoID that just happens to not correspond to a video, you'll still get back a 200. It would be better to use an API request, like this (note that it might be easier to use the request-json library instead of just the request library):
request = require('request-json');
var client = request.newClient('https://www.googleapis.com/youtube/v3/');
youtubeVideoExist = function (youtubeId){
var apikey ='YOUR_API_KEY'; // register for a javascript API key at the Google Developer's Console ... https://console.developers.google.com/
client.get('videos/?part=id&id='+youtubeId+'&key='+apikey, function(err, res, body) {
if (body.items.length) {
return true; // if youtube video exists
}
else {
return false;
}
});
};
Using youtube-feeds module. Works fast (~200ms) and no need API_KEY
youtube = require("youtube-feeds");
existsFunc = function(youtubeId, callback) {
youtube.video(youtubeId, function(err, result) {
var exists;
exists = result.id === youtubeId;
console.log("youtubeId");
console.log(youtubeId);
console.log("exists");
console.log(exists);
callback (exists);
});
};
var notExistentYoutubeId = "y0srjasdkfjcKC4eY"
existsFunc (notExistentYoutubeId, console.log)
var existentYoutubeId = "y0srjcKC4eY"
existsFunc (existentYoutubeId, console.log)
output:
❯ node /pathToFileWithCodeAbove/FileWithCodeAbove.js
youtubeId
y0srjcKC4eY
exists
true
true
youtubeId
y0srjasdkfjcKC4eY
exists
false
false
All you need is to look for the thumbnail image. In NodeJS it would be something like
var http = require('http');
function isValidYoutubeID(youtubeID) {
var options = {
method: 'HEAD',
host: 'img.youtube.com',
path: '/vi/' + youtubeID + '/0.jpg'
};
var req = http.request(options, function(res) {
if (res.statusCode == 200){
console.log("Valid Youtube ID");
} else {
console.log("Invalid Youtube ID");
}
});
req.end();
}
API_KEY is not needed. It is quite fast because there is only header check for statusCode 200/404 and image is not loaded.

Node.js HTTP GET request content type

I am trying to do a GET request from node.js. This request is to a REST server which will access Hbase and return the data. The GET request contains all necessary Hbase info (table, key, column-family etc.) in it. Following is the code.
var http = require('http');
var url = {
host: '127.0.0.1',
port: 8000,
path: '/table-name/key/column-family',
headers: {
'Content-Type': 'application/octet-stream'
},
};
http.get(url, function(resp){
console.log("Status: " + resp.statusCode);
console.log("Header: " + JSON.stringify(resp.headers));
resp.setEncoding('utf8');
var completeResponse = '';
resp.on('data', function (chunk) {
completeResponse += chunk;
});
resp.on('end', function(chunk) {
console.log(completeResponse);
});
});
My problem is that the response I get is not always an octet-stream as requested. Most of the time data is in valid format with a header like the following.
{"content-length":"454","x-timestamp":"1395469504346","content-type":"application/octet-stream"}
But, say 1 out of 10 times the response is an XML string with a header like following.
{"content-type":"text/xml","content-length":"793"}
The status code is 200 in both cases and I am always requesting for an existing key. This behavior is seemingly random and not caused by any particular key.
How do I ensure that the response is always an octet-stream and not XML / JSON?
As stated in the comments, you need to set the Accept header to specify the content type you expect (accept) in response from the server. Note that you may accept more than one type of response.
The Content-Type header specify the type of what's in the body of the message. It can be set by your client in case of POST/PATCH request, or by the server in its response. On the receiving side, it is used to know how to handle the body content.
For more detail, you can refer to the comprehensive MDN content negotiation documentation

Resources