Got a server that sends huge files ~50MB as response.
Using http and piping the file as follows:
var http = require('http'),
fileSystem = require('fs'),
path = require('path');
http.createServer(function(request, response) {
var filePath = path.join(__dirname, 'myfile.mp3');
var stat = fileSystem.statSync(filePath);
response.writeHead(200, {
'Content-Type': 'audio/mpeg',
'Content-Length': stat.size
});
var readStream = fileSystem.createReadStream(filePath);
// We replaced all the event handlers with a simple call to readStream.pipe()
readStream.pipe(response);
})
.listen(2000);
As expected, it takes a huge amount of time to send the response. But I want to have a responsive page when the request is being made.
Anyway I can send the progress of the response that is sent have it show at the frontend?
Can use I socket.io?
What I have tried so far:
Tried calling the pipe event on the readStream but it was called only once.
.on("pipe", (src) => {
console.log("Something is piping into the writer.");
// assert.equal(src, reader);
})
Related
Got to build a server from scratch that takes a audio file and sends it over a stream to multiple end-users. I have looked into streams and buffers and have some basic idea about sending text data via buffers to client-side but in my scenario, I also need to start playing the audio when the buffer is received at the user end.
Figured out the way to send audio as streams:
var http = require('http'),
fileSystem = require('fs'),
path = require('path'),
util = require('util');
http.createServer(function(request, response) {
var filePath = 'read.mp3';
var stat = fileSystem.statSync(filePath);
response.writeHead(200, {
'Content-Type': 'audio/mpeg',
'Content-Length': stat.size
});
var readStream = fileSystem.createReadStream(filePath,{ highWaterMark: 1 * 16 });
readStream.on('data', (chunk) => {
response.write(chunk);
});
console.log("Serving");
})
.listen(2000);
I am new to node js.
I wrote a server using node js which will read the request data and save that data into an image file.(Because the data coming is image data itself.)
The node js script I wrote is :
const http = require('http');
const fs = require('fs');
const server = http.createServer();
server.on('request', function(request, respond) {
var body = '';
filePath = '1.jpg';
request.on('data', function(data) {
body += data;
});
request.on('end', function (){
fs.appendFile(filePath, body, function() {
respond.end();
});
});
});
server.listen(8080);
And from the terminal on same machine , I fired a curl command to send the image :
curl -X POST --data #tmp.jpg 127.0.0.1:8080
The tmp.jpg is opening perfectly on my machine.
But 1.jpg (created by node js) is not opening.
What can be the problem ?
Any help would be highly appreciated.
The issue lies with your usage of curl. To send binary data, use the --data-binary flag instead:
curl -X POST --data-binary #image.jpg 127.0.0.1:8080
Also, you need to use writeFile instead of appendFile. The latter will add each request's data to the same file, which wil not be readable as image.
Furthermore, when processing binary data I find it easier to use buffers instead of string. The request processing would look like this:
server.on('request', function(req, respond) {
filePath = '1.jpg';
var chunks = []
req.on('data', d => {
chunks.push(d)
})
req.on('end', function (){
var data = Buffer.concat(chunks)
fs.writeFile(filePath, data, function() {
respond.end();
});
});
});
UPDATE
Thanks to #robertklep and #vallo for pointing out that I was not parsing the multipart request properly.
Here's the updated server code with some re-worked sample code from Busboy:
'use strict';
// Require
var http = require('http');
var Busboy = require('busboy');
var fs = require('fs');
// Server
var server = http.createServer(function(request, response) {
if (request.method === 'POST') {
var busboy = new Busboy({ headers: request.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
file.pipe(fs.createWriteStream(`../db/images/${filename}`));
});
busboy.on('finish', function() {
response.writeHead(200, { 'Connection': 'close' });
response.end("That's all folks!");
});
return request.pipe(busboy);
}
response.writeHead(404);
response.end();
});
server.listen(8000, '192.168.7.25', () => {});
I am trying to post a jpg to an endpoint but the resulting image cannot be opened :
The file “image_copy.jpg” could not be opened. It may be damaged or
use a file format that Preview doesn’t recognize.
Some background:
Everything (servers, storage) are being hosted locally
Have made a decision to only use native Node modules like http and fs due to storage constraints on a microcontroller board
Am using form-data as it eases the pain of multi-part forms and uploads as well as sets the correct request headers
Here is some sample code broken into two scripts:
Server
'use strict';
// Require
var http = require('http');
// Server
var server = http.createServer((request, response) => {
var body = [];
request.on('data', function(chunk) {
body.push(chunk);
});
request.on('end', function() {
saveImage(Buffer.concat(body),null);
response.statusCode = 200;
response.end('thanks')
});
});
server.listen(8000, '192.168.7.25', () => {});
// Process
function saveImage(data,callback) {
var fs = require('fs');
fs.writeFile('../db/images/image_copy.jpg', data, function(err) {});
}
Client
'use strict';
// Require
var FormData = require('form-data');
var fs = require('fs');
var http = require('http');
// Vars
var form = new FormData();
// Process
form.append('my_file', fs.createReadStream('/temp/1.jpg'));
var request = http.request({
hostname: '192.168.7.25',
port: 8000,
path: '/api/premises/v1/image',
method: 'POST',
headers: form.getHeaders()
});
form.pipe(request);
request.on('response', function(res) {
console.log(res.statusCode);
});
After executing, the jpg is uploaded and saved to the correct file location (and also has the same file size as the source jpg) but the new image can't be opened.
Even if I encode the incoming chunk as binary and set the encoding on fs.writeFile to binary, I get a similar result.
What am I doing wrong? Thank you!
The client is uploading in multipart/form-data format, which is a format that can contain, amongst others, file data.
However, this means that the server should parse this format to extract the file data. Right now, the server is just taking the request body verbatim and writing it to a file.
The multiparty module can help you, and one of its usage examples shows you how to hook it up with http.Server: https://github.com/pillarjs/multiparty#usage
var multiparty = require('multiparty');
var http = require('http');
var util = require('util');
http.createServer(function(req, res) {
// parse a file upload
var form = new multiparty.Form();
form.parse(req, function(err, fields, files) {
res.writeHead(200, {'content-type': 'text/plain'});
res.write('received upload:\n\n');
res.end(util.inspect({fields: fields, files: files}));
});
}).listen(8000);
Using that, you can extract the file data from (I think) files.my_file and write it to the file.
To show my students a simple HTTP request and response that they could capture using Wireshark, I whipped up a simple Node.js HTTP server:
var fs = require('fs');
var http = require('http');
var port = 80;
var file = process.argv[2]; //This file contains a 42 byte HTML page
http.createServer(function (req, res) {
res.writeHead(200, { 'content-type' : 'text/html' }); // Sends first packet
fs.createReadStream(file).pipe(res); // Sends second packet
}).listen(port);
Unfortunately, the two lines transmitting the HTTP header and the HTML are sent as two separate TCP packets (even though they are both quite small). It would be simpler for my students if the HTTP header and HTML were just one packet. How could I change my code to do this?
var http = require('http');
var fs = require('fs');
var file = process.argv[2];
http.createServer(function(request, response) {
response.writeHeader(200, {"Content-Type": "text/html;"});
fs.readFile(file, function (err, html) {
if (err) {
throw err;
}
response.write(html);
response.end();
});
}).listen(8000);
the reason it won't work is that Node.js runs everything asynchronously. When you are loading your html file, the server creation starts the same time. By the time you are about to write your html to your tcp socket, the file most likely won't be ready.
I see what you were trying to do before... I misread your code because of the indentation. Let me know if this snippet works.
try using something like-
var file = process.argv[2];
fs.readFile(file, function (err, html) {
if (err) {
throw err;
}
http.createServer(function(request, response) {
response.writeHeader(200, {"Content-Type": "text/html"});
response.write(html);
response.end();
}).listen(8000);
});
I need to stream files from a client (nodejs command line) and a server (express nodejs).
This is the client side:
var request = require('request');
var fs = require('fs');
// ...
var readStream = fs.createReadStream(file.path);
readStream.on('end', function() {
that.emit('finished');
});
readStream.pipe(request.post(target));
// ...
This is the server side:
var fs = require('fs');
var path = require('path');
// ...
app.post('/:filename', function(req, res) {
req.setEncoding('binary');
var filename = path.basename(req.params.filename);
filename = path.resolve(destinationDir, filename);
var dst = fs.createWriteStream(filename);
req.pipe(dst);
req.on('end', function() {
res.send(200);
});
});
// ...
All is working, files are saved correctly on the server side... but they are about 50% bigger than the source files. I tried to see difference between the two files with hexdump and the server side file has similar content but with 0xC2 sometimes. I guess this is related to encoding.
Don't call req.setEncoding('binary').
This will convert every single chunk into strings and is mainly intended if you want to read strings from the stream. As you directly pipe the request to a file, you don't need to do it.