My goal is to accept an uploaded file and stream it to Wistia using the the Wistia Upload API. I need to be able to add fields to the HTTP request, and I don't want the file to touch the disk. I'm using Node, Express, Request, and Busboy.
The code below has two console.log statements. The first returns [Error: not implemented] and the second returns [Error: form-data: not implemented]. I'm new to streaming in Node, so I'm probably doing something fundamentally wrong. Any help would be much appreciated.
app.use("/upload", function(req, res, next) {
var writeStream = new stream.Writable();
writeStream.on("error", function(error) {
console.log(error);
});
var busboy = new Busboy({headers: req.headers});
busboy.on("file", function(fieldname, file, filename, encoding, mimetype) {
file.on("data", function(data) {
writeStream.write(data);
});
file.on("end", function() {
request.post({
url: "https://upload.wistia.com",
formData: {
api_password: "abc123",
file: new stream.Readable(writeStream)
}
}, function(error, response, body) {
console.log(error);
});
});
});
req.pipe(busboy);
});
I am not to familiar with the busboy module, but there errors you are getting are from attempting to use un-implemented streams. Whenever you create a new readable or writable stream directly from the stream module you have to create the _read and _write methods respectively Stream Implementors (node.js api). To give you something to work with the following example is using multer for handling multipart requests, I think you'll find multer is easier to use than busboy.
var app = require('express')();
var fs = require('fs');
var request = require('request');
app.use(multer());
app.post("/upload", function(req, res, next) {
// create a read stream
var readable = fs.createReadStream(req.files.myfile.path);
request.post({
url: "https://upload.wistia.com",
formData: {
api_password: "abc123",
file: readable
}
}, function(err, res, body) {
// send something to client
})
});
I hope this helps unfortunately I am not familiar with busboy, but this should work with multer, and as I said before there problem is just that you are using un-implemented streams I'm sure there is a way to configure this operation with busboy if you wanted.
If you want to use multipart (another npm) here is a tutorial:
http://qnimate.com/stream-file-uploads-to-storage-server-in-node-js/
Related
I am making a multer storage engine which makes stream connection between client and S3 Server.
At middle of the stream, my code examine chunks and send it to S3.
I could get a file stream from node.js server. But when I request file array upload, node inspector shows only one stream. What should I do?
Stream engine snippet
CustomStreamEngine.prototype._handleFile = function _handleFile (req, file, cb) {
// for inspect
req.files.length // 1
file;
};
request controller
var streamStorage = multer({
storage: streamEngine()
});
dev.post('/rec_test', streamStorage.array('source'), (req, res, next) => {
});
I just published this streaming multipart/form-data parser on npm as form-parser:
You should be able to do the following:
dev.post('/rec_test', async (req, res, next) => {
// Parse request
await parser(req, async ({ fieldType, fieldName, fieldContent }) => {
// Log all fields
console.log({ fieldType, fieldName, fieldContent });
// Handle 'source' file fields
if (fieldType === 'file' && fieldName === 'source[]') {
// Get file info
const { fileName, fileType, fileStream } = fieldContent;
// Upload fileStream to S3 :-)
}
});
});
Hope it's helpful.
K
I think you can add some logs to https://github.com/expressjs/multer/blob/master/lib/make-middleware.js to check.
Currently, I use axios on the client to send multi files to the server with multer. And I can see all files in the function
busboy.on('file', function (fieldname, fileStream, filename, encoding, mimetype), but there is only one file at a time, and this function will call the _handfile function of the custom storage, so that I think it is the reason for your issue.
Hope it can help you
Is there a way that using express a route consumer can send an input stream to the endpoint and read it?
In short I want the endpoint user upload a file by streaming it instead of the multipart/form way. Something like:
app.post('/videos/upload', (request, response) => {
const stream = request.getInputStream();
const file = stream.read();
stream.on('done', (file) => {
//do something with the file
});
});
Is it possible to do it?
In Express, the request object is an enhanced version of http.IncomingMessage, which "...implements the Readable Stream interface".
In other words, request is already a stream:
app.post('/videos/upload', (request, response) => {
request.on('data', data => {
...do something...
}).on('close', () => {
...do something else...
});
});
If your intention is to first read the entire file into memory (probably not), you can also use bodyParser.raw():
const bodyParser = require('body-parser');
...
app.post('/videos/upload', bodyParser.raw({ type : '*/*' }), (request, response) => {
let data = req.body; // a `Buffer` containing the entire uploaded data
...do something...
});
I'm new to node.js. What I'm trying to do is to stream the upload of a file from web browser to a cloud storage through my node.js server.
I'm using 'express', 'request' and 'busboy' modules.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
"upload": file
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
But I keep getting the following error on the server. What am I doing wrong here? Any help is appreciated.
Error: Part terminated early due to unexpected end of multipart data
at node_modules\busboy\node_modules\dicer\lib\Dicer.js:65:36
at nextTickCallbackWith0Args (node.js:420:9)
at process._tickCallback (node.js:349:13)
I realize this question is some 7 months old, but I shall answer it here in an attempt help anyone else currently banging their head against this.
You have two options, really: Add the file size, or use something other than Request.
Note: I edited this shortly after first posting it to hopefully provide a bit more context.
Using Something Else
There are some alternatives you can use instead of Request if you don't need all the baked in features it has.
form-data can be used by itself in simple cases, or it can be used with, say, got. request uses this internally.
bhttp advertises Streams2+ support, although in my experience Streams2+ support has not been an issue for me. No built in https support, you have to specify a custom agent
got another slimmed down one. Doesn't have any special handling of form data like request does, but is trivially used with form-data or form-data2. I had trouble getting it working over a corporate proxy, though, but that's likely because I'm a networking newb.
needle seems pretty light weight, but I haven't actually tried it.
Using Request: Add the File Size
Request does not (as of writing) have any support for using transfer-encoding: chunked so to upload files with it, you need to add the file's size along with the file, which if you're uploading from a web client means that client needs to send that file size to your server in addition to the file itself.
The way I came up with to do this is to send the file metadata in its own field before the file field.
I modified your example with comments describing what I did. Note that I did not include any validation of the data received, but I recommend you do add that.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
// Use this to cache any fields which are file metadata.
var fileMetas = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
// Be sure to match this prop name here with the pattern you use to detect meta fields.
var meta = fileMetas[fieldname + '.meta'];
if (!meta) {
// Make sure to dump the file.
file.resume();
// Then, do some sort of error handling here, because you cannot upload a file
// without knowing it's length.
return;
}
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
// value + options form of a formData field.
"upload": {
value: file,
options: {
filename: meta.name,
knownLength: meta.size
}
}
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
// Use whatever pattern you want. I used (fileFieldName + ".meta").
// Another good one might be ("meta:" + fileFieldName).
if (/\.meta$/.test(name)) {
// I send an object with { name, size, type, lastModified },
// which are just the public props pulled off a File object.
// Note: Should probably add error handling if val is somehow not parsable.
fileMetas[name] = JSON.parse(val);
console.log(`file metadata: name: ${name}, value: ${value}`);
return;
}
// Otherwise, process field as normal.
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
On the client, you need to then send the metadata on the so-named field before the file itself. This can be done by ordering an <input type="hidden"> control before the file and updating its value onchange. The order of values sent is guaranteed to follow the order of inputs in appearance. If you're building the request body yourself using FormData, you can do this by appending the appropriate metadata before appending the File.
Example with <form>
<script>
function extractFileMeta(file) {
return JSON.stringify({
size: file.size,
name: file.name,
type: file.type,
lastUpdated: file.lastUpdated
});
}
function onFileUploadChange(event) {
// change this to use arrays if using the multiple attribute on the file input.
var file = event.target.files[0];
var fileMetaInput = document.querySelector('input[name=fileUpload.meta]');
if (fileMetaInput) {
fileMetaInput.value = extractFileMeta(file);
}
}
</script>
<form action="/upload-to-cloud">
<input type="hidden" name="fileUpload.meta">
<input type="file" name="fileUpload" onchange="onFileUploadChange(event)">
</form>
Example with FormData:
function onSubmit(event) {
event.preventDefault();
var form = document.getElementById('my-upload-form');
var formData = new FormData();
var fileUpload = form.elements['fileUpload'];
var fileUploadMeta = JSON.stringify({
size: fileUpload.size,
name: fileUpload.name,
type: fileUpload.type,
lastUpdated: fileUpload.lastUpdated
});
// Append fileUploadMeta BEFORE fileUpload.
formData.append('fileUpload.meta', fileUploadMeta);
formData.append('fileUpload', fileUpload);
// Do whatever you do to POST here.
}
I am trying to upload a file to a server (built using Java) by reading from a mongodb gridfs stream.
exports.upload = function(req, res, next) {
var IHUrl = config.api.url + "PhotosServlet";
var data = req.body;
var file1 = api.gfs.createReadStream({
_id: data.fileId
})
var formData = {
"imgTyp": data.imgTyp,
"listingid": data.listingid,
"scaleTech": data.scaleTech,
"SPC": data.SPC,
"SPCUID": data.SPCUID,
"varRand": data.varRand,
"file1": file1
};
var r = request.post({
url: IHUrl,
formData: formData
}, function(error, IHResponse, body) {
if (error) {
res.send(500, error);
console.log("Error occured uploading file1")
} else {
console.log("Upload successful", IHResponse);
res.send(200, IHResponse);
}
});
next();
};
File is already uploaded in mongodb gridfs before I start uploading the file to upstream server.
I am using following nodejs libraries:
request, gridfs-stream
I am getting the following error from upstream server:
javax.servlet.ServletException: Processing of multipart/form-data request failed. Stream ended unexpectedly
What could be going wrong here?
I realized that its been a while and its a problem some of you may encounter and the solution was not really what I posted in the comments as I found more problems later. The issue that we had was that the java servlet that we were posting the multipart form data was not able to handle chunked data and setting headers won't do you any good. using request library will not help you here. I had to make use of restler (https://www.npmjs.com/package/restler) to send the entire multipart data in a single chunk.
I'm trying to POST a raw body with restify. I have the receive side correct, when using POSTman I can send a raw zip file, and the file is correctly created on the server's file system. However, I'm struggling to write my test in mocha. Here is the code I have, any help would be greatly appreciated.
I've tried this approach.
const should = require('should');
const restify = require('restify');
const fs = require('fs');
const port = 8080;
const url = 'http://localhost:' + port;
const client = restify.createJsonClient({
url: url,
version: '~1.0'
});
const testPath = 'test/assets/test.zip';
fs.existsSync(testPath).should.equal(true);
const readStream = fs.createReadStream(testPath);
client.post('/v1/deploy', readStream, function(err, req, res, data) {
if (err) {
throw new Error(err);
}
should(res).not.null();
should(res.statusCode).not.null();
should(res.statusCode).not.undefined();
res.statusCode.should.equal(200);
should(data).not.null();
should(data.endpoint).not.undefined();
data.endpoint.should.equal('http://endpointyouhit:8080');
done();
});
Yet the file size on the file system is always 0. I'm not using my readStream correctly, but I'm not sure how to correct it. Any help would be greatly appreciated.
Note that I want to stream the file, not load it in memory on transmit and receive, the file can potentially be too large for an in memory operation.
Thanks,
Todd
One thing is that you would need to specify a content-type of multi-part/form-data. However, it looks like restify doesn't support that content type, so you're probably out of luck using the restify client to post a file.
To answer my own question, it doesn't appear to be possible to do this with the restify client. I also tried the request module, which claims to have this capability. However, when using their streaming examples, I always had a file size of 0 on the server. Below is a functional mocha integration test.
const testPath = 'test/assets/test.zip';
fs.existsSync(testPath).should.equal(true);
const readStream = fs.createReadStream(testPath);
var options = {
host: 'localhost'
, port: port
, path: '/v1/deploy/testvalue'
, method: 'PUT'
};
var req = http.request(options, function (res) {
//this feels a bit backwards, but these are evaluated AFTER the read stream has closed
var buffer = '';
//pipe body to a buffer
res.on('data', function(data){
buffer+= data;
});
res.on('end', function () {
should(res).not.null();
should(res.statusCode).not.null();
should(res.statusCode).not.undefined();
res.statusCode.should.equal(200);
const json = JSON.parse(buffer);
should(json).not.null();
should(json.endpoint).not.undefined();
json.endpoint.should.equal('http://endpointyouhit:8080');
done();
});
});
req.on('error', function (err) {
if (err) {
throw new Error(err);
}
});
//pipe the readstream into the request
readStream.pipe(req);
/**
* Close the request on the close of the read stream
*/
readStream.on('close', function () {
req.end();
console.log('I finished.');
});
//note that if we end up with larger files, we may want to support the continue, much as S3 does
//https://nodejs.org/api/http.html#http_event_continue