I am trying to upload a file to a server (built using Java) by reading from a mongodb gridfs stream.
exports.upload = function(req, res, next) {
var IHUrl = config.api.url + "PhotosServlet";
var data = req.body;
var file1 = api.gfs.createReadStream({
_id: data.fileId
})
var formData = {
"imgTyp": data.imgTyp,
"listingid": data.listingid,
"scaleTech": data.scaleTech,
"SPC": data.SPC,
"SPCUID": data.SPCUID,
"varRand": data.varRand,
"file1": file1
};
var r = request.post({
url: IHUrl,
formData: formData
}, function(error, IHResponse, body) {
if (error) {
res.send(500, error);
console.log("Error occured uploading file1")
} else {
console.log("Upload successful", IHResponse);
res.send(200, IHResponse);
}
});
next();
};
File is already uploaded in mongodb gridfs before I start uploading the file to upstream server.
I am using following nodejs libraries:
request, gridfs-stream
I am getting the following error from upstream server:
javax.servlet.ServletException: Processing of multipart/form-data request failed. Stream ended unexpectedly
What could be going wrong here?
I realized that its been a while and its a problem some of you may encounter and the solution was not really what I posted in the comments as I found more problems later. The issue that we had was that the java servlet that we were posting the multipart form data was not able to handle chunked data and setting headers won't do you any good. using request library will not help you here. I had to make use of restler (https://www.npmjs.com/package/restler) to send the entire multipart data in a single chunk.
Related
I have an API in one nodejs project as below which receive multiple attachment from UI:
const upload = multer()
router.post('/upload', upload.array("attachments"),controller.getSomething);
getSomething is supposed to call another POST API using Axios only, which is in another NodeJs project which accept these attachments and process it. It as well accept multiple files via multer.
I am unsure how could i send multiple files as a request from one Nodejs project to another at once. could you please favour.
I had to set formdata as below:
const formData=new FormData();
for(let file of req.files){
formData.append("attachments",file.buffer,file.originalname);
}
And passed the formdata to other api via axios.
You can do the following steps:
When you upload the temporary files (coming from UI), save them in the temporary folder.
Pass all the files names to the POST API using Axios.
In the post API, read all the files from the temporary folder and stream them to the destination.
controller.getSomething = (req, res, next) => {
// get the file names
const fileNames = req.files.map(filename => filename);
// now post this filenames to the
axios.post('/pathname', {fileNames})
// or, you can use get request
}
Reading files in the post Request:
var promises= ['file1.css', 'file2.css'].map(function(_path){
return new Promise(function(_path, resolve, reject){
fs.readFile(_path, 'utf8', function(err, data){
if(err){
console.log(err);
resolve(""); //following the same code flow
}else{
resolve(data);
}
});
}.bind(this, _path));
});
Promise.all(promises).then(function(results){
//Put your callback logic here
response.writeHead(200, {"Content-Type": "text/css"});
results.forEach(function(content){response.write(content)});
response.end();
});
#copied from this link. You should check the different answers that can help you.
On my node express server, I am receiving a pdf file. I am using the below code to get the pdf contents from the request
var data = new Buffer('');
request.on('data', function (chunk) {
data = Buffer.concat([data, chunk]);
});
request.on('end', function() {
console.log('PDF data is '+JSON.stringify(data));
});
Now that PDF content is available on node, I need to send it as it is to a J2EE server. In order to do that, I am first saving the PDF file in the node server, reading it from the node server and then piping it to request.post (https://github.com/request/request)
var req = require('request');
fs.writeFile('abc.pdf', data, 'binary', function(err) {
if (err) {
console.log('Error ' + JSON.stringify(err) );
throw err;
}
var source = fs.createReadStream('abc.pdf');
//send our data via POST request
source.pipe(req.post('http://'+j2ee_host+':'+j2ee_port+'/myjavaapp/Upload')
});
This works fine. However, I feel the part of saving the PDF file on the node server and then reading it is (before posting to the J2EE server using request module) is completely unnecessary, as I am not making any changes to the file.
Once I have the PDF contents in 'data' variable, I would like to directly post them to the J2EE server. However, I have not been able to find a way to use the request module to directly post file contents. I have seen some examples related to POST using request module but they refer to formData. In my case, I don't have formData but instead reading the file from request and directly posting it to the J2EE server.
Is there a way to achieve this and avoid the file write and read?
EDIT
Below is my complete code
function upload(request, response) {
var data = new Buffer('');
request.on('data', function (chunk) {
data = Buffer.concat([data, chunk]);
});
request.on('end', function () {
fs.writeFile('abc.pdf', data, 'binary', function(err){
if (err) {
console.log('Error ' + JSON.stringify(err) );
throw err;
}
var source = fs.createReadStream('abc.pdf');
source.pipe(req.post('http://'+j2ee_host+':'+j2ee_port+'/myj2eeapp/Upload'));
})
})
}
You can pipe directly from the data request to the servlet
var req = require('request');
function upload(request, response) {
var target = req.post('http://'+j2ee_host+':'+j2ee_port+'/myjavaapp/Upload');
request.pipe(target);
target.on('finish', function () {
console.log('All done!');
//send the response or make a completed callback here...
});
}
I would like to upload multiple image files to my server. On the client side I m using dropzone.js. My back-end server is sails.js and I would like to use formidable in order to update the client with a progress-bar ( I am fairly new to sails.js and node.js)
This is my controller (written with typescript)
export var upload = function (req: express.Request, res: express.Response, next) {
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
});
form.on('progress', function (bytesReceived, bytesExpected) {
// calculating perscent and logging progress message
});
form.on('error', function (err) {
console.error(err); // --> here I get "Request aborted"
});
form.on('end', function (fields, files) {
// here I save files to disk with fs-extra and I
// send back to client appropriate status
});
}
I keep getting 'Request aborted' in the error handler. I have made a few search on stackoverflow and on google without findind a clear answer. The problem seems to be related to multi-part form data and the appropriate express middleware to use in order to parse the body. How can I set my sails.js application in order to upload a file using dropzone.js on the client and sails+formidable on the server?
Any help will be greatly appreciate
With Dropzone and Sails.js, you have to:
add the definination of the filename in dropzone configuration:
Dropzone.options.myDropzone = { paramName: "fileName" }
use this command to get back the uploaded file:
req.file('fileName').upload(function (err, uploadedFiles) {
});
uploadedFiles contains the file
My goal is to accept an uploaded file and stream it to Wistia using the the Wistia Upload API. I need to be able to add fields to the HTTP request, and I don't want the file to touch the disk. I'm using Node, Express, Request, and Busboy.
The code below has two console.log statements. The first returns [Error: not implemented] and the second returns [Error: form-data: not implemented]. I'm new to streaming in Node, so I'm probably doing something fundamentally wrong. Any help would be much appreciated.
app.use("/upload", function(req, res, next) {
var writeStream = new stream.Writable();
writeStream.on("error", function(error) {
console.log(error);
});
var busboy = new Busboy({headers: req.headers});
busboy.on("file", function(fieldname, file, filename, encoding, mimetype) {
file.on("data", function(data) {
writeStream.write(data);
});
file.on("end", function() {
request.post({
url: "https://upload.wistia.com",
formData: {
api_password: "abc123",
file: new stream.Readable(writeStream)
}
}, function(error, response, body) {
console.log(error);
});
});
});
req.pipe(busboy);
});
I am not to familiar with the busboy module, but there errors you are getting are from attempting to use un-implemented streams. Whenever you create a new readable or writable stream directly from the stream module you have to create the _read and _write methods respectively Stream Implementors (node.js api). To give you something to work with the following example is using multer for handling multipart requests, I think you'll find multer is easier to use than busboy.
var app = require('express')();
var fs = require('fs');
var request = require('request');
app.use(multer());
app.post("/upload", function(req, res, next) {
// create a read stream
var readable = fs.createReadStream(req.files.myfile.path);
request.post({
url: "https://upload.wistia.com",
formData: {
api_password: "abc123",
file: readable
}
}, function(err, res, body) {
// send something to client
})
});
I hope this helps unfortunately I am not familiar with busboy, but this should work with multer, and as I said before there problem is just that you are using un-implemented streams I'm sure there is a way to configure this operation with busboy if you wanted.
If you want to use multipart (another npm) here is a tutorial:
http://qnimate.com/stream-file-uploads-to-storage-server-in-node-js/
I did not know what to write as the title, as I am having a very strange issue. What I am trying to do is upload and save *.html files on server. Following is the code structure:
Jade Template (Form):
#template-uploader
form(enctype='multipart/form-data')
input(name='file', type='file')
input#upload-template(type='button', value='Upload')
JS (Form Handle):
//Upload Btn Click Event Handler
$('#upload-template').on('click', function(){
event.stopPropagation();
event.preventDefault();
uploadFiles();
});
// Upload the files using AJAX
function uploadFiles()
{
var formData = $('input[type=file]')[0].files;
$.ajax({
url: 'template/upload',
type: 'POST',
xhr: function() { // Custom XMLHttpRequest
var myXhr = $.ajaxSettings.xhr();
if(myXhr.upload){
// For handling the progress of the upload
}
return myXhr;
},
data: formData[0],
cache: false,
processData: false, // Don't process the files
contentType: false, // Set content type to false as jQuery will tell the server its a query string request
success: function(data, textStatus, jqXHR)
{
console.log('Data');
console.log(data);
if(typeof data.error === 'undefined')
{
// Success so call function to process the form
}
else
{
// Handle errors here
console.log('ERRORS: ' + data.error);
}
},
error: function(jqXHR, textStatus, errorThrown)
{
// Handle errors here
console.log('ERRORS: ' + errorThrown);
// STOP LOADING SPINNER
}
});
}
Server (Node.js)
//Route handler for template file uploaded
router.post('/template/upload', function(req, res) {
console.log('Uploading Files');
console.log(req.files);
});
Now the issue is that when I select a file and click the upload button, an ajax request is made. I have logged the data that I am sending and seems good at the client end. On server side there are however two issues.
(Issue solved by answer from #Scimonster)I don't see any file in req.files param. I did this before in Express 3.x without any issues. Now I am using Express 4.x, and maybe I am missing something.
The second issue is that when the request is sent to server, the terminal immediately logs the console.log('Uploading Files') message. But I don't receive any error or success message on client side. I also don't see in terminal that a POST request was received for the mentioned route. However after 2 minutes (every time), the terminal logs the request received for the route including the console.log() message. And this is when I get a response on client side.
Terminal Logging:
Uploading Files
undefined
POST /dashboard/template/upload 200 **120004ms**
Uploading Files
undefined
This is beyond me. I don't think there are any pre-flight checks generated by client side.If someone can provide insight as to what might the issue be, that'd be great.
req.files from Express 3 is from the body parser middleware. In 4.x, this is no longer packaged with Express. You can install multer and include it as explained in the docs to enable req.files.