GridFS piping to Express response issue - node.js

I'm developing a web application where I store uploaded images inside MongoDB's GridFS. I have a problem trying to pipe the readable stream to the response, as it works sometimes but not every time. All images is stored inside GridFS and none of the files are corrupt as I'm able to extract the file with mongofiles and view it and the image looks exactly like the uploaded one.
Piping to the response works for some images, but some don't and I am pulling my hair off about this - I can't pinpoint the issue.
I am using gridfs-stream (1.1.0) with ExpressJS (4.0.0) and here's the response route:
exports.show = function (req, res, next) {
var id = req.params.id;
GridFS.exist({ _id: id }, function (err, exist) {
if (err) return handleError(err);
if (!exist) return res.send(404);
try {
var readStream = GridFS.createReadStream({ _id: gridId }).pipe(res);
} catch (err) {
return res.send(500, err);
}
});
};
And here's the upload route:
exports.create = function (req, res, next) {
var mime = req.files.file.mimetype;
var image = req.files.file.path;
var filename = req.files.file.originalname;
var writeStream = GridFS.createWriteStream({
filename: filename,
mode: 'w',
content_type: mime
});
writeStream.on('close', function (file) {
console.log(file);
return res.json(200, { status: 'success', url: '/api/images/' + file._id });
});
fs.createReadStream(image).pipe(writeStream);
};
Now as I mentioned, this works for some images, but not every image. I'm using Node 0.12.0 and MongoDB 2.6.8
Tell me if you need any additional information and I'll try to provide it.

Related

Can't parse form-data twice

What I am trying to do is delete the images with same id before I upload a new image, if the user has inputed image. Multer does not provide a way to detect if there is an input only, it has to upload the file first.
So I figured out to use another form-data libarary to do just that. But when I do that multer doesn't recieve the form-data to upload. So it doesn't upload anything. The image only get delete, but no new images are added.
The problem is I can't parse for data twice. How can i fix it? Or is there a workaround?
Code explination
I am using multyparty to detect if the user has included an image in a form.
form.parse(req, async (err, fields, files) => {
if (err) console.log(err);
else if (files.image) removeImagesWithSameId(req.params.id)
});
If there an image in the form then delete the previous uploaded image
const removeImagesWithSameId = (id) => {
fs.unlink(__dirname + id + ".jpg", (err) => {
if (err) console.log(err)
console.log("image Delete")
})
}
Then upload the new image, using multer.
upload(req, res, (err) => {
if (err) console.log(err)
else if (req.file) console.log("Image uploaded")
});
Execpt the new image not getting uploaded, because multer not receiving the data and req.file is undefined.
My questions is why multer is not receiving the data when ever I add multiparty, and how to fix it?
Whole the code
const express = require('express');
const app = express();
const multiparty = require('multiparty');
const form = new multiparty.Form();
var upload = multer({ dest: __dirname }).single('image');
const removeImagesWithSameId = (id) => {
fs.unlink(__dirname + id + ".jpg", (err) => {
if (err) console.log(err)
console.log("image Delete");
})
}
app.post('/upload/:id', (req, res) => {
form.parse(req, async (err, fields, files) => {
if (err) console.log(err);
else if (files.image) removeImagesWithSameId(req.params.id)
});
upload(req, res, (err) => {
if (err) console.log(err)
else if (req.file) console.log("Image uploaded")
});
})
Note this is only demonstration of the probelm. My actaul code is bit longer then that.
For people who is asking about the requst, I am using nodemon, but this equivalent curl requst.
curl --location --request PUT 'localhost/upload/1593735936343' \
--form 'image=#/C:/Users/Hamza/images/test.jpg'

Set filname by id generated by mongoose using multer node js

I am new in Node js API and I'm trying to upload an Image using multer + express + mongoose + postman
CODE:
var storage = multer.diskStorage({
destination: function (request, file, callback) {
callback(null, 'public/images/course');
},
filename: function (request, file, callback) {
return callback(null, file.originalname)
}
});
var upload = multer({storage : storage})
router.post('/course', upload.single('thumbnail'),async(req, res) => {
try{
var course = new Course({
name : req.body.name,
thumbnail : "placeholder" // set to path where file is uploaded
})
await course.save()
res.status(201).send(course)
}catch(e){
res.status(400).send(e)
}
})
I use postman to post request using form data and it creates an image with its originalFilename but i want the filename to be id generated by mongoose and i have seen somewhere that i can use filesystem for that but is there any way i can upload file after id is generated because when i do like this
var storage = multer.diskStorage({
destination: function (request, file, callback) {
callback(null, 'public/images/course');
},
filename: function (request, file, callback) {
if (request.data) {
console.log(file)
// TODO: consider adding file type extension
fileExtension = file.originalname.split('.')[1]
return callback(null, `${request.path}-${request.data._id.toString()}.${fileExtension}`);
}
// fallback to the original name if you don't have a book attached to the request yet.
return callback(null, file.originalname)
}
});
var upload = multer({storage : storage}).single('thumbnail')
router.post('/course',(req, res) => {
console.log(req.body)
const course = new Course({
name : req.body.name,
thumbnail : req.body.name
})
//console.log(course)
req.data = course
//console.log(req.file)
upload(req, res, function (err) {
if (err instanceof multer.MulterError) {
// A Multer error occurred when uploading.
} else if (err) {
// An unknown error occurred when uploading.
}
// Everything went fine.
})
})
Then i got request body empty.
Thanks in advance

A super simple Multer question, nested file uploading

I'm trying to do an image upload with Multer and Express. Doing just an image upload is going fine and everything works, the problem is, I want to send more than just one photo. Let me explain. I got the most basic form ever, not worth sharing. Then when submitting the form with JUST an image, the axios request looks like this:
async onSubmit() {
const formData = new FormData();
formData.append('file', this.person.personData.file)
this.obj.file = formData
try {
await axios.post('/projects/new', this.obj.file);
this.message = 'Uploaded';
} catch (err) {
console.log(err);
this.message = 'Something went wrong'
}
},
The post route in Express to receive the image looks like this:
personRoutes.post('/new', upload.single('file'), (req, res) => {
console.log('BODY: ', req.body)
console.log('REQ.FILE: ', req.file)
const person = new Person({
personData: {
file: req.file.path
}
});
person.save()
.then(result => {
console.log('YES', result)
res.redirect('/projects')
})
.catch(err => {
console.log('KUT', err)
})
});
req.file is the upload.single('file') file. Req.body will hold the text fields, if there were any. Ez Pz, so far so good. Now, where things get a bit sketchy is, what if I wanted to upload more than just one photo? So my obj object would not only hold a file property but a few others aswell. Currently I am directly sending the formData file with
await axios.post('/projects/new', this.obj.file);
But if my obj contained more than just a file, I would have to to this:
await axios.post('/projects/new', this.obj);
But what in the world should my Express post route look like? Because req.file will now (as far as I know) forever be undefined because file is not defined inside req object. It is defined in req.body object. Accessing the file as req.obj.file won't do anything. Any help would be very much appreciated. I don't even know if it is possible. And if not, what other options do I have?
Thanks in advance!
upload.array('file') should work. any number of files will be received.
here is an example:
multer code:
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, 'uploads')
},
filename: (req, file, cb) => {
cb(null, "image"+Date.now()+file.originalname);
}
});
const fileFilter = (req,file,cb)=>{
if(file.mimetype==="image/jpeg" || file.mimetype==="image/png"){
cb(null, true);
}else{
cb(new Error("File type is not acceptable"),false);
}
}
const uploadImages = multer({storage:storage,
limits:{
fileSize: 1024*1024*10
},
fileFilter:fileFilter}).array("shopImage");
app.post code:
app.post("/shop", function(req,res){
uploadImages(req,res,function(err){
if(err){
console.log(err);
res.status(400).json({message:err.message});
}else{
console.log(req.files);
console.log(req.body);
....
}
});
....
})

"media type unrecognized" when uploading an image to Twitter using npm twit

I am working on an express node app that posts to twitter when a user inputs an image into a form. I am saving the image locally before uploading, which works. After I encode the file to base64, I try to upload the base64-encoded file to Twitter using twit's media/upload feature. When I do this, I get an error saying "media type unrecognized."
Here is my code:
app.post('/tweet', function(req, res){
var time = new Date().getTime()
let image = req.files.image
var imgpath = './images/img' + time + '.jpg'
image.mv(imgpath, function(err) {
if (err){
return res.status(500).send(err);
}
});
var b64content = fs.readFileSync(imgpath, { encoding: 'base64' })
T.post('media/upload', {media: b64content}, function(err, data, res) {
if (err) console.log(err);
console.log(data);
T.post('statuses/update', {status: 'posted picture at: ' + time, media_ids: [data.media_id_string]}, function(err, params, res) {
if (err) console.log(err);
console.log(params);
});
});
return res.redirect('/')
})
Thank you!
Got it!. I needed to put the T.post code in the brackets of image.mv's function
use postMediaChunked function
var filePath = '/absolute/path/to/file.png'
T.postMediaChunked({ file_path: filePath }, function (err, data, response) {
console.log(data)
})

Post file from one server to another,using node.js , needle , busboy/multer

I would like to move a small image from one server to another (both running node). As I search, I haven't found enough. This post remains unanswered.
As I started experimenting I wrote the following to the first server :
app.post("/move_img", function(req, res) {
console.log("post handled");
fs.readFile(__dirname + "/img_to_move.jpg", function(err, data) {
if (err) throw err;
console.log(data);
needle.post(server2 + "/post_img", {
data: data,
name : "test.jpg"
}, function(result) {
console.log(result);
res.send("ok");
});
});
});
This part seems to be working as I could be writing the data in the same server (using fs.writeFile) recreate the img.
Now as I am trying to handle the post in the other server I have a problem.
Server2:
app.post('/post_img', [ multer({ dest: './uploads/images'}), function(req, res) {
console.log("body ",req.body) // form fields
console.log("files ",req.files) // form files
res.send("got it");
}]);
This way i get an empty object in the files and the following in the body: { 'headers[Content-Type]': 'application/x-www-form-urlencoded', 'headers[Content-Length]': '45009' }
I think I could use busboy as an alternative but I can't make it to work. Any advice, tutorial would be welcome.
I solved my problem by using the following code,
server1 (using needle) :
app.post("/move_img", function(req, res) {
console.log("post handled")
var data = {
image:{
file: __dirname + "/img_to_move.jpg",
content_type: "image/jpeg"}
}
needle.post(server2 + "/post_img", data, {
multipart: true
}, function(err,result) {
console.log("result", result.body);
});
})
Server 2:
app.use('/post_img',multer({
dest: '.uploads/images',
rename: function(fieldname, filename) {
return filename;
},
onFileUploadStart: function(file) {
console.log(file.originalname + ' is starting ...')
},
onFileUploadComplete: function(file) {
console.log(file.fieldname + ' uploaded to ' + file.path)
}
}));
app.post('/post_img', function(req, res) {
console.log(req.files);
res.send("File uploaded.");
});
An alternative for the server 1 is the following (using form-data module):
var form = new FormData();
form.append('name', 'imgTest.jpg');
form.append('my_file', fs.createReadStream(__dirname + "/img_to_move.jpg"));
form.submit(frontend + "/post_img", function(err, result) {
// res – response object (http.IncomingMessage) //
console.log(result);
});
I'd simply read your file from the first server with the function readFile() and then write it to the other server with the function writeFile().
Here you can see use of both functions in one of my servers.
'use strict';
const express = require('express');
const multer= require('multer');
const concat = require('concat-stream');
const request = require('request');
const router = express.Router();
function HttpRelay (opts) {}
HttpRelay.prototype._handleFile = function _handleFile (req, file, cb) {
file.stream.pipe(concat({ encoding: 'buffer' }, function (data) {
const r = request.post('/Endpoint you want to upload file', function (err, resp, body) {
if (err) return cb(err);
req.relayresponse=body;
cb(null, {});
});
const form = r.form();
form.append('uploaded_file', data, {
filename: file.originalname,
contentType: file.mimetype
});
}))
};
HttpRelay.prototype._removeFile = function _removeFile (req, file, cb) {
console.log('hello');
cb(null);
};
const relayUpload = multer({ storage: new HttpRelay() }).any();
router.post('/uploadMsgFile', function(req, res) {
relayUpload(req, res, function(err) {
res.send(req.relayresponse);
});
});
module.exports = router;
see multer does all the tricks for you.
you just have to make sure you use no middle-ware but multer to upload files in your node starting point.
Hope it does the tricks for you also.

Resources