how to download a file saved in gridFS using nodeJS - node.js

I need to download a resume from GridFS, below is the code ive written to do it, but this seems to not give me a physical file for download, this is used to reading the contents. How can i download the file?
exports.getFileById = function(req, res){
var conn = mongoose.connection;
var gfs = Grid(conn.db, mongoose.mongo);
var id = req.params.ID;
gfs.exist({_id: id,root: 'resume'}, function (err, found) {
if (err) return handleError(err);
if (!found)
return res.send('Error on the database looking for the file.');
gfs.createReadStream({_id: id,root: 'resume'}).pipe(res);
});
};

Hope this helps!
exports.downloadResume = function(req, res){
var conn = mongoose.connection;
var gfs = Grid(conn.db, mongoose.mongo);
gfs.findOne({ _id: <resumeId>, root: <collectionName> }, function (err, file) {
if (err) {
return res.status(400).send(err);
}
else if (!file) {
return res.status(404).send('Error on the database looking for the file.');
}
res.set('Content-Type', file.contentType);
res.set('Content-Disposition', 'attachment; filename="' + file.filename + '"');
var readstream = gfs.createReadStream({
_id: <resumeId>,
root: '<collectionName>'
});
readstream.on("error", function(err) {
res.end();
});
readstream.pipe(res);
});
};

I took hints from accepted answer. But I had to jump through some hoops to get it working hope this helps.
const mongodb = require('mongodb');
const mongoose = require('mongoose');
const Grid = require('gridfs-stream');
eval(`Grid.prototype.findOne = ${Grid.prototype.findOne.toString().replace('nextObject', 'next')}`);
const mongoURI = config.mongoURI;
const connection = mongoose.createConnection(mongoURI);
app.get('/download', async (req, res) => {
var id = "<file_id_xyz>";
gfs = Grid(connection.db, mongoose.mongo);
gfs.collection("<name_of_collection>").findOne({ "_id": mongodb.ObjectId(id) }, (err, file) => {
if (err) {
// report the error
console.log(err);
} else {
// detect the content type and set the appropriate response headers.
let mimeType = file.contentType;
if (!mimeType) {
mimeType = mime.lookup(file.filename);
}
res.set({
'Content-Type': mimeType,
'Content-Disposition': 'attachment; filename=' + file.filename
});
const readStream = gfs.createReadStream({
_id: id
});
readStream.on('error', err => {
// report stream error
console.log(err);
});
// the response will be the file itself.
readStream.pipe(res);
}
});

Related

Nodejs formidable async mongoose await issue

I'm trying to make a OCR parse of an image.
All the things works well but I have a problem this mongoose and syncronysm.
But cannot use "await" on the mongoose find call as the function is not async. How do I solve that.
Here is my code:
// post processImage
router.post('/', async (req, res) => {
try {
var baseUrl;
const form = formidable({ multiples: true });
form.parse(req, function (error, fields, files) {
var imatgeAProcessar = files.image.path;
var extname = path.extname(files.image.name);
getTextFromImage(imatgeAProcessar) // OCR process of the image
.then(res => {
const boss_name_req = res.boss_name;
const boss = Boses.findOne({"name" : boss_name_req}).exec();
// ERROR HERE // return nothing althought it exist on database (no await?)
console.log(JSON.stringify(boss)); // writes "{}"
const processedImage = {
"success": true,
"boss_name": boss.name,
"boss_image": baseUrl + 'images/' + boss.num + ".png"
}
res.json(processedImage);
})
});
} catch (err) {
res.json({message: err});
}
});
*edited
// post processImage
router.post('/', async(req, res) => {
try {
var baseUrl;
const form = formidable({ multiples: true });
var formfields = await new Promise(function(resolve, reject) {
form.parse(req, function(err, fields, files) {
if (err) {
reject(err);
return;
}
resolve(files);
}); // form.parse
});
var imatgeAProcessar = formfields.image.path;
var extname = path.extname(formfields.image.name);
const res = await getTextFromImage(imatgeAProcessar)
const boss_name_req = res.boss_name;
const boss = await Boses.findOne({ "name": boss_name_req }).limit(4).skip(0).exec();
const processedImage = {
"success": true,
"boss_name": boss.name,
"boss_image": baseUrl + 'images/' + boss.num + ".png"
}
res.json(processedImage)
} catch (err) {
res.json({ message: err });
}
});
Finally I found the way... I wrote a callback on the findOne call as :
const boss = Boss.findOne({"name" : boss_name_req})
.then( resMongoose => {
try {
const processedImage = {
"success": true,
"gym": resOCR.gym,
"boss_name": resMongoose.name,
}
res.json(processedImage);
} catch (err) {
res.json({message: err});
}
});

I am getting error on submitting the form

While connecting to mongodb, I am getting a warning
"DeprecationWarning: current URL string parser is deprecated, and will
be removed in a future version. To use the new parser, pass option {
useNewUrlParser: true } to MongoClient.connect."
while submitting the form - I am getting
The database connection must be open to store files
at GridFSStorage._handleFile (C:\Users\charan puli\Desktop\upload\node_modules\multer-gridfs-storage\lib\gridfs.js:341:17)
at C:\Users\charan puli\Desktop\upload\node_modules\multer\lib\make-middleware.js:144:17
at allowAll (C:\Users\charan puli\Desktop\upload\node_modules\multer\index.js:8:3)
at wrappedFileFilter (C:\Users\charan puli\Desktop\upload\node_modules\multer\index.js:44:7)
at Busboy. (C:\Users\charan puli\Desktop\upload\node_modules\multer\lib\make-middleware.js:114:7)
at Busboy.emit (events.js:182:13)
//////////////////app.js///////////////////////
//middle wares
app.use(bodyparser.json())
app.use(methodOverride('_method'))
app.set("view engine","ejs")
//connection
var mongoURI='mongodb+srv://user:password#clusterpuli-xs9yc.mongodb.net/test?retryWrites=true'
mongoose.connect(mongoURI,{useNewUrlParser:true})
.then(()=>{console.log('connected successfully');
})
.catch(err=>{console.log(err);
})
var conn=mongoose.connection
var gfs
conn.once('open',() =>{
gfs = Grid(conn.db, mongoose.mongo);
gfs.collection('questions')
})
//create storage object
var storage = new GridFsStorage({
url: mongoURI,
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
const filename =buf.toString('hex')+path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: 'questions'
};
resolve(fileInfo);
});
});
}
});
const upload = multer({ storage });
//#route /upload POST
app.post('/upload',upload.single('file'),(req,res)=>{
res.json({'file':req.file})
})
var port= 3000
app.get("/",(req,res)=>{
res.render('index')
})
app.listen(port,()=>{
console.log(`app is running at ${port}`);
})
////////////////////////////////////
expected - json file object, connected Successfully
actual - database connection must be open
This should work for you, I have tested it:
//connection
const mongoURI = 'mongodb+srv://user:password#clusterpuli-xs9yc.mongodb.net/test?retryWrites=true';
const promise = mongoose.connect(mongoURI, { useNewUrlParser: true });
const conn = mongoose.connection;
let gfs;
conn.once('open',() => {
gfs = Grid(conn, mongoose.mongo);
gfs.collection('questions');
});
//create storage object
const storage = new GridFsStorage({
db: promise,
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => {
if (err) {
return reject(err);
}
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: 'questions'
};
resolve(fileInfo);
});
});
}
});
const upload = multer({ storage });
See this reference here for further information:
Multer's GridFS storage engine
It has to do with the new url parser of the new mongo client of the Node.js MongoDB Driver API. See this reference: MongoClient
I had the same issue that's why I ended up on your question.
//connection
var mongoURI='mongodb+srv://charanpuli:Charan#1999#clusterpuli-xs9yc.mongodb.net/test?retryWrites=true'
var conn = mongoose.connection
MongoClient.connect(db, { useNewUrlParser: true })
.then(()=>{console.log('connected successfully');
})
.catch(err=>{console.log(err);
})
var gfs
conn.once('open',() =>{
gfs = Grid(conn.db, mongoose.mongo);
gfs.collection('questions')
})

Node.JS downloading hundreds of files simultaneously

I am trying to download more that 100 files at the same time. But when I execute the downloading function my macbook freezes(unable to execute new tasks) in windows also no download(but doesn't freeze) and no download progress in both case(idle network).
Here is my download module:
var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
router.get('/', function (req, res, next) {
links.find({dlStatus: false}, function (err, docs) {
if (err) {
console.log(err);
res.end();
} else if (!docs) {
console.log('No incomplete downloads!');
res.end();
} else {
for (var i = 0; i < docs.length; i++) {
//todo scraping
var video = youtubedl(docs[i].url, [], {cwd: __dirname});
// Will be called when the download starts.
video.on('info', function (info) {
console.log('Download started');
console.log(info);
});
video.pipe(fs.createWriteStream('./downloads/' + docs[i].id + '-' + i + '.mp4'));
video.on('complete', function complete(info) {
links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
if (err)console.log(err);
else console.log('Download completed!')
});
});
}
}
});
});
module.exports = router;
Now can anyone please help me here? I am using this module for downloading files.
The solution is using async in this case.
Try it this way....with async.each()
var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
var async = require('async')
router.get('/', function (req, res, next) {
links.find({dlStatus: false}, function (err, docs) {
if (err) {
console.log(err);
res.end();
} else if (!docs) {
console.log('No incomplete downloads!');
res.end();
} else {
async.each(docs,function(doc,cb){
var video = youtubedl(doc.url, [], {cwd: __dirname});
// Will be called when the download starts.
video.on('info', function (info) {
console.log('Download started');
console.log(info);
});
video.pipe(fs.createWriteStream('./downloads/' + docs.id + '-' + i + '.mp4'));
video.on('complete', function complete(info) {
links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
if (err){
console.log(err);
cb(err);
}
else {
console.log('Download completed!');
cb()
}
});
});
},function(err){
if(err)
return console.log(err);
console.log("Every thing is done,Here!!");
})
}
});
});
module.exports = router;
And you can process every thing in batch too using async.eachLimits().

Post file from one server to another,using node.js , needle , busboy/multer

I would like to move a small image from one server to another (both running node). As I search, I haven't found enough. This post remains unanswered.
As I started experimenting I wrote the following to the first server :
app.post("/move_img", function(req, res) {
console.log("post handled");
fs.readFile(__dirname + "/img_to_move.jpg", function(err, data) {
if (err) throw err;
console.log(data);
needle.post(server2 + "/post_img", {
data: data,
name : "test.jpg"
}, function(result) {
console.log(result);
res.send("ok");
});
});
});
This part seems to be working as I could be writing the data in the same server (using fs.writeFile) recreate the img.
Now as I am trying to handle the post in the other server I have a problem.
Server2:
app.post('/post_img', [ multer({ dest: './uploads/images'}), function(req, res) {
console.log("body ",req.body) // form fields
console.log("files ",req.files) // form files
res.send("got it");
}]);
This way i get an empty object in the files and the following in the body: { 'headers[Content-Type]': 'application/x-www-form-urlencoded', 'headers[Content-Length]': '45009' }
I think I could use busboy as an alternative but I can't make it to work. Any advice, tutorial would be welcome.
I solved my problem by using the following code,
server1 (using needle) :
app.post("/move_img", function(req, res) {
console.log("post handled")
var data = {
image:{
file: __dirname + "/img_to_move.jpg",
content_type: "image/jpeg"}
}
needle.post(server2 + "/post_img", data, {
multipart: true
}, function(err,result) {
console.log("result", result.body);
});
})
Server 2:
app.use('/post_img',multer({
dest: '.uploads/images',
rename: function(fieldname, filename) {
return filename;
},
onFileUploadStart: function(file) {
console.log(file.originalname + ' is starting ...')
},
onFileUploadComplete: function(file) {
console.log(file.fieldname + ' uploaded to ' + file.path)
}
}));
app.post('/post_img', function(req, res) {
console.log(req.files);
res.send("File uploaded.");
});
An alternative for the server 1 is the following (using form-data module):
var form = new FormData();
form.append('name', 'imgTest.jpg');
form.append('my_file', fs.createReadStream(__dirname + "/img_to_move.jpg"));
form.submit(frontend + "/post_img", function(err, result) {
// res – response object (http.IncomingMessage) //
console.log(result);
});
I'd simply read your file from the first server with the function readFile() and then write it to the other server with the function writeFile().
Here you can see use of both functions in one of my servers.
'use strict';
const express = require('express');
const multer= require('multer');
const concat = require('concat-stream');
const request = require('request');
const router = express.Router();
function HttpRelay (opts) {}
HttpRelay.prototype._handleFile = function _handleFile (req, file, cb) {
file.stream.pipe(concat({ encoding: 'buffer' }, function (data) {
const r = request.post('/Endpoint you want to upload file', function (err, resp, body) {
if (err) return cb(err);
req.relayresponse=body;
cb(null, {});
});
const form = r.form();
form.append('uploaded_file', data, {
filename: file.originalname,
contentType: file.mimetype
});
}))
};
HttpRelay.prototype._removeFile = function _removeFile (req, file, cb) {
console.log('hello');
cb(null);
};
const relayUpload = multer({ storage: new HttpRelay() }).any();
router.post('/uploadMsgFile', function(req, res) {
relayUpload(req, res, function(err) {
res.send(req.relayresponse);
});
});
module.exports = router;
see multer does all the tricks for you.
you just have to make sure you use no middle-ware but multer to upload files in your node starting point.
Hope it does the tricks for you also.

Stream buffer to client in Express

I have request handler to send file from MongoDB (GridFS) to client like below, but it use data variable so content is in memory. I need to make this in streaming mode and send file in chunks to client. I can't regognize how to pipe buffer to response. Look at second code - it doesn't work, but show something what i need.
Maybe it is useful: Data in GridFS is Base64 encoded, but may be changed if streaming can be more efficient.
In-Memory version
router.get('/get/:id', function(req,res){
getById(req.params.id, function(err, fileId){
new GridStore(db, fileId, "r").open(function(err, gridStore) {
res.set('Content-Type', gridStore.contentType);
var stream = gridStore.stream(true);
var data = '';
stream.on("data", function(chunk) {
data += chunk;
});
stream.on("end", function() {
res.send(new Buffer(data, 'base64'));
});
});
});
});
Streaming mode version
router.get('/get/:id', function(req,res){
getById(req.params.id, function(err, fileId){
new GridStore(db, fileId, "r").open(function(err, gridStore) {
res.set('Content-Type', gridStore.contentType);
var stream = gridStore.stream(true);
stream.on("data", function(chunk) {
new Buffer(chunk, 'base64').pipe(res);
});
stream.on("end", function() {
res.end();
});
});
});
});
Update
I think I'm close to resolve this. I found this works, but does't decode from Base64:
new GridStore(db, fileId, "r").open(function(err, gridStore) {
res.set('Content-Type', gridStore.contentType);
gridStore.stream(true).pipe(res);
});
exports.sendFile = function(db, res, fileId) {
var grid = require('gridfs-stream');
var gfs = grid(db, mongoose.mongo);
var on_error = function(){
res.status(404).end();
};
var readstream = gfs.createReadStream({
filename: fileId,
root: 'r'
});
readstream.on('error', function(err) {
if (('\'' + err + '\'') === '\'Error: does not exist\'') {
return on_error && on_error(err);
}
throw err;
});
return readstream.pipe(res);
}
I found a solution, but think that can be better. I use base64-stream module to decode Base64 stream. Solution below:
router.get('/get/:id', function(req,res){
getById(req.params.id, function(err, fileId){
new GridStore(db, fileId, "r").open(function(err, gridStore) {
res.set('Content-Type', gridStore.contentType);
gridStore.stream(true).pipe(base64.decode()).pipe(res);
});
});
});
stream.on("data", function(chunk) {
res.send(chunk.toString('utf8'));
});

Resources