I am trying to write a file into MongoDB using mongoose and GridFS.
However, gridfs writestream is not firing any of the events - close or finish.
Also, it is not firing even the 'error event'(Just in case if there is any error). My nodejs version is 4.4.5 .
Code is below:
var mongoose = require('mongoose');
var formidable = require('formidable'),
http = require('http'),
util = require('util'),
fs = require('fs-extra');
var Regex = require("regex");
var fs = require('fs');
var path=require('path');
var grid =require("gridfs-stream");
var createRequirement = function (req, res) {
var form = new formidable.IncomingForm({
uploadDir: __dirname + '/upload'
});
form.multiples = true;
form.keepExtensions = true;
files = [],
fields = [];
form.on('field', function (field, value) {
})
form.on('file', function (field, file) {
console.log(file.name);
console.log('File uploaded : ' + file.path);
grid.mongo = mongoose.mongo;
var gfs = grid(db.db);
var writestream = gfs.createWriteStream({
filename: file.name,
mode: 'w'
});
fs.createReadStream(file.path).pipe(writestream);
//Below event is not fired.
writestream.on('finish', function (file) {
Company.findOne({
"users.userName": req.user.userName
}).then(function (data) {
var company = data;
if (!company) {
return res.status(404).send({
'Not Found': 'Company data not found'
});
} else {
Contact.findByIdAndUpdate(
file._id, {
$push: {
"attachments": {
id: file._id
}
}
}, {
safe: true,
upsert: true,
new: true
},
function (err, model) {
console.log(err);
}
);
}
});
})
});
form.parse(req);
return;
};
I could figure out the solution after lot of tries .
It was mongoose connection which was causing the issue.
gridfs-stream expects direct Mongo-DB connection.
I wish they had streamlined things and facilitated usage of Mongoose connection.
Related
I want to save and retrieve a Word doc and PDF file with a size of 1 MB, directly in MongoDB with Node.js. How can I do this is there any article explain about it or can some one help me on this.
Here is the standalone node js code to save the file as binary data in MongoDB. As the maximum file size is 1MB, you can save it in normal collection rather than GridFs.
This can be extended to run as web apps using "express" or "hapi" frameworks. You may need to refer the respective tutorial for that.
Save the file as binary data in MongoDB:-
Note: I have the sample file in "docs" directory. So, I have prefixed it with docs (i.e. "/docs/testplan.docx"). You can remove that if you don't need it.
var MongoClient = require('mongodb').MongoClient;
var Binary = MongoClient.Binary;
var fs = require('fs');
var assert = require('assert');
var url = 'mongodb://localhost:27017/test';
var binaryFileData = fs.readFileSync(__dirname + "/docs/testplan.docx");
var insertDocument = function(db, callback) {
db.collection('file_save').insertOne( {
"fileName" : "testplan.docx",
"fileData" : binaryFileData
}, function(err, result) {
assert.equal(err, null);
console.log("Inserted a document into the collection.");
callback();
});
};
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
insertDocument(db, function() {
console.log("Closing the database connection...")
db.close();
});
});
Read the file data and save it to disk:-
var MongoClient = require('mongodb').MongoClient;
var Binary = MongoClient.Binary;
var fs = require('fs');
var assert = require('assert');
var url = 'mongodb://localhost:27017/test';
var findDocument = function (fileName, db, callback) {
db.collection('file_save').findOne({"fileName" : fileName }, function (err, document) {
assert.equal(null, err);
console.log(document.fileName);
fs.writeFileSync("testplan_out.docx", document.fileData.buffer);
console.log("File has been written to disk");
callback();
});
};
MongoClient.connect(url, function (err, db) {
assert.equal(null, err);
findDocument("testplan.docx", db, function () {
db.close();
});
});
that works perfectly alright! But I'm trying to upload the document from the POSTMAN and I'm developing my project with MEAN stack.
//document-model.js
var mongoose = require('mongoose'),
Schema = mongoose.Schema;
var documentSchema = {
docFile: {
type: String
}
};
mongoose.model('FileDocument', documentSchema);
//document-route.js
var express = require('express'),
documentRoute = express.Router(),
document = require('../controllers/document-controller');
documentRoute.post('/upload', document.uploadDocument);
module.exports = documentRoute;
//document-controller.js
var document = {
uploadDocument: function (req, res) {
var fileDocument = new FileDocument({
docFile: req.body.docFile
});
fileDocument.save(function (err, result) {
if (err) {
res.status(500).send(err);
} else {
res.status(200).send('Document Uploaded Successfully');
}
});
}
};
I was trying in this way but it is not uploading to mongodb the result is gives in the mongo shell.
{ "_id" : ObjectId("59693872b8b83f42b42a3b9f"), "docFile" : "", "__v" : 0 }
I want to get the data from oraclde db using nodejs and disply it over angularjs based ui. PFB my code for service.js :
var async = require('async');
var oracledb = require('oracledb');
var dbConfig = require('../utility/dbconfig.js');
var response = require('../utility/response.js');
var bodyParser = require('body-parser');
var express = require('express');
var router = express.Router();
router.use(bodyParser.json());
var dbdata='';
var app = express();
var oracledb = require('oracledb');
var dbConfig = require('../utility/dbconfig.js');
var doconnect = function(cb) {
oracledb.getConnection(
{
user : dbConfig.user,
password : dbConfig.password,
connectString : dbConfig.connectString
},
cb);
console.log('Connection was successful!');
};
var dorelease = function(conn) {
conn.close(function (err) {
if (err)
console.error(err.message);
});
console.log('Connection closed successfully!');
};
// Optional Object Output Format
var doquery_object = function (conn, cb) {
conn.execute(
"SELECT d.dc_name,c.cobrand_name,c.cobrand_id,c.IS_CACHERUN_DISABLED,c.is_channel,c.environment,c.COBRAND_STATUS_ID,c.deployment_mode,c.db_name,c.gatherer_group FROM cobrand_master c,dc_master d where d.dc_id = c.dc_id ORDER BY c.display_priority",
{},
{ outFormat: oracledb.OBJECT },
function(err, result)
{
if (err) {
return cb(err, conn);
} else {
console.log("----- cobrand_master details (OBJECT output format) --------");
console.log(result.rows);
return cb(null, conn);
}
});
};
async.waterfall(
[
doconnect,
doquery_object
],
function (err, conn) {
if (err) { console.error("In waterfall error cb: ==>", err, "<=="); }
if (conn)
dorelease(conn);
});
When I am doing npm start in command prompt, I am getting the data in JSON format.I want to save the output and send it over angular js UI. Since I am new to this, can anyone please help me with simple steps to do it with an example.
Im currently working on a NodeJS/Mongo project where I need to pull all the documents from a collection. I currently have the following code written:
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
assert = require('assert');
var server = new Server('[server]', 27017);
var authDB = new Db('admin', server);
var DB1250 = new Db('1250', server);
var findDocuments = function (callback) {
authDB.authenticate("Username", "Password");
DB1250.open(function (error, db) {
if (error) {
console.log(error);
}
else {
console.log("successfully accessed: ", db);
callback;
var cursor = db.collection('Patients').find();
console.log('cursor ', cursor);
cursor.forEach(function (error, document) {
if (error) {
console.log('Document does not exist. Error: ', error);
}
else {
console.log('Document: ', document);
}
});
}
});
};
findDocuments(function (data) {
});
I am able to authenticate/connect to the server, connect to the DB, and connect to the collection. When I enter the forEach loop to iterate through all the documents, I keep getting the error "Callback is not a function". Can you guys see what I am doing wrong?
I believe the cursor you have there has not resolved into an array, so forEach is not a valid method. You might be looking for eachAsync, which will wait for the query to return before iterating.
Alternatively, you can wait for the query promise to resolve, ie:
cursor.then(docs => docs.forEach(callback));
which I personally find a little clearer.
Heres the solution I came up with after using Mongoose:
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
Mongoose = require('mongoose');
assert = require('assert');
var findDocuments = function (callback) {
var options = { server: { socketOptions: { keepAlive: 1000 } } };
var connectionString = 'mongodb://username:password#server:27017/admin';
// Connected handler
Mongoose.connect(connectionString, function (err) {
var db = Mongoose.connection.useDb('db');
var collection = db.collection("collection");
collection.find().stream()
.on('data', function (document) {
console.log(document);
})
.on('error', function (err) {
// handle error
console.log(err);
})
.on('end', function () {
// final callback
});
});
// Error handler
Mongoose.connection.on('error', function (err) {
console.log(err);
});
// Reconnect when closed
Mongoose.connection.on('disconnected', function () {
self.connectToDatabase();
});
};
findDocuments(function () {
db.close();
});
cursor.forEach() is an asynchronous operation but returns nothing. So you shouldn't do db.close() or likewise below the line somethere synchronously.
Besides; I am not sure which version Node driver you are refering to but in the recent ones like v3+ cursor.forEach() doesn't take an error first type callback like the one that you use in your code. It will take an "iterator" and an "end" callback such as;
cursor.forEach(doc => console.log(doc),
err => err ? console.log(err)
: db.close())
So the above code will just work fine, iterating and processing the documents one by one as they appear without waiting up until all of them have been retrieved to the memory.
I am trying to figure out how I can post an image directly to GridFS without storing it anywhere on the server as a temporary file first.
I am using Postman (chrome ext.) to post a file, and I manage to store this post as a file using:
req.pipe(fs.createWriteStream('./test.png'));
I am also able to store directly to GridFS from a readStream when the readStream is created from a file on the server. (see code)
I have the following files, saveFromReq.js which listens for the POST and basically just passes this on to the savePic.js.
saveFromReq.js:
var express = require('express');
var app = express();
var savePic = require('./savePic');
var fs = require('fs');
var GridStore = require('mongodb').GridStore;
var pic = './square.png';
var picID;
//When the following
//var pic = fs.createReadStream('./square.png', {autoClose: true});
//is not commented out, and 'req' is replaced with 'pic' in the savePic function,
//the file square.png is stored correctly to GridFS
app.post('/picture', function(req, res){
savePic(req, function(id){});
res.writeHead(200, {'Content-Type': 'text' });
res.end("Sucsess!\n");
});
app.listen(process.env.PORT || 3413);
savePic.js:
var savePic = function(req, callback){
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
BSON = require('mongodb').pure().BSON,
assert = require('assert');
fs = require('fs');
//When the following
//req.pipe(fs.createWriteStream('./test.png'));
//is not commented out, the correct image is stored to test.png, and
//the sequence after req.on("data"... starts
//(That sequence does not start at all when this is commented out..)
var fileId = new ObjectID();
var db = new Db('testDB', new Server('localhost', 27017));
// Establish connection to db
db.open(function(err, db) {
var gridStore = new GridStore(db, 'test', 'w');
//open
gridStore.open(function(err, gridStore) {
console.log("opened");
req.on("data", function (data) {
console.log("data recieved");
gridStore.write(data, function (err, gridStore) {
if (err) {
console.log("error writing file");
}
});
});
req.on("end", function () {
gridStore.close(function (err, gridStore) {
if (!err) {
console.log("The file has been stored to database.");
db.close();
}
});
});
req.pipe(gridStore);
});
});
callback(fileId);
};
module.exports = savePic;
Any help would be greatly appreciated!
gridfs-stream makes that pretty easy:
// `gfs` is a gridfs-stream instance
app.post('/picture', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}));
res.send("Success!");
});
while #robertklep's answer is correct, I would like to add something to his answer. This code shows how you can send back the stored file's metadata.
app.post('/picture', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}).on('close', function(savedFile){
console.log('file saved', savedFile);
return res.json({file: savedFile});
}));
})
This worked for me with mongoose:
var gfs = Grid(mongoose.connection.db, mongoose.mongo);
var writeStream = gfs.createWriteStream({
filename: name,
mode: 'w',
content_type: 'video/mp4'
});
writeStream.on('close', function() {
console.log('close event');
});
fs.createReadStream('uploads/' + name + '/' + name + '.mp4').pipe(writeStream);
console.log('stream.write: ' + name + '/' + name + '.mp4');
I am struggling a couple of days with getting the video on client side browser. That is what I tried so far:
var readstream = gfs.createReadStream({
filename: file.filename
});
readstream.on('data', function(data) {
res.write(data);
console.log(data);
});
readstream.on('end', function() {
res.end();
});
readstream.on('error', function (err) {
console.log('An error occurred!', err);
throw err;
});
My Data on MongoDB side looks like:
db.fs.chunks.find()
{ "_id" : ObjectId("5757e76df14741bf0391aaca"), "files_id" : ObjectId("5757e76df14741bf0391aac8"), "n" : 0, "data" : BinData(0,"AAAAIGZ0eXBpc29....
And the contentType is 'video/mp4':
logging on browser side prints this:
Object { 0: "�", 1: "�", 2: "�", 3: " ", 4: "f", 5: "t", 6: "y", 7: "p", 8: "i", 9: "s", 85003 more… }
Could someone please save my live? I hope you do not see my post as not convenient in this place.
Complete code to insert the txtfile in mongodb using gridfs in nodejs.This works well `
var mongoose=require("mongoose");
var gridfsstream=require("gridfs-stream");
var fs=require("fs");
mongoose.connect("mongodb://localhost:27017/testimage");
var conn=mongoose.connection;
gridfsstream.mongo=mongoose.mongo;
conn.once("open",function()
{
console.log("database connected successfully");
var gfs=gridfsstream(conn.db);
var writestream=gfs.createWriteStream({
filename:"danger.txt"
});
fs.createReadStream("sivakasi.txt").pipe(writestream);
writestream.on("close",function(file)
{
console.log(file.filename +"stored successfully into mongodb using gridfs");
});
writestream.on("error",function(file)
{
console.log(file.filename +"not stored into mongodb using gridfs");
});
});
conn.on("error",function()
{
console.log("database not connected try again!!!");
});
`
complete code to post the image from html to nodejs store that image in mongodb using gridfs system and display that image in server.This code works well.
var express=require("express");
var bodyparser=require("body-parser");
var multer=require("multer");
var app=express();
var upload = multer({ dest: '/tmp/'});
app.use(bodyparser.urlencoded({extended:false}));
app.post("/uploadimage",upload.single("file"),function(request,response)
{
var mongoose=require("mongoose");
var gridfsstream=require("gridfs-stream");
var fs=require("fs");
mongoose.connect("mongodb://localhost:27017/testimage");
var con=mongoose.connection;
gridfsstream.mongo=mongoose.mongo;
con.once("open",function()
{
console.log("test image database connected successfully");
var gfs=gridfsstream(con.db);
var readstream=fs.createReadStream(request.file.originalname);
var writestream=gfs.createWriteStream({
filename:"mentorpicthree.jpg"
});
readstream.pipe(writestream);
writestream.on("close",function()
{
console.log("image stored in mongodb database successfully");
fs.readFile(request.file.originalname,function(err,data)
{
if(err)
{
response.writeHead(404,{"Content-Type":"text/plain"});
console.log("error");
}
else
{
response.writeHead(200,{"Content-Type":"image/jpg"});
response.end(data);
}
});
});
writestream.on("error",function()
{
console.log("image not stored in mongodb database");
});
});
con.on("error",function()
{
console.log("database not connected try again!!!");
});
});
app.listen(8086,function()
{
console.log("server running on port 8086");
});
<html>
<head>
<title>FILE UPLOAD</title>
</head>
<body>
<p>Ryan Dhal</p>
<form action="http://127.0.0.1:8086/uploadimage" method="POST" enctype="multipart/form-data">
<input type="file" name="file">
<br>
<input type="submit" value="UPLOAD">
</form>
</body>
</html>
I am trying to figure out how I can post an image directly to GridFS without storing it anywhere on the server as a temporary file first.
I am using Postman (chrome ext.) to post a file, and I manage to store this post as a file using:
req.pipe(fs.createWriteStream('./test.png'));
I am also able to store directly to GridFS from a readStream when the readStream is created from a file on the server. (see code)
I have the following files, saveFromReq.js which listens for the POST and basically just passes this on to the savePic.js.
saveFromReq.js:
var express = require('express');
var app = express();
var savePic = require('./savePic');
var fs = require('fs');
var GridStore = require('mongodb').GridStore;
var pic = './square.png';
var picID;
//When the following
//var pic = fs.createReadStream('./square.png', {autoClose: true});
//is not commented out, and 'req' is replaced with 'pic' in the savePic function,
//the file square.png is stored correctly to GridFS
app.post('/picture', function(req, res){
savePic(req, function(id){});
res.writeHead(200, {'Content-Type': 'text' });
res.end("Sucsess!\n");
});
app.listen(process.env.PORT || 3413);
savePic.js:
var savePic = function(req, callback){
var Db = require('mongodb').Db,
MongoClient = require('mongodb').MongoClient,
Server = require('mongodb').Server,
ReplSetServers = require('mongodb').ReplSetServers,
ObjectID = require('mongodb').ObjectID,
Binary = require('mongodb').Binary,
GridStore = require('mongodb').GridStore,
Grid = require('mongodb').Grid,
Code = require('mongodb').Code,
BSON = require('mongodb').pure().BSON,
assert = require('assert');
fs = require('fs');
//When the following
//req.pipe(fs.createWriteStream('./test.png'));
//is not commented out, the correct image is stored to test.png, and
//the sequence after req.on("data"... starts
//(That sequence does not start at all when this is commented out..)
var fileId = new ObjectID();
var db = new Db('testDB', new Server('localhost', 27017));
// Establish connection to db
db.open(function(err, db) {
var gridStore = new GridStore(db, 'test', 'w');
//open
gridStore.open(function(err, gridStore) {
console.log("opened");
req.on("data", function (data) {
console.log("data recieved");
gridStore.write(data, function (err, gridStore) {
if (err) {
console.log("error writing file");
}
});
});
req.on("end", function () {
gridStore.close(function (err, gridStore) {
if (!err) {
console.log("The file has been stored to database.");
db.close();
}
});
});
req.pipe(gridStore);
});
});
callback(fileId);
};
module.exports = savePic;
Any help would be greatly appreciated!
gridfs-stream makes that pretty easy:
// `gfs` is a gridfs-stream instance
app.post('/picture', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}));
res.send("Success!");
});
while #robertklep's answer is correct, I would like to add something to his answer. This code shows how you can send back the stored file's metadata.
app.post('/picture', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}).on('close', function(savedFile){
console.log('file saved', savedFile);
return res.json({file: savedFile});
}));
})
This worked for me with mongoose:
var gfs = Grid(mongoose.connection.db, mongoose.mongo);
var writeStream = gfs.createWriteStream({
filename: name,
mode: 'w',
content_type: 'video/mp4'
});
writeStream.on('close', function() {
console.log('close event');
});
fs.createReadStream('uploads/' + name + '/' + name + '.mp4').pipe(writeStream);
console.log('stream.write: ' + name + '/' + name + '.mp4');
I am struggling a couple of days with getting the video on client side browser. That is what I tried so far:
var readstream = gfs.createReadStream({
filename: file.filename
});
readstream.on('data', function(data) {
res.write(data);
console.log(data);
});
readstream.on('end', function() {
res.end();
});
readstream.on('error', function (err) {
console.log('An error occurred!', err);
throw err;
});
My Data on MongoDB side looks like:
db.fs.chunks.find()
{ "_id" : ObjectId("5757e76df14741bf0391aaca"), "files_id" : ObjectId("5757e76df14741bf0391aac8"), "n" : 0, "data" : BinData(0,"AAAAIGZ0eXBpc29....
And the contentType is 'video/mp4':
logging on browser side prints this:
Object { 0: "�", 1: "�", 2: "�", 3: " ", 4: "f", 5: "t", 6: "y", 7: "p", 8: "i", 9: "s", 85003 more… }
Could someone please save my live? I hope you do not see my post as not convenient in this place.
Complete code to insert the txtfile in mongodb using gridfs in nodejs.This works well `
var mongoose=require("mongoose");
var gridfsstream=require("gridfs-stream");
var fs=require("fs");
mongoose.connect("mongodb://localhost:27017/testimage");
var conn=mongoose.connection;
gridfsstream.mongo=mongoose.mongo;
conn.once("open",function()
{
console.log("database connected successfully");
var gfs=gridfsstream(conn.db);
var writestream=gfs.createWriteStream({
filename:"danger.txt"
});
fs.createReadStream("sivakasi.txt").pipe(writestream);
writestream.on("close",function(file)
{
console.log(file.filename +"stored successfully into mongodb using gridfs");
});
writestream.on("error",function(file)
{
console.log(file.filename +"not stored into mongodb using gridfs");
});
});
conn.on("error",function()
{
console.log("database not connected try again!!!");
});
`
complete code to post the image from html to nodejs store that image in mongodb using gridfs system and display that image in server.This code works well.
var express=require("express");
var bodyparser=require("body-parser");
var multer=require("multer");
var app=express();
var upload = multer({ dest: '/tmp/'});
app.use(bodyparser.urlencoded({extended:false}));
app.post("/uploadimage",upload.single("file"),function(request,response)
{
var mongoose=require("mongoose");
var gridfsstream=require("gridfs-stream");
var fs=require("fs");
mongoose.connect("mongodb://localhost:27017/testimage");
var con=mongoose.connection;
gridfsstream.mongo=mongoose.mongo;
con.once("open",function()
{
console.log("test image database connected successfully");
var gfs=gridfsstream(con.db);
var readstream=fs.createReadStream(request.file.originalname);
var writestream=gfs.createWriteStream({
filename:"mentorpicthree.jpg"
});
readstream.pipe(writestream);
writestream.on("close",function()
{
console.log("image stored in mongodb database successfully");
fs.readFile(request.file.originalname,function(err,data)
{
if(err)
{
response.writeHead(404,{"Content-Type":"text/plain"});
console.log("error");
}
else
{
response.writeHead(200,{"Content-Type":"image/jpg"});
response.end(data);
}
});
});
writestream.on("error",function()
{
console.log("image not stored in mongodb database");
});
});
con.on("error",function()
{
console.log("database not connected try again!!!");
});
});
app.listen(8086,function()
{
console.log("server running on port 8086");
});
<html>
<head>
<title>FILE UPLOAD</title>
</head>
<body>
<p>Ryan Dhal</p>
<form action="http://127.0.0.1:8086/uploadimage" method="POST" enctype="multipart/form-data">
<input type="file" name="file">
<br>
<input type="submit" value="UPLOAD">
</form>
</body>
</html>