Read/write binary data to MongoDB in Node.js - node.js

I've been able to successfully write binary data (an image) to MongoDB in Node.js. However I can't find clear documentation on how to read it back.
Here's how I'm writing the image to MongoDB:
var imageFile = req.files.myFile;
var imageData = fs.readFileSync(imageFile.path);
var imageBson = {};
imageBson.image = new db.bson_serializer.Binary(imageData);
imageBson.imageType = imageFile.type;
db.collection('images').insert(imageBson, {safe: true},function(err, data) {
I'd appreciate any pointers on reading the image from Mongo using Node. I'm assuming there's a function like "db.bson_deserializer...". Thanks!

Found the answer:
var imageFile = req.files.myFile;
fs.exists(imageFile.path, function(exists) {
if(exists)
{
console.log("File uploaded: " + util.inspect(imageFile));
fs.readFile(imageFile.path, function(err, imageData) {
if (err) {
res.end("Error reading your file on the server!");
}else{
//when saving an object with an image's byte array
var imageBson = {};
//var imageData = fs.readFileSync(imageFile.path);
imageBson.image = new req.mongo.Binary(imageData);
imageBson.imageType = imageFile.mimetype;
console.log("imageBson: " + util.inspect(imageBson));
req.imagesCollection.insert(imageBson, {safe: true},function(err, bsonData) {
if (err) {
res.end({ msg:'Error saving your file to the database!' });
}else{
fs.unlink(imageFile.path); // Deletes the file from the local disk
var imageBson = bsonData[0];
var imageId = imageBson._id;
res.redirect('images/' + imageId);
}
});
}
});
} else {
res.end("Oddly your file was uploaded but doesn't seem to exist!\n" + util.inspect(imageFile));
}
});

The MongoDB part isn't complicated. Once a Buffer is in the model, just let the db save it. MongoDB converts that into BinData. 80% of this code is just getting an image into and out of a PNG file.
People say don't store images in MongoDB, but icons/thumbs are tiny. Having said that, it might be a good idea to have an icons collection and only store them once using a hash of the image data as the _id.
model class example
class MyModel {
_icon: Buffer
get icon(): Buffer {
return this._icon
}
set icon(value: Buffer) {
this._icon = value
}
}
image helper
static async loadImage(url: string) {
var files = require('../lib/files')
var buffer = await files.urlContents(url, true)
return buffer
}
static async saveImage(image: Buffer, path: string) {
var files = require('../lib/files')
files.write(path, image.buffer)
return path
}
files helper
function urlResponse(url, binary) {
var request = require("request")
if (binary)
request = request.defaults({ encoding: null })
return new Promise(function (resolve, reject) {
request(url, function (error, res, body) {
if (error || res.statusCode !== 200 || body.includes('Incomplete response received from application'))
resolve({ statusCode: res?.statusCode !== 200 ? (res?.statusCode || 500) : 500 });
else
resolve(res);
});
});
}
async function urlContents(url, binary) {
var res = await urlResponse(url, binary)
if (binary)
return Buffer.from(res.body)
else
return res.body
}
function write(fileName, contents) {
fs.writeFileSync(fileName, contents)
}
mongodb helper
// ...saving
myModel.icon = loadImage('http://some.site.com/image.png')
collection.insertOne(myModel)
// ..getting
myModel = collection.findOne(query) // now myModel contains icon
saveImage(myModel.icon, '/home/toddmo/pictures/wow.png')

Related

Saving buffer/stream that comes from nodejs server

I am not sure what I am doing wrong.
I have a html content and want to save it as pdf. I use html-pdf (from npm) and a download library http://danml.com/download.html
Actually when I directly save to file or show it as a result I can get the pdf without problem. But I call my webservice method from a js method and I have a stream/buffer as a return value and saving with the 'download' library
Here is my code
pdf.create(html, options).toFile('./mypdf.pdf', function (err, res) {
if (err) return console.log(err);
console.log(res);
});
pdf.create(html,options).toBuffer(function (err, buffer) {
if (err) return reject(err);
return resolve(buffer);
});
//res.setHeader('Content-type', 'application/pdf');
pdf.create(html, options).toStream(function (err, stream) {
if (err) return res.send(err);
//res.type('pdf');
return resolve(stream);// .pipe(res);
});
I can save the content as a pdf it works fine. but when I try to send stream or buffer, somehow the page is empty. I opened the both pdf files with notepad. There are some differences. For example, probless one is 44kb the other one 78 kb. and the empty one contains also the following lines
%PDF-1.4 1 0 obj << /Title (��) /Creator (��) /Producer (�� Q t 5 .
5 . 1) /CreationDate (D:20190524152156)
endobj
I think toBuffer or toStream method has a problem in my case. Because the stream seems not bad. at least I can see that it is a pdf file (no error, just page is empty)
Anyway, here is my API router
let result = await
routerVoucher.CreatePdfStream(req.query.VoucherCode,req.query.AppName);
res.setHeader('Content-type', 'application/pdf');
res.type('pdf');
//result.pipe(res);
res.end(result, 'binary');
and here is my js consumer
$.ajax({
type: "GET",
url: '/api/vouchers/GetLicensePdf',
data:data,
success: function (pdfFile) {
if (!pdfFile)
throw new Error('There is nothing to download');
download(pdfFile,voucherCode + '.pdf', 'application/pdf')
I've solved the problem.
Firstly I converted buffer to base64
const base64 = buffer.toString('base64')
and then converted base64 to blob by using the following code
function base64toBlob (base64Data, contentType) {
contentType = contentType || '';
var sliceSize = 1024;
var byteCharacters = atob(base64Data);
//var byteCharacters = decodeURIComponent(escape(window.atob(base64Data)))
var bytesLength = byteCharacters.length;
var slicesCount = Math.ceil(bytesLength / sliceSize);
var byteArrays = new Array(slicesCount);
for (var sliceIndex = 0; sliceIndex < slicesCount; ++sliceIndex) {
var begin = sliceIndex * sliceSize;
var end = Math.min(begin + sliceSize, bytesLength);
var bytes = new Array(end - begin);
for (var offset = begin, i = 0 ; offset < end; ++i, ++offset) {
bytes[i] = byteCharacters[offset].charCodeAt(0);
}
byteArrays[sliceIndex] = new Uint8Array(bytes);
}
return new Blob(byteArrays, { type: contentType });
}
and then
again I've used my download method (from download.js library) as follow
download(new Blob([base64toBlob(base64PDF,"application/pdf")]),
voucherCode + '.pdf', "application/pdf");
then everything is fine :)

Need help parsing id3 tags asyncrhonously in node.js using id3-parser

I have a folder full of mp3s that I would like to parse using id3-parser npm module so that I can return an rss feed. The id3-parser documentation says to use this structure:
id3.parse(filebuffer).then(function(tag){
console.log(tag.title);
});
The id3-parser module uses the promise-a-plus module. The problem with my code is that the tag gets read at the very end of the stack. Here is my test code:
var id3 = require('id3-parser');
var fs = require('fs');
fs.readdir("podcasts", function (err, files) {
files.forEach( function (file) {
var stats = fs.statSync("podcasts/" + file);
if (stats.isFile()) {
var fbuff = fs.readFileSync("podcasts/" + file);
id3.parse(fbuff).then( function (tag) {
console.log("***"+tag.title);
});
console.log("Parsed id3 tag " + file);
}
console.log("Done file");
});
console.log("Done readdir");
});
The output is:
Parsed id3 tag test.mp3
Done file
Done readdir
***Title tag
I cannot for the life of me figure out how to properly wait for the tag to be parsed. Parsing should happen before I put it into the rss feed (have no trouble using the rss module to create that xml feed -- code not shown). I have been reading about promises and I am missing something. I am sure I just need to completely refactor my code, but not sure how to read every file (currently using readdir and forEach) and add each file to the feed.
Try this:
var id3 = require('id3-parser')
var fs = require('fs')
var path = require('path')
function promiseFromCallback (fn) {
return new Promise(function (resolve, reject) {
return fn(function (err, result) {
return err ? reject(err) : resolve(result)
})
})
}
function dirID3 (directory) {
return promiseFromCallback(function (gotDir) {
return fs.readdir(directory, gotDir)
}).then(function (files) {
return Promise.all(files.map(function (file) {
var filePath = path.join(directory, file)
return promiseFromCallback(function (gotStat) {
return fs.stat(filePath, gotStat)
}).then(function (stat) {
return stat.isFile() ? promiseFromCallback(function (gotBuffer) {
return fs.readFile(filePath, gotBuffer)
}).then(function (fbuff) {
return id3.parse(fbuff)
}).then(function (tag) {
return { path: filePath, tag: tag }
}) : null
})
}))
}).then(function (id3Items) {
var result = {}
id3Items.forEach(function (id3Item) {
if (id3Item) result[id3Item.path] = id3Item.tag
})
return result
})
}
Then use it like this:
dirID3('podcasts').then(function (dirInfo) {
console.log('ID3 info for directory', dirInfo)
})

How to get binary data from ng-file-upload file object?

I'm attempting to use the ng-file-upload directive to provide file upload functionality in my angular app.
I've got it working for the most part - I can select multiple files and loop through to grab the file name and file types. I just can't seem to figure out where the actual binary data of each file is stored in the file object.
I tried using the approach outlined in this post - AngularJS Upload a file and send it to a DB, but that results in a an error that "$q is not defined".
function create_blob(file) {
var deferred = $q.defer();
var reader = new FileReader();
reader.onload = function () {
deferred.resolve(reader.result);
};
reader.readAsDataURL(file);
return deferred.promise;
}
So then I tried the approach outlined in this post - Send an uploaded image to the server and save it in the server, but again I'm running into an error reading "dataURI.split is not a function".
function dataURItoBlob(dataURI) {
var binary = atob(dataURI.split(',')[1]);
var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
var array = [];
for (var i = 0; i < binary.length; i++) {
array.push(binary.charCodeAt(i));
}
return new Blob([new Uint8Array(array)], {
type: mimeString
});
}
The code I'm using is as follows:
function create_blob(file) {
var deferred = $q.defer();
var reader = new FileReader();
reader.onload = function () {
deferred.resolve(reader.result);
};
reader.readAsDataURL(file);
return deferred.promise;
}
function dataURItoBlob(dataURI) {
var binary = atob(dataURI.split(',')[1]);
var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
var array = [];
for (var i = 0; i < binary.length; i++) {
array.push(binary.charCodeAt(i));
}
return new Blob([new Uint8Array(array)], {
type: mimeString
});
}
$scope.uploadFiles = function (files) {
$scope.files = files;
angular.forEach(files, function (file) {
if (file && !file.$error) {
//var reader = new FileReader();
//console.log(reader.readAsDataURL(file));
//var binary = create_blob(file);
var fileBinary = dataURItoBlob(file);
$http({
url: root + '/DesktopModules/ServiceProxy/API/NetSuite/InsertCaseFile',
method: "POST",
//headers: { 'caseId': id, 'fileName': file.name, fileContent: $.base64.encode(file) }
headers: { 'caseId': id, 'fileName': file.name, fileContent: fileBinary }
}).
success(function (data, status, headers, config) {
//if (data == true) {
// getCase();
// $scope.newMessage = "";
// //toaster.pop('success', "", "Message succesfully submitted.",0);
//}
}).
error(function (data, status, headers, config) {
});
file.upload.progress(function (evt) {
file.progress = Math.min(100, parseInt(100.0 * evt.loaded / evt.total));
});
}
});
}
What am I overlooking?
It depends on what format your DB is accepting for file upload. If it support multipart form data, then you can just use
Upload.upload({file: file, url: my/db/url}).then(...);
if it accepts post requests with file's binary as content of the request (like CouchDB, imgur, ...) then you can do
Upload.http({data: file, url: my/db/url, headers: {'Content-Type': file.type}})...;
if you db just accept json objects and you want to store the file as base64 data url in the database like this question then you can do
Upload.dataUrl(file, true).then(function(dataUrl) {
$http.post(url, {
fileBase64DataUrl: dataUrl,
fileName: file.name,
id: uniqueId
});
})

Secondary tile web url

I have to pin secondary tile in my windows phone 8.1 application.
I followed the msdn tutorial : http://code.msdn.microsoft.com/windowsapps/secondary-tiles-sample-edf2a178/
It does work with internal image (ms-appx://.. ) but not with web url (http://)
working sample:
var logo = new Windows.Foundation.Uri("ms-appx:///Images/square30x30Tile-sdk.png");
var currentTime = new Date();
var TileActivationArguments = data.ad_id + " WasPinnedAt=" + currentTime;
var tile = new Windows.UI.StartScreen.SecondaryTile(data.ad_id,
data.subject,
TileActivationArguments,
logo,
Windows.UI.StartScreen.TileSize.square150x150);
tile.visualElements.foregroundText = Windows.UI.StartScreen.ForegroundText.light;
tile.visualElements.square30x30Logo = logo;
tile.visualElements.showNameOnSquare150x150Logo = true;
var selectionRect = this.element.getBoundingClientRect();
// Now let's try to pin the tile.
// We'll make the same fundamental call as we did in pinByElement, but this time we'll return a promise.
return new WinJS.Promise(function (complete, error, progress) {
tile.requestCreateForSelectionAsync({ x: selectionRect.left, y: selectionRect.top, width: selectionRect.width, height: selectionRect.height }, Windows.UI.Popups.Placement.above).done(function (isCreated) {
if (isCreated) {
complete(true);
} else {
complete(false);
}
});
});
And if I use
var logo = new Windows.Foundation.Uri(data.images[0]);
I got an invalid parameter exception.
You can take a look at the documentation for the SecondaryTile.Logo property. In it you'll see this:
The location of the image. This can be expressed as one of these schemes:
ms-appx:///
ms-appdata:///local/
You can download the image first and then set it using the ms-appdata:///local/ scheme. I'm not sure that changing the logo with something from the Internet is a good idea, though. This should be the app's logo, so it should be in the package.
I found the solution
fileExists: function (fileName) {
var applicationData = Windows.Storage.ApplicationData.current;
var folder = applicationData.localFolder;
return folder.getFileAsync(fileName).then(function (file) {
return file;
}, function (err) {
return null;
});
},
download: function (imgUrl, imgName) {
return WinJS.xhr({ url: imgUrl, responseType: "blob" }).then(function (result) {
var blob = result.response;
var applicationData = Windows.Storage.ApplicationData.current;
var folder = applicationData.localFolder;
return folder.createFileAsync(imgName, Windows.Storage.
CreationCollisionOption.replaceExisting).then(function (file) {
// Open the returned file in order to copy the data
return file.openAsync(Windows.Storage.FileAccessMode.readWrite).
then(function (stream) {
return Windows.Storage.Streams.RandomAccessStream.copyAsync
(blob.msDetachStream(), stream).then(function () {
// Copy the stream from the blob to the File stream
return stream.flushAsync().then(function () {
stream.close();
});
});
});
});
}, function (e) {
//var msg = new Windows.UI.Popups.MessageDialog(e.message);
//msg.showAsync();
});
},
var self = this;
this.download(data.images[0], data.ad_id).then(function () {
self.fileExists(data.ad_id).then(function (file) {
var logo = new Windows.Foundation.Uri("ms-appdata:///Local/" + data.ad_id);
....
I need to download the image, store it and then I can use ms-appdata:///Local

How to render image as pdf (canvas and pdfkit)

I have a function on nodejs that generates an image from many images and then generate a pdf file from that. Im trying with just one image but i need to add more, but this doesnt seems to work
function HelperHandler() {
this.pdf = function(req, res, next) {
var doc = new PDFDocument;
mergeImages(function(err, image) {
if (err)
return res.json(err);
doc.image(image, 100, 100);
doc.output(function(string) {
res.contentType = "application/pdf";
res.send(string);
});
})
}
}
var mergeImages = function(callback) {
var Canvas = require("canvas")
, fs = require("fs");
fs.readFile(global.root_path + "/images/bg.jpg", function(err, data) {
if (err)
callback("error loading image");
else {
var canvas = new Canvas(408, 939)
, img = new Canvas.Image(data);
ctx = canvas.getContext("2d");
img.onload = function() {
ctx.drawImage(img, 0, 0, 408, 939);
}
canvas.toDataURL('image/png', function(err, str) {
callback(null, str);
});
}
});
}
Error
Error: ENAMETOOLONG, name too long 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZgAAAOrCAYAAABqSpB/AAAABmJLR0QA/wD/AP+gvaeTAAAF5UlEQVR4nO3BMQEAAADCoPVPbQo/oAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAICjAWckAAHF4EUcAAAAAElFTkSuQmCC'
at Object.openSync (fs.js:427:18)
at Object.readFileSync (fs.js:284:15)
at Function.open (/Users/jtomasrl/code/app/server/node_modules/pdfkit/js/image.js:27:28)
at PDFDocument.image (/Users/jtomasrl/code/app/server/node_modules/pdfkit/js/mixins/images.js:27:26)
at /Users/jtomasrl/code/app/server/lib/handler/current/helper.js:15:11
at /Users/jtomasrl/code/app/server/lib/handler/current/helper.js:41:9
at /Users/jtomasrl/code/app/server/node_modules/canvas/lib/canvas.js:217:7
You can use a buffer or a path with PDFKit image.
But you can't use a base64 URL, you need to decode this string to a buffer.
To use base64 data:
doc.image(new Buffer(image.replace('data:image/png;base64,',''), 'base64'), 100, 100); // this will decode your base64 to a new buffer
More information on base64 de/encode with Node Buffer here.

Resources