I am trying to attach files to sendgrid without storing them to the disk. I want to use stream to handle that.
var multer = require('multer');
var upload = multer({ storage: multer.memoryStorage({})});
mail = new helper.Mail(from_email, subject, to_email, content);
console.log(req.body.File);
attachment = new helper.Attachment(req.body.File);
mail.addAttachment(attachment)
I think is not possible using streams because:
multer library with MemoryStorage stores the entire file contents on memory in a Buffer object (not a Stream)
Sendgrid library doesn't support Readable streams as input.
But you can achieve it using the returned buffer as attachment like:
var multer = require('multer')
, upload = multer({ storage: multer.memoryStorage({})})
, helper = require('sendgrid').mail;
app.post('/send', upload.single('attachment'), function (req, res, next) {
// req.file is the `attachment` file
// req.body will hold the text fields, if there were any
var mail = new helper.Mail(from_email, subject, to_email, content)
, attachment = new helper.Attachment()
, fileInfo = req.file;
attachment.setFilename(fileInfo.originalname);
attachment.setType(fileInfo.mimetype);
attachment.setContent(fileInfo.buffer.toString('base64'));
attachment.setDisposition('attachment');
mail.addAttachment(attachment);
/* ... */
});
It may affects memory usage (out of memory errors) if used with big attachments or high concurrency.
Related
I'm using Multer/multer-storage-cloudinary to upload images directly to Cloudinary rather first uploading it to a local temp directory, then sending it to Cloudinary:
const express = require('express');
const router = express.Router({mergeParams:true});
if (app.get('env') == 'development'){ require('dotenv').config(); }
const crypto = require('crypto');
const cloudinary = require('cloudinary').v2;
const { CloudinaryStorage } = require('multer-storage-cloudinary');
const multer = require('multer');
const { storage } = require('../cloudinary');
const upload = multer({storage});
//configure cloudinary upload settings
cloudinary.config({
cloud_name:process.env.CLOUDINARY_NAME,
api_key: process.env.CLOUDINARY_API_KEY,
api_secret: process.env.CLOUDINARY_API_SECRET
});
const storage = new CloudinaryStorage({
cloudinary: cloudinary,
folder: ('book_tracker/'+process.env.CLOUDINARY_FOLDER+'posts'),
allowedFormats: ['jpeg', 'jpg', 'png'],
filename: function (req, file, cb) {
let buf = crypto.randomBytes(16);
buf = buf.toString('hex');
let uniqFileName = file.originalname.replace(/\.jpeg|\.jpg|\.png/ig, '');
uniqFileName += buf;
console.log(req.body);
cb(undefined, uniqFileName );
}
});
const middleware = {
function asyncErrorHandler: (fn) =>
(req, res, next) => {
Promise.resolve(fn(req, res, next))
.catch(next);
}
}
/* POST create user page */
router.post('/register', upload.single('image'), asyncErrorHandler(postRegister));
What I'm running into is that the response I'm getting in req.file is not the full Cloudinary response which includes public_id, etc. Instead it's like this:
{
fieldname: 'image',
originalname: 'My Headshot.jpg',
encoding: '7bit',
mimetype: 'image/jpeg',
path: 'https://res.cloudinary.com/<cloudinary_name>/image/upload/v1611267647/<public_id>.jpg',
size: 379632,
filename: '<public_id>'
}
It's been a while since I worked with multer-storage-cloudinary, though that storage was taken directly from an old project that would return the correct information. Is there something in multer, or multer-storage-cloudinary, that I need to set in order to put the full cloudinary response into req.file?
The multer-storage-cloudinary package is a third party package that integrates multer and Cloudinary in a streamlined way, but it doesn't expose all possible options or responses from the Cloudinary SDK or API
In your example, it's not returning the full API response from Cloudinary, but a subset of the fields, because the file object's filename, path, and size properties are taken from the Cloudinary API response (from the public_id, secure_url, and bytes properties of the API response respectively), but the other fields aren't mapped: https://github.com/affanshahid/multer-storage-cloudinary#file-properties
If you need the full set of response values (or some specific values not mapped already) you can:
Ask the package maintainer to add support for other fields
Fork the package and map additional fields yourself; the fields are mapped here though I'm not sure what else may need to be changed: https://github.com/affanshahid/multer-storage-cloudinary/blob/1eb903d44ac6dd42eb1ab655b1e108acd97ed4ca/src/index.ts#L83-L86
Switch from using that package for wrapping the Cloudinary SDK to use the Cloudinary SDK directly in your own code, so you can handle the response directly.
Leave it as it is now and make a separate call to the Cloudinary Admin API to fetch the other details of the image(s): https://cloudinary.com/documentation/admin_api#get_resources
Leave it as-is, but add a notification_url so that as well as the API call response, the details of the new upload will be sent in an HTTP POST request to a URL of your choice: https://cloudinary.com/documentation/notifications
The notification_url can be specified in the Upload API call, Upload Preset, or at the account-level in the Cloudinary account settings.
I'm trying to upload a large file (7GB) to my server. For this I'm using multer:
const express = require('express');
const multer = require('multer');
const {
saveLogFile,
} = require('../controller/log');
const router = express.Router();
const upload = multer();
router.post('/', upload.single('file'), saveLogFile);
In my saveLogFile controller, which is of format saveLogFile = async (req,res) => { ... } I want to get req.file. The multer package should give me the uploaded file with req.file. So when I try to upload small files (<2GB) It goes successfully. But when I try to upload files over 2GB, I get the following error:
buffer.js:364
throw new ERR_INVALID_OPT_VALUE.RangeError('size', size);
^
RangeError [ERR_INVALID_OPT_VALUE]: The value "7229116782" is invalid for option "size"
How can I bypass it? Actually, All I need is access for the uploaded file in my saveLogFile Controller.
The reason for this is probably that node will run out of memory as your using multer without passing any options. From the docs:
In case you omit the options object, the files will be kept in memory
and never written to disk.
Try using the dest or storage option in order to use a temporary file for the upload:
const upload = multer({ dest: './some-upload-folder' });
router.post('/', upload.single('file'), saveLogFile);
That I would like to convert into a byte array.
I get the file with <v-file-input> and this is the input of it.
I would like to convert it on the client side, then send it to the backend to be uploaded to the sql server.
I've tried to search it on google for hours.
Try to add this file object to FormData and send it to nodejs. On a server side you can use multer to decode multipart formdata to req.file for instance
on a client side:
const formData = new FormData()
formData.append('file', file)
const { data: result } = await axios.post(`/api/upload-image`, formData)
on a server side:
const multer = require('multer')
const upload = multer()
...
router.post('/upload-image', upload.single('file'), uploadImageFile)
...
uploadImageFile(req, res) {
// multer writes decoded file content to req.file
const byteContent = req.file
res.end()
}
I am trying to receive a file on Node.js Express framework (via express-fileupload middleware), and then POST it directly to another server via the request package without saving it on the first device's disk and then re-reading it.
const streamifier = require('streamifier');
const fileUpload = require('express-fileupload');
app.use(fileUpload());
app.post('/upload', function(req, res) {
var fileBuffer = req.files.upload.data;
var fileReadStream = streamifier.createReadStream(fileBuffer);
//Use 'request' to send fileReadStream to another API for additional processing
}
However, the receiving API does not receive any file when I run this code.
When I run this other code, however, it works fine, but fs is only able to create a ReadStream from a file on the server, so it would involve me saving the file to some temporary hard disk location first, reading it again, and then sending it to the remote API, which I think is a bit wasteful.
var fs = require('fs');
app.post('/upload', function(req, res) {
var fileReadStream = fs.createReadStream('test.txt');
//Use 'request' to send fileReadStream to another API for additional processing
}
Any idea why the 1st code block results in no file received by the external API, while the 2nd one works perfectly fine? It seems that fileReadStream would be the same in both cases.
Turns out I should have just been sending the buffer by itself without transforming it into a ReadStream. However, I did have to add the name to it, or the receiving API wouldn't recognize that I was sending a file.
const fileUpload = require('express-fileupload');
app.use(fileUpload());
app.post('/upload', function(req, res) {
var fileBuffer = req.files.upload.data;
fileBuffer.name = 'test.png';
//Use 'request' to send fileBuffer to another API for additional processing
}
Is there a way to extract file from http req without saving it on server ?
as there are some parser like Multer and Multiply , but they save file on server location first.
Can we parse file in node.js as we do in c# .Net
Request.Files
Resolved by using Multer in memory storage mode.
Like:-
var storage = multer.memoryStorage();
var upload = multer({ storage: storage });
this will keep file in buffer
and you can extract file like this
app.post('/upload', upload.any(), function(req, res) {
console.log(req.files);
});