Convert a file using FFMPEG and upload to AWS S3 Nodejs - node.js

Hey everyone so quick question I want to allow a user to upload a WebM file and convert it using FFmpeg to mp4. I am using Nodejs for the backend and already have a route that uploads files to Amazon S3 file storage. But let's say I wanted to send that file and not store it anywhere but convert it to mp4 from the request itself is that possible? If not is it possible to take an s3 file URL and convert it to mp4? Can anybody point me in the right direction as to what is possible and the best way to do this?
basically all I want to do is
const objectUrl = createObjectURL(Blob);
ffmpeg -i objectURL S3OutputLocation
or
ffmpeg -i myS3InputLocation myS3OutputLocation

Okay so there is a couple of things you have to do in order to make this work.
1. you need to set up a local instance of multer as you need to upload the file locally before going to s3. I tried to do it with s3 directly but it seemed to use a lot of costly and time-consuming read operations that took much longer than writing the file to the server first. I found this to be the best solution.
You do this like so:
const localStorage = multer.diskStorage({
destination: function(req, file, cb) {
const destination = __dirname + "\\canvas-uploads";
console.log("destination", destination);
cb(null, destination);
},
filename: function(req, file, cb) {
const filename = req.body.id + "." + file.mimetype.toString().slice(file.mimetype.toString().lastIndexOf("/") + 1);
console.log("filename", filename);
cb(null, filename);
}
});
const uploadLocal = multer({
storage: localStorage
});
You need to set up ffmpeg-fluent and wrap it in a promise so you can be sure it's finished with all your processing (uploading to s3 and etc in the same route.) for convenience.
you do this like so:
router.post('/upload-temp', uploadLocal.array("upload"), async(req, res, next) =>{
res.json({id: req.body.id});
});
router.post('/ffmpeg', async(req, res, next) => {
try {
const reqPath = path.join(__dirname, '../upload/canvas-uploads/');
const {id, type} = req.body;
const localFileInput = `${reqPath}${id}.webm`;
const localFileOutput = `${reqPath}${id}.${type}`;
console.log("localInput", localFileInput);
console.log("localOutput", localFileOutput);
const key = `canvas/${id}.${type}`;
await new Promise((resolve, reject) => {
ffmpeg().input(localFileInput)
.withOutputFormat(type)
.output(localFileOutput)
.on('end',async ()=> {
const fileContent = await fs.readFileSync(localFileOutput);
await fs.unlinkSync(localFileInput);
await fs.unlinkSync(localFileOutput);
const params = {
Bucket: process.env.BUCKET_NAME,
Key: key,
Body: fileContent
}
await s3.putObject(params).promise();
resolve();
}).run();
})
res.send("success")
} catch(error){
console.log(error);
res.send(error);
}
});

Related

Folder /tmp is not working in Vercel Production. Giving error of EORFs

Folder Structure image# Multer.js File
const multer = require("multer");
const path = require("path");
const fs = require("fs");
const httpStatus = require("http-status");
const ApiError = require("../utils/ApiError")
const logger = require("../utils/logger");
const multerUpload = async (req, res, next) => {
let fileName = "";
let storage = multer.diskStorage({
destination: function (req, file, callback) {
fs.mkdir(path.join(path.resolve(), "/tmp"), (err) => {
if (err) {
logger.error("mkdir tmp %o", err);
}
callback(null, path.join(path.resolve(), "/tmp"));
});
},
filename: function (req, file, callback) {
fileName = file.fieldname + "-" + req.query.eventId + Date.now() + path.extname(file.originalname);
logger.info("filename of uploadSheet===> %s", fileName);
callback(null, fileName);
},
});
// below code is to read the added data to DB from file
var upload = multer({
storage: storage,
fileFilter: function (req, file, callback) {
var ext = path.extname(file.originalname);
if (ext !== '.xlsx') {
return callback(new Error('Only Excel sheets are allowed'))
}
callback(null, true)
},
}).single("sheet");
upload(req, res, async function (err) {
if (err) {
next(new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message));
} else {
req.fileName = fileName;
next();
}
})
}
module.exports = multerUpload;
It gives error of EORFS read only file in vercel production but the code works fine in local.
I'm trying to upload the excel sheet file from the Api and then read the data from it and add it into the Mongodb.
I once encountered this same problem working with Heroku a long time ago, I haven't worked with vercel but with quick research, I will say this is the cause, vercel does not provide storage for you to upload files to in production, you need a separate service for that like Amazon S3, but there also exists Azure File Storage and Google Cloud Storage.
alternatively, if you don't want to add more services to your project, you can just convert the image to base64 string and save it as text(but need to make the field/column read-only so it does not get corrupted) NOT the best alternative but it was something I once did
To use /tmp in server functions, you should just use /tmp/your-file. Remove path.resolve().
Only if you need to store something temporarily, you may try to use /tmp directory.
Limit 512 MB + no guaranty - https://github.com/vercel/vercel/discussions/5320

Writing file in /tmp in a Firebase Function does not work

I am writing a Firebase function that exposes an API endpoint using express. When the endpoint is called, it needs to download an image from an external API and use that image to make a second API call. The second API call needs the image to be passed as a readableStream. Specifically, I am calling the pinFileToIPFS endpoint of the Pinata API.
My Firebase function is using axios to download the image and fs to write the image to /tmp. Then I am using fs to read the image, convert it to a readableStream and send it to Pinata.
A stripped-down version of my code looks like this:
const functions = require("firebase-functions");
const express = require("express");
const axios = require("axios");
const fs = require('fs-extra')
require("dotenv").config();
const key = process.env.REACT_APP_PINATA_KEY;
const secret = process.env.REACT_APP_PINATA_SECRET;
const pinataSDK = require('#pinata/sdk');
const pinata = pinataSDK(key, secret);
const app = express();
const downloadFile = async (fileUrl, downloadFilePath) => {
try {
const response = await axios({
method: 'GET',
url: fileUrl,
responseType: 'stream',
});
// pipe the result stream into a file on disc
response.data.pipe(fs.createWriteStream(downloadFilePath, {flags:'w'}))
// return a promise and resolve when download finishes
return new Promise((resolve, reject) => {
response.data.on('end', () => {
resolve()
})
response.data.on('error', () => {
reject()
})
})
} catch (err) {
console.log('Failed to download image')
console.log(err);
throw new Error(err);
}
};
app.post('/pinata/pinFileToIPFS', cors(), async (req, res) => {
const id = req.query.id;
var url = '<URL of API endpoint to download the image>';
await fs.ensureDir('/tmp');
if (fs.existsSync('/tmp')) {
console.log('Folder: /tmp exists!')
} else {
console.log('Folder: /tmp does not exist!')
}
var filename = '/tmp/image-'+id+'.png';
downloadFile(url, filename);
if (fs.existsSync(filename)) {
console.log('File: ' + filename + ' exists!')
} else {
console.log('File: ' + filename + ' does not exist!')
}
var image = fs.createReadStream(filename);
const options = {
pinataOptions: {cidVersion: 1}
};
pinata.pinFileToIPFS(image, options).then((result) => {
console.log(JSON.stringify(result));
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Authorization, Origin, X-Requested-With, Accept");
res.status(200).json(JSON.stringify(result));
res.send();
}).catch((err) => {
console.log('Failed to pin file');
console.log(err);
res.status(500).json(JSON.stringify(err));
res.send();
});
});
exports.api = functions.https.onRequest(app);
Interestingly, my debug messages tell me that the /tmp folder exists, but the file of my downloaded file does not exist in the file system.
[Error: ENOENT: no such file or directory, open '/tmp/image-314502.png']. Note that the image can be accessed correctly when I manually access the URL of the image.
I've tried to download and save the file using many ways but none of them work. Also, based on what I've read, Firebase Functions allow to write and read temp files from /tmp.
Any advice will be appreciated. Note that I am very new to NodeJS and to Firebase, so please excuse my basic code.
Many thanks!
I was not able to see you are initializing the directory as suggested in this post:
const bucket = gcs.bucket(object.bucket);
const filePath = object.name;
const fileName = filePath.split('/').pop();
const thumbFileName = 'thumb_' + fileName;
const workingDir = join(tmpdir(), `${object.name.split('/')[0]}/`);//new
const tmpFilePath = join(workingDir, fileName);
const tmpThumbPath = join(workingDir, thumbFileName);
await fs.ensureDir(workingDir);
Also, please consider that if you are using two functions, the /tmp directory would not be shared as each one has its own. Here is an explanation from Doug Stevenson. In the same answer, there is a very well explained video about local and global scopes and how to use the tmp directory:
Cloud Functions only allows one function to run at a time in a particular server instance. Functions running in parallel run on different server instances, which have different /tmp spaces. Each function invocation runs in complete isolation from each other. You should always clean up files you write in /tmp so that they don't accumulate and cause a server instance to run out of memory over time.
I would suggest using Google Cloud Storage extended with Cloud Functions to achieve your goal.

POSTing from Angular to Node. Getting empty body on node

Im posting two fields in Angular to a NodeJs endpoint.
I usually post on the body and everything is perfect at Node, but this time, I have to post a form to upload a file.
So this is my code for posting the form data (Angular side):
var APIURL = sessionStorage.getItem('endPoint') + "profile/updateeocoverage";
let formData = new FormData();
formData.append("data", JSON.stringify(this.payLoad));
if (this.eofile) {
formData.append("myfile", this.eofile, this.eofile.name);
}
this.httpClient.post(APIURL, formData).subscribe(
result => {
....
My problem is that I always retrieved the body at node as follows:
router.post('/updateeocoverage', async (req, res, next) => {
console.log(req.body)
return;
....
But with the method Im using now in Angular, req.body is retrieving {}
Is the POST wrong, or the router wrong at Node side?
Thanks.
UPDATE ON PABLO'S ANSWER BELOW (SOLUTION PROVIDED) for whoever runs into this issue:
Using Multer solved the problem, but as he said some workaround is needed as to set the file name, but most important it is required to authenticate the user, so:
const multer = require('multer')
const path = require('path')
To authenticate the user, I send the authentication parameters on the header. Sending it as formdata.append didn't work for me. This sets true or false to upload the file, otherwise anyone can upload anything to the route:
async function authenticateUser(req, file, cb) {
let tempcred = JSON.parse(req.headers.data)
let credentials = tempcred.credentials;
let userData = await utils.isValidUser((credentials), false);
if (userData.isValid == false) {
cb(null, false)
return;
}
else {
cb(null, true)
}
}
Then, since Multer uploads the file with a random name, and I need to save it with the user ID name and the file extension, I do the following:
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, 'uploads/eofiles/')
},
filename: async function (req, file, cb) {
let tempcred = JSON.parse(req.headers.data)
let credentials = tempcred.credentials;
let userid = await utils.decrypt(credentials.userid, process.env.SECRET);
cb(null, userid + path.extname(file.originalname))
}
});
Finally, I declare the upload variable for using it with Multer:
var upload = multer({ storage: storage, fileFilter: authenticateUser })
And set the router:
router.post('/updateeofile', upload.single("myfile"), async (req, res, next) => {
let filename = req.file.filename //gets the file name
...
...
do my stuff, save on database, etc
...
...
});
For the record, "myfile" is the input file id.
And this is how I upload the file from Angular:
var APIURL = sessionStorage.getItem('endPoint') + "eoset/updateeofile";
const httpOptions = {
headers: new HttpHeaders({
'data': `${JSON.stringify(this.payLoad)}`
})
};
let formData = new FormData();
if (this.eofile) {
formData.append("myfile", this.eofile, this.eofile.name);
}
this.httpClient.post(APIURL, formData, httpOptions).subscribe(
result => {
...
...
...
},
error => {
});
I spent 6 hours on this today. I hope this helps you and saves you some time.
Try using Multer
npm i multer
const multer = require('multer')
const upload = multer({ dest: 'uploads/' })
router.post('/updateeocoverage', upload.single('myfile'), function (req, res, next) {
res.json(JSON.parse(req.body.data))
})
You will need to indicate the destination folder (line 2) and work out the file name, extension, etc.
First thing to do is ensure that the body isn't empty before making the request, so do a console.log before the request:
console.log(formData);
this.httpClient.post(APIURL, formData).subscribe(
result => {
....
After that, if the body is filled correctly, try to put the body like this on the request method:
this.httpClient.post(APIURL, {formData})
or
this.httpClient.post(APIURL, {
"field1": formData.field1,
...
})
and if these two things don't correct your issue, problaby you have something wrong on back-end side

Nodejs save uploaded file

I have an application and in that application, I want to use some file upload mechanism.
My requirement is:
Once the file is uploaded, its name will be changed to something unique, like uuid4(). I will store this name in the database later.
I have written something like that, however I have a couple of questions:
const multer = require('multer');
const upload = multer();
router.post('/', middleware.checkToken, upload.single('file'), (req,res,next)=>{
// key:
// file : "Insert File Here"
console.log("req:");
console.log(req.file);
const str = req.file.originalname
var filename = str.substring(0,str.lastIndexOf('.'));
// I will use filename and uuid for storing it in the database
// I will generate unique uuid for the document and store the document
// with that name
var extension = str.substring(str.lastIndexOf('.') + 1, str.length);
// HERE!
res.status(200).json();
})
I have seen examples of storing it in the diskStorage:
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage })
However, as far as I understood, this is a configuration outside of the API call. Meaning that I cannot modify it every time I call this API. I want to assign different names to the file, and I need that name(uuid) to save that name in the database.
How can I preserve such functionality?
Thanks to #Rashomon and #Eimran Hossain Eimon, I have solved the issue. In case of anyone wonders the solution, here it is:
const multer = require('multer');
var storage = multer.diskStorage({
destination: function (req, file, cb) {
// the file is saved to here
cb(null, '/PATH/TO/FILE')
},
filename: function (req, file, cb) {
// the filename field is added or altered here once the file is uploaded
cb(null, uuidv4() + '.xlsx')
}
})
var upload = multer({ storage: storage })
router.post('/', middleware.checkToken, upload.single('file'), (req,res,next)=>{
// the file is taken from multi-form and the key of the form must be "file"
// visible name of the file, which is the original, uploaded name of the file
const name = req.file.originalname;
// name of the file to be stored, which contains unique uuidv4
const fileName = req.file.filename;
// get rid of the extension of the file ".xlsx"
const file_id = fileName.substring(0, fileName.lastIndexOf('.'));
// TODO
// Right now, only xlsx is supported
const type = "xlsx";
const myObject = new DatabaseObject({
_id : new mongoose.Types.ObjectId(),
file_id: file_id,
name : name,
type: "xlsx"
})
myObject .save()
.then(savedObject=>{
// return some meaningful response
}).catch(err=>{
// return error response
})
})
This solves my current issue. Thanks for helping. For future improvements, I'll add error case for:
In case uuidv4 returns an id which already exists(I believe it is highly unlikely since the object contains some timestamp data), rerun the renaming function.
In case there is an error in saving to the database, I should delete the uploaded file to avoid future conflicts.
If you have solutions for those problems too, I'm much appreciated.
I think you got it wrong... you said that
I cannot modify it every time I call this API.
But actually, the filename is called every time for every file. Let me explain this portion of code...
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
Here look at the callback function (denoted by cb) :
1st argument null in callback function is like convention. You always pass null as the first argument in a callback function. See this Reference
2nd argument determines what the file should be named inside the destination folder. So, here you can specify any function, which can return you a unique filename every time.
Since you are using mongoose...
I think it would be better if you implement your function uniqueFileName() using the mongoose method within your Schema (in which you want to save the file path) and call it in your route handler. Learn More
No need. Because you're using timestamps.
In case of an error in saving to the database, you can delete the uploaded file using this code to avoid future conflicts. Try this:
const multer = require('multer');
const fs = require('fs'); // add this line
var storage = multer.diskStorage({
destination: function (req, file, cb) {
// the file is saved to here
cb(null, '/PATH/TO/FILE')
},
filename: function (req, file, cb) {
// the filename field is added or altered here once the file is uploaded
cb(null, uuidv4() + '.xlsx')
}
})
var upload = multer({ storage: storage })
router.post('/', middleware.checkToken, upload.single('file'), (req,res,next)=>{
// the file is taken from multi-form and the key of the form must be "file"
// visible name of the file, which is the original, uploaded name of the file
const name = req.file.originalname;
// name of the file to be stored, which contains unique uuidv4
const fileName = req.file.filename;
// get rid of the extension of the file ".xlsx"
const file_id = fileName.substring(0, fileName.lastIndexOf('.'));
// TODO
// Right now, only xlsx is supported
const type = "xlsx";
const myObject = new DatabaseObject({
_id : new mongoose.Types.ObjectId(),
file_id: file_id,
name : name,
type: "xlsx"
})
myObject .save()
.then(savedObject=>{
// return some meaningful response
}).catch(err=>{
// add this
// Assuming that 'path/file.txt' is a regular file.
fs.unlink('path/file.txt', (err) => {
if (err) throw err;
console.log('path/file.txt was deleted');
});
})
})
also see NodeJS File System Doc

Given the local relative url to an image, how can I get actual image to send

I am writing an express app where one endpoint takes in an image and I would then send that image to a third-party Api. I used multer to save the image to disk, and I have the relative file path to the image, but the API I am trying to call needs the actual image or an image url. The problem is that i'm passing just a string value that contains the filepath to the image. Here is what I have currently:
var multer = require('multer')
var storage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, './uploads/');
},
filename: function(req, file, cb) {
cb(null, Date.now() + file.originalname);
}
});
var upload = multer({ storage: storage });
And I need to call the image in this api call here:
router.post('/images/tags/info/image', upload.single('fileName'), function(req, res, next) {
console.log(req.file.path);
var imageUrl = req.file.path;
app.models.predict('APP_KEY', imageUrl).then(
function (response) {
var responseJson = JSON.stringify(response.data.outputs[0].data.concepts);
var data = collectTags(responseJson);
data.then(function(value) {
res.json(value);
});
},
function (err) {
console.log(err);
});
});
When I console.log(req.file.filePath), I get a valid filepath, but I need the actual image to pass into the app.models.predict(API_KEY, image)
You can just read the image from the file system and send it to the api.
e.g.
const bitmap = await q.nfcall(fs.readFile,req.file.path);
//remove the temp image
fs.unlink(req.file.path);
const buffer = new Buffer(bitmap).toString('base64');
app.models.predict('APP_KEY', buffer);
Example sending the image as base64 (using async/await).

Resources