Upload screenshots to google cloud storage bucket with Fluent-ffmpeg - node.js

I am currently using multer to upload videos to my google storage bucket, and fluent-ffmpeg to capture thumbnails of the videos. Videos are being uploaded into the buckets correctly, but not the thumbnails from ffmpeg. How can I change the location of the thumbnails to my google storage bucket?
Back-End Video upload
require ('dotenv').config()
const express = require('express');
const router = express.Router();
const multer = require("multer");
var ffmpeg = require('fluent-ffmpeg');
const multerGoogleStorage = require('multer-google-storage');
const { Video } = require("../models/Video");
const {User} = require("../models/User")
const { auth } = require("../middleware/auth");
var storage = multer({
destination: function (req, file, cb) {
cb(null, 'videos/')
},
filename: function (req, file, cb) {
cb(null, `${Date.now()}_${file.originalname}`)
},
fileFilter: (req, file, cb) => {
const ext = path.extname(file.originalname)
if (ext !== '.mp4' || ext !== '.mov' || ext !== '.m3u' || ext !== '.flv' || ext !== '.avi' || ext !== '.mkv') {
return cb(res.status(400).end('Error only videos can be uploaded'), false);
}
cb(null,true)
}
})
// Set location to google storage bucket
var upload = multer({ storage: multerGoogleStorage.storageEngine() }).single("file")
router.post("/uploadfiles", (req, res) => {
upload(req, res, err => {
if (err) {
return res.json({sucess: false, err})
}
return res.json({ success: true, filePath: res.req.file.path, fileName: res.req.file.filename})
})
});
Back-end thumbnail upload
router.post("/thumbnail", (req, res) => {
let thumbsFilePath = "";
let fileDuration = "";
ffmpeg.ffprobe(req.body.filePath, function (err, metadata) {
console.dir(metadata);
console.log(metadata.format.duration);
fileDuration = metadata.format.duration;
})
ffmpeg(req.body.filePath)
.on('filenames', function (filenames) {
console.log('Will generate ' + filenames.join(', '))
thumbsFilePath = "thumbnails/" + filenames[0];
})
.on('end', function () {
console.log('Screenshots taken');
return res.json({ success: true, thumbsFilePath: thumbsFilePath, fileDuration: fileDuration })
})
//Can this be uploaded to google storage?
.screenshots({
// Will take 3 screenshots
count: 3,
folder: '/thumbnails/',
size: '320x240',
//Names file w/o extension
filename:'thumbnail-%b.png'
});
});
Front-end video upload
const onDrop = (files) => {
let formData = new FormData();
const config = {
header: {'content-type': 'multipart/form-data'}
}
console.log(files)
formData.append("file", files[0])
axios.post('/api/video/uploadfiles', formData, config)
.then(response => {
if (response.data.success) {
let variable = {
filePath: response.data.filePath,
fileName: response.data.fileName
}
setFilePath(response.data.filePath)
//Thumbnail
axios.post('/api/video/thumbnail', variable)
.then(response => {
if (response.data.success) {
setDuration(response.data.fileDuration)
setThumbnail(response.data.thumbsFilePath)
} else {
alert("Failed to generate a thumbnail");
}
})
} else {
alert('Failed to save video to the server')
}
})
}

Here you can find the sample code of an application web page prompting the user to supply a file to be stored in Cloud Storage. The code is configuring bucket using environment variables and creates a new blob in the bucket to upload the file data.
I hope this information helps.

You may have to just move them after they're generated with ffmpeg.
For example, I'm writing them to a temp directory outputted by ffmpeg, and then moving after to a Cloud Storage bucket in my cloud function:
const uploadResult = await bucket.upload(targetTempFilePath, {
destination: targetStorageFilePath,
gzip: true
});
Not sure which environment you're using (flex, cloud run, etc) but these were the instructions I was referencing, and are generally the same steps you'll want to follow: https://firebase.google.com/docs/storage/extend-with-functions

Related

Expressjs multer middleware save file in dynamic destination

I am using multer with expressjs and I am trying to change the DIR to save images depending on the request to a dynamic destination, my code is working fine but always save images inside the post directory, I'm using this middleware with multi requests.
1- How can I make the directory dynamic! example: to save to ./public/products if req product & save to ./public/posts if req post
2- How to make sure that the file is uploaded to the directory with no errors in the controller! then save the path to the database!
3- Is this the best practice to use multer ! in middleware level!
multer middleware fileUpload.js
const multer = require("multer");
const mkdirp = require("mkdirp");
const fs = require("fs");
const getDirImage = () => {
// const DIR = './public/uploads/products';
return `./public/posts`;
};
let storage = multer.diskStorage({
destination: (req, file, cb) => {
console.log(req.params,'&&&&&&&&&&&&',file);
let DIR = getDirImage();
if (!fs.existsSync(DIR)) {
fs.mkdirSync(DIR, { recursive: true });
}
cb(null, DIR);
},
filename: (req, file, cb) => {
const fileName = "overDress" + Date.now() + "" +
file.originalname.toLowerCase().split(' ').join('-');
cb(null, fileName)
},
});
const upload = multer({
storage: storage,
limits: {
fileSize: 1024 * 1024 * 10 //upto 2 megabytes per file.
},
fileFilter: (req, file, cb) => {
if (file.mimetype == "image/png" || file.mimetype == "image/jpg" ||
file.mimetype == "image/jpeg") {
cb(null, true);
} else {
cb(null, false);
return cb(new Error('File types allowed .jpeg, .jpg and .png!'));
}
}
});
module.exports = upload;
product.js route
const controller = require('../controllers/product.controller');
import { Router } from 'express'; //import from esm
import upload from '../middleware/fileUpload'
const router = Router();
router
.get('/', controller.list)
.post('/', upload.single('image'), controller.create)
.get('/:id', controller.getOne)
export default router;
and my create controller:
exports.create = async (req, res, next) => {
const { name, title, description,subtitle} = req.body;
if (!name || !title) {
return res.status(400).send({
message: 'Please provide a title and a name to create a product!',
});
}
try {
if (req.file) {
req.body.image = req.file.destination + '/' + req.file.filename;
}
const PRODUCT_MODEL = {
name: req.body.name,
title: req.body.title,
description: req.body.description,
image: req.body.image,
};
try {
const product = await Product.create(PRODUCT_MODEL);
console.log('product crerated');
return res.status(201).json(product);
} catch (error) {
console.log(error);
return res.status(500).send({
message: 'Something went wrong: ' + error,
});
}
} catch (error) {
return res.status(500).json(error);
}
};

NodeJS / Multer : How to replace exising file in directory with new file

I want to replace existing file in directory when upload new new , I'm using nodeJS and multer, the idea is to read directory content, delete it and then upload the new file.
const express = require('express')
const router = express.Router()
const authorize = require('_middleware/authorize')
const Role = require('_helpers/role')
const uploadImage = require("./uploadAdvertising")
const fs = require('fs')
const uploadDir = '/public/advertising/'
// routes
router.post('/', authorize(Role.Admin), uploadImage, createAdvertising);
module.exports = router;
async function createAdvertising(req, res, next) {
fs.readdir(global.__basedir + uploadDir, ((err, files) => {
if (err) throw err
for (const file of files){
fs.unlink(global.__basedir + uploadDir + file, err => {
if (err) throw err;
});
}
}))
if (!req.file) return res.status(412).send({
message: `image is required`,
});
uploadImage(req, res)
.then(data => {
return res.status(201).send({
message: 'Advertising image saved'
})
}).catch(err => {
return res.status(500).send({
message: `Could not upload the file, ${err}`,
});
})
}
multer
const util = require("util");
const multer = require("multer");
const maxSize = 10 * 1024 * 1024;
const uploadDir = '/public/advertising/'
const fs = require('fs')
let storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, global.__basedir + uploadDir)
req.uploadDirectory = global.__basedir + uploadDir
},
filename: (req, file, cb) => {
cb(null, file.originalname)
},
});
let uploadImage = multer({
storage: storage,
fileFilter: (req, file, cb) => {
//reject
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true)
} else {
cb(new Error("image should be png or jpeg extension"), false)
}
},
limits: {fileSize: maxSize},
}).single("imageAdvertising");
let uploadFileMiddleware = util.promisify(uploadImage);
module.exports = uploadFileMiddleware;
can some one take a look and tell what's wrong with this code ! it delete all content even the new uploaded file !
just try like this in the multer file
note : write the path of uploaded file in unlinkSync and readdirSync
fileFilter: (req, file, cb) => {
let files = fs.readdirSync('write the path');
if(files.includes(file.originalname)){
fs.unlinkSync('pwrite the pathath'+ file.originalname);
}
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true)
} else {
cb(new Error("image should be png or jpeg extension"), false)
}
}
and remove extra code like createAdvertising file

Multer: How to have the filename same with the one save in database

Right now I'm able to basically save just the name or originalname of the image file in both:-
database (using mongoose) and,
image folder
But this way of doing it will have flaw when trying to delete an image. For example if I try to upload the same image twice. Multer will not copy the new image (the same image) again in the destination location or folder (/upload). So if later on, I try to delete a data from:- (bare in mind that right now I've two data that have the same image name)
database (using mongoose)
delete the image affiliate with the data from folder /upload
The other data that use the same image will not have one anymore.
So, I've tried putting new Date().toISOString().replace(/[-T:\.Z]/g, "") on the filename on both the image saved in local folder /upload and document in mongoDb. But obviously that won't work cause the generated date slightly different for both case.
Below is my current code:-
Multer.js
// Mutler
const multer = require('multer')
// Path
const path = require('path')
// File Remove
const fileRemove = require('fs')
const {
// File Base FOlder Location
FILE_BASE_FOLDER_LOCATION = path.resolve(__dirname + '/', '../'),
// Image Folder Location
IMAGE_FOLDER_LOCATION = FILE_BASE_FOLDER_LOCATION + '/upload/',
} = process.env
// storage img
const storageImgFile = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, IMAGE_FOLDER_LOCATION)
},
filename: (req, file, cb) => {
// renaming the image file to have the 'date' and the original file name
cb(null, new Date().toISOString().replace(/[-T:\.Z]/g, "") + '-' + file.originalname)
}
})
// filter img types
const filterImgFile = (req, file, cb) => {
const fileTypes = ['image/png', 'image/jpg', 'image/jpeg']
if(fileTypes.includes(file.mimetype)) cb(null, true)
else cb('Only .png .jpg and .jpeg format allowed!', false)
}
// Img FIle Upload Middleware
const uploadImgFile = multer({
storage: storageImgFile,
filterImgFile: filterImgFile
// limits: { fieldSize: 10000000000 }
})
// Img Removing Handler
const handleImgRemove = (res, imgName) => {
fileRemove.unlink(IMAGE_FOLDER_LOCATION + imgName, (err) => {
if(err) {
return res.status(500).json({
success: false,
error: `Failed at removing file from upload folder`,
data: err
})
}
})
}
module.exports = {
imgFolderLocation: IMAGE_FOLDER_LOCATION,
uploadImgFile,
handleImgRemove,
}
AddNewImage Route:-
router.post('/add', uploadImgFile.single('file'), async(req, res) => {
let { desc } = req.body
const image = new Image({
// the date will be slightly different or not same with the 'imagename' saved in 'local' destination
imgName: new Date().toISOString().replace(/[-T:\.Z]/g, "") + '-' + req.file.originalname,
desc: desc
});
image.save()
.then(res => {
return res.status(200).json({
success: true,
count: res.length,
data: res
})
})
.catch(err => {
console.log(err)
return res.status(500).json({
success: false,
error: `Failed to upload new image!`,
data: err
})
})
})
DeleteImage Route:-
router.delete('/delete/:id', async(req, res) => {
await Image.findByIdAndDelete(req.params.id)
.then(data => {
// - remove image from upload folder
handleImgRemove(res, data.imgName)
return res.status(200).json({
success: true,
count: data.length,
data: data
})
})
.catch(err => {
return res.status(500).json({
success: false,
error: `Failed to delete image from DB!`,
data: err
})
})
})
I'm hoping to get both (imgName saved in mongoDb) and filename saved in local destination folder /upload will be the same. But I can't. So how can I make sure that both of them will have the same naming system or ways?
const multer = require("multer");
const uuid = require("uuid").v4;
const path = require("path");
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, "uploads");
},
filename: (req, file, cb) => {
const ext = path.extname(file.originalname);
const originalname = `${uuid()}${ext}`;
cb(null, originalname);
},
});
const upload = multer({ storage });
npm install uuid
You would need to give each image a unique name using uuid, you can do this by giving each image a filename like in the example. This will also slove the problem where the same file won't be stored twice. You can additionally add the current date to the filename to further reduce the chance of two files having the same name.
This is what the image object will look like
{
fieldname: 'file',
originalname: 'image1.jpg',
encoding: '7bit',
mimetype: 'image/jpeg',
destination: 'uploads',
filename: '31d41f39-b4ad-467c-bb09-601596240fe8.jpg',
path: 'uploads\\31d41f39-b4ad-467c-bb09-601596240fe8.jpg',
size: 961388
}

how to upload a image file in node.js?

var title="this is title";
var content="this is content";
const config = { headers: {'Accept': 'application/json', 'Content-Type': 'multipart/form-data' } };
const form = new FormData()
let file =event.target.files[0]
form.append('file', file)
form.append('title', title)
form.append('content', content)`enter code here`
Axios.post("http://localhost:3001/article/get/123", form,config ).then((res)=>{
console.log(res.data)
})
in node I have used multer for upload image or anything.
Below is the code for upload which I have used as a middleware.
const util = require("util");
const path = require("path");
const multer = require("multer");
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "./Uploads") // folder path where to upload
},
filename: function (req, file, cb) {
cb(null, file.originalname + "-" + Date.now() + path.extname(file.originalname))
}
});
const maxSize = 1 * 20000 * 20000; // file size validation
const uploadFiles = multer({ storage: storage, limits: { fileSize: maxSize } }).array("myfiles", 10); // key name should be myfiles in postman while upload
const uploadFilesMiddleware = util.promisify(uploadFiles);
module.exports = uploadFilesMiddleware;
Below is the function which I have created in controller for upload and file check.
fileUpload = async (req, res) => {
try {
let userCode = req.headers.user_code;
await upload(req, res);
if (req.files.length <= 0) {
return res.status(httpStatusCode.OK).send(responseGenerators({}, httpStatusCode.OK, 'Kindly select a file to upload..!!', true));
}
let response = [];
for (const element of req.files) {
let data = await service.addFileData(element, userCode);
response.push(data); // for file path to be stored in database
}
if (response && response.length > 0) {
return res.status(httpStatusCode.OK).send(responseGenerators(response, httpStatusCode.OK, 'File uploaded sucessfully..!!', false));
} else {
return res.status(httpStatusCode.OK).send(responseGenerators({}, httpStatusCode.OK, 'Failed to upload file kindly try later..!!', true));
}
} catch (error) {
logger.warn(`Error while fetch post data. Error: %j %s`, error, error)
return res.status(httpStatusCode.INTERNAL_SERVER_ERROR).send(responseGenerators({}, httpStatusCode.INTERNAL_SERVER_ERROR, 'Error while uploading file data', true))
}
}
and the route will go like this.
router.post('/upload/file', fileUploadController.fileUpload);
And be sure to keep same name in postman while file upload as in middleware.
The above code is in react.js. I want to do same work in node.js and the file will be upload from the public folder. main issue is how to upload image file in format like we have in frontend event.target.files[0]

Upload a file to Google Cloud, in a specific directory

How to upload a file on Google Cloud, in a specific bucket directory (e.g. foo)?
"use strict";
const gcloud = require("gcloud");
const PROJECT_ID = "<project-id>";
let storage = gcloud.storage({
projectId: PROJECT_ID,
keyFilename: 'auth.json'
});
let bucket = storage.bucket(`${PROJECT_ID}.appspot.com`)
bucket.upload("1.jpg", (err, file) => {
if (err) { return console.error(err); }
let publicUrl = `https://firebasestorage.googleapis.com/v0/b/${PROJECT_ID}.appspot.com/o/${file.metadata.name}?alt=media`;
console.log(publicUrl);
});
I tried:
bucket.file("foo/1.jpg").upload("1.jpg", ...)
But there's no upload method there.
How can I send 1.jpg in the foo directory?
In Firebase, on the client side, I do:
ref.child("foo").put(myFile);
bucket.upload("1.jpg", { destination: "YOUR_FOLDER_NAME_HERE/1.jpg" }, (err, file) => {
//Do something...
});
This will put 1.jpg in the YOUR_FOLDER_NAME_HERE-folder.
Here is the documentation. By the way, gcloud is deprecated and you should use google-cloud instead.
UPDATE 2020
according to google documentation:
const { Storage } = require('#google-cloud/storage');
const storage = new Storage()
const bucket = storage.bucket('YOUR_GCLOUD_STORAGE_BUCKET')
const blob = bucket.file('youFolder/' + 'youFileName.jpg')
const blobStream = blob.createWriteStream({
resumable: false,
gzip: true,
public: true
})
blobStream.on('error', (err) => {
console.log('Error blobStream: ',err)
});
blobStream.on('finish', () => {
// The public URL can be used to directly access the file via HTTP.
const publicUrl = ('https://storage.googleapis.com/'+ bucket.name + '/' + blob.name)
res.status(200).send(publicUrl);
});
blobStream.end(req.file.buffer)//req.file is your original file
Here you go...
const options = {
destination: 'folder/new-image.png',
resumable: true,
validation: 'crc32c',
metadata: {
metadata: {
event: 'Fall trip to the zoo'
}
}
};
bucket.upload('local-image.png', options, function(err, file) {
// Your bucket now contains:
// - "new-image.png" (with the contents of `local-image.png')
// `file` is an instance of a File object that refers to your new file.
});
If accessing from the same project projectId , keyFilename,.. not required,I use the below code for both upload and download , it works fine.
// Imports the Google Cloud client library
const Storage = require('#google-cloud/storage');
const storage = new Storage();
var destFilename = "./test";
var bucketName = 'cloudtesla';
var srcFilename = 'test';
const options = {
destination: destFilename,
};
//upload file
console.log("upload Started");
storage.bucket(bucketName).upload(srcFilename, {}, (err, file) => {
if(!err)
console.log("upload Completed");
else
console.log(err);
});
//Download file
console.log("Download Started");
storage
.bucket(bucketName)
.file(srcFilename)
.download(options)
.then(() => {
console.log("Download Completed");
})
.catch(err => {
console.error('ERROR:', err);
});
To upload inside specific directory in .NET Core, use
var uploadResponse= await storageClient.UploadObjectAsync(bucketName, $"{foldername}/"+fileName, null, memoryStream);
This should upload your file 'fileName' inside folder 'foldername' in the bucket
I think just adding foo/ to the filename should work, like bucket.upload("foo/1.jpg", (err, file) ... In GCS, directories just a matter of having a '/' in the file name.
If you want to use async-await while uploading files into storage buckets the callbacks won't do the job, Here's how I did it.
async function uploadFile() {
const destPath = 'PATH_TO_STORAGE/filename.extension';
await storage.bucket("PATH_TO_YOUR_BUCKET").upload(newFilePath, {
gzip: true,
destination: destPath,
});
}
Hope it helps someone!

Resources