Get progress of firebase admin file upload - node.js

I'm trying to get the progress of a 1 minute video uploading to firebase bucket storage using the admin sdk. I've seen a lot about using firebase.storage().ref.child..... but I'm unable to do that with the admin sdk since they don't have the same functions. This is my file upload:
exports.uploadMedia = (req, res) => {
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs');
const busboy = new BusBoy({ headers: req.headers, limits: { files: 1, fileSize: 200000000 } });
let mediaFileName;
let mediaToBeUploaded = {};
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
if(mimetype !== 'image/jpeg' && mimetype !== 'image/png' && mimetype !== 'video/quicktime' && mimetype !== 'video/mp4') {
console.log(mimetype);
return res.status(400).json({ error: 'Wrong file type submitted, only .png, .jpeg, .mov, and .mp4 files allowed'})
}
// my.image.png
const imageExtension = filename.split('.')[filename.split('.').length - 1];
//43523451452345231234.png
mediaFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), mediaFileName);
mediaToBeUploaded = { filepath, mimetype };
file.pipe(fs.createWriteStream(filepath));
file.on('limit', function(){
fs.unlink(filepath, function(){
return res.json({'Error': 'Max file size is 200 Mb, file size too large'});
});
});
});
busboy.on('finish', () => {
admin
.storage()
.bucket()
.upload(mediaToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: mediaToBeUploaded.mimetype
}
}
})
.then(() => {
const meadiaUrl = `https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${mediaFileName}?alt=media`;
return res.json({mediaUrl: meadiaUrl});
})
.catch((err) => {
console.error(err);
return res.json({'Error': 'Error uploading media'});
});
});
req.pipe(busboy);
}
This works okay right now, but the only problem is that the user can't see where their 1 or 2 minute video upload is at. Currently, it's just a activity indicator and the user just sits their waiting without any notice. I'm using react native on the frontend if that helps with anything. Would appreciate any help!

I was able to implement on the client side a lot easier... but it works perfect with image and video upload progress. On the backend, I was using the admin sdk, but frontend I was originally using the firebase sdk.
this.uploadingMedia = true;
const imageExtension = this.mediaFile.split('.')[this.mediaFile.split('.').length - 1];
const mediaFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const response = await fetch(this.mediaFile);
const blob = await response.blob();
const storageRef = storage.ref(`${mediaFileName}`).put(blob);
storageRef.on(`state_changed`,snapshot=>{
this.uploadProgress = (snapshot.bytesTransferred/snapshot.totalBytes);
}, error=>{
this.error = error.message;
this.submitting = false;
this.uploadingMedia = false;
return;
},
async () => {
storageRef.snapshot.ref.getDownloadURL().then(async (url)=>{
imageUrl = [];
videoUrl = [url];
this.uploadingMedia = false;
this.submitPost(imageUrl, videoUrl);
});
});

export const uploadFile = (
folderPath,
fileName,
file,
generateDownloadURL = true,
updateInformationUploadProgress
) => {
return new Promise((resolve, reject) => {
try {
const storageRef = firebaseApp.storage().ref(`${folderPath}/${fileName}`)
const uploadTask = storageRef.put(file)
uploadTask.on(
'state_changed',
snapshot => {
if (updateInformationUploadProgress) {
const progress =
(snapshot.bytesTransferred / snapshot.totalBytes) * 100
updateInformationUploadProgress({
name: fileName,
progress: progress,
})
}
},
error => {
console.log('upload error: ', error)
reject(error)
},
() => {
if (generateDownloadURL) {
uploadTask.snapshot.ref
.getDownloadURL()
.then(url => {
resolve(url)
})
.catch(error => {
console.log('url error: ', error.message)
reject(error)
})
} else {
resolve(uploadTask.snapshot.metadata.fullPath)
}
}
)
} catch (error) {
reject(error)
}
})
}

Related

Kuzzle / Minio example usage

Dose the Kuzzle or Minio development teams have a working example of using the Kuzzle S3 plugin for Minio? I have the following but my file isnt being uploaded and the pre-signed url is referring to https://your-s3-bucket.s3.eu-west-3.amazonaws.com/
const fs = require("fs");
const fsPromises = require('fs').promises;
// Create a JS File object instance from a local path using Node.js
const fileObject = require("get-file-object-from-local-path");
// Promise based HTTP client for the browser and node.js
const axios = require('axios');
// Loads the Kuzzle SDK modules
const {
Kuzzle,
WebSocket
} = require('kuzzle-sdk');
var start = new Date();
const webSocketOptionsObject = {
"autoReconnect": true,
"ssl": true,
"port": 443
};
const kuzzle = new Kuzzle(new WebSocket('myurl.com', webSocketOptionsObject));
const credentials = { username: 'xyz123', password: 'fithenmgjtkj' };
const path = __dirname + "\\" + "yellow_taxi_data.csv"; // the "\\" is for Windows path
var fileData = {};
// check file exists
fs.access(path, fs.F_OK, (err) => {
if (err) {
console.error(err)
return
}
fileData = new fileObject.LocalFileData(path);
// Adds a listener to detect connection problems
kuzzle.on('networkError', error => {
console.error('Network Error:', error);
});
});
const connectToKuzzle = async () => {
// Connects to the Kuzzle server
await kuzzle.connect();
return await kuzzle.auth.login('local', credentials);
// console.log('jwt auth token: ', jwt);
}
const disConnectFromKuzzle = async () => {
console.log('Disconnected from Kuzzle');
kuzzle.disconnect();
var time = new Date() - start;
// sec = Math.floor((time/1000) % 60);
console.log('Execution time in milliseconds: ', time);
}
const presignedURL = async () => {
// Get a Presigned URL
const result = await kuzzle.query({
controller: 's3/upload',
action: 'getUrl',
uploadDir: 'proxybucket', // directory name inside the Bucket specified in the s3 plugin bucket name
filename: fileData.name
});
console.log("result: ", result);
return result;
}
const loadFileStream = async () => {
console.log('getting file: ', path);
targetFile = null;
await fs.promises.readFile(path)
.then(function (result) {
console.log("file loaded------", result.length);
targetFile = result;
})
.catch(function (error) {
console.log(error);
return;
});
return targetFile;
}
const kuzzleValidate = async (kuzzleResource) => {
// console.log("kuzzleResource: ", kuzzleResource.result.fileKey);
// validate
// Validate and persist a previsously uploaded file.
// https://docs.kuzzle.io/official-plugins/s3/2/controllers/upload/validate/
const Presult = await kuzzle.query({
// Kuzzle API params
"controller": "s3/upload",
"action": "validate",
// File key in S3 bucket
"fileKey": kuzzleResource.result.fileKey
});
console.log('validate: ', Presult.result.fileUrl);
}
const uploadFile = async (fileBuffer, kuzzleResource, jwt) => {
// options at https://github.com/axios/axios
const axiosOptions = {
headers: {
'Content-Type': fileData.type
},
maxBodyLength: 200000000 // 200,000,000 bytes 200 Mb
};
// PUT the fileBuffer to the Kuzzle S3 endpoint
// https://github.com/axios/axios
axios.defaults.headers.common['Authorization'] = jwt;
const response = await axios.put(kuzzleResource.result.uploadUrl, fileBuffer, axiosOptions)
.then((response) => {
console.log('file uploaded......');
})
.catch(function (error) {
console.log("File upload error: ", error);
return;
});
return "Upload successful";
}
if (fileData) {
connectToKuzzle().then((jwt) => {
console.log(jwt);
// upload(jwt);
presignedURL().then((kuzzleResource) => {
loadFileStream().then((fileBuffer) => {
uploadFile(fileBuffer, kuzzleResource, jwt).then((doneMessage) => {
console.log("doneMessage: ", doneMessage);
}).then(() => {
kuzzleValidate(kuzzleResource).then(() => {
disConnectFromKuzzle();
});
});
});
});
});
}
I'm looking to upload to a Minio bucket and obtain a pre-signedURL so I can store it in a document later.
You can change the endpoint configuration to set a different s3-compatible endpoint who can be a Minio one.
This configuration can be changer under the plugins.s3.endpoint key. You should also disable the usage of default s3 path.
Example:
app.config.set('plugins.s3.endpoint', 'https://minio.local');
app.config.set('plugins.s3.s3ClientOptions.s3ForcePathStyle', false);

configuring the image file reducer Google Firestore

The following google cloud function properly uploads an image, but I would also like to compress the image as to avoid unnecessary charges due to large files being uploaded. I am using the image reducer extension from Firebase and it works but the issue is that the image file no longer shows up on my user table. is there something i need to configure in the extension so that the image url in the user table is overwritten by the reduced image??
exports.uploadImage = (req, res) => {
const BusBoy = require("busboy")
const path = require("path")
const os = require("os")
const fs = require("fs")
const busboy = new BusBoy({ headers: req.headers })
let imageToBeUploaded = {}
let imageFileName
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== `image/jpeg` && mimetype !== `image/png`) {
return res.status(400).json({ error: `Not an acceptable file type` })
}
// my.image.png => ['my', 'image', 'png']
const imageExtension = filename.split(".")[filename.split(".").length - 1]
// 32756238461724837.png
imageFileName = `${Math.round(
Math.random() * 1000000000000
).toString()}.${imageExtension}`
const filepath = path.join(os.tmpdir(), imageFileName)
imageToBeUploaded = { filepath, mimetype }
file.pipe(fs.createWriteStream(filepath))
})
busboy.on("finish", () => {
admin
.storage()
.bucket(config.storageBucket)
.upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl = `https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageFileName}?alt=media`
return db.doc(`/users/${req.user.uid}`).update({ imageUrl })
})
.then(() => {
return res.json({ message: "image uploaded successfully" })
})
.catch(err => {
console.error(err)
return res.status(500).json({ error: "something went wrong" })
})
})
busboy.end(req.rawBody)
}
The Resize Images extension only handles the resizing of the image in Cloud Storage. It does not update data in any other location, including Cloud Firestore. If you need such functionality, you'll need to create it yourself.
Also see:
this discussion on the extensions open-source repo about allowing to specify a callback that gets invoked after resizing.

Upload multiple images on Firebase using NodeJS and Busboy

I created function for uploading a single image on Firebase using NodeJS and Busboy, which returns image url. Allowed image extensions are only .jpg and .png. It will generate random filename and create filepath with storageBucket.
However, I am struggling to refactor this function, so I could upload multiple images. I tried several attempts, but no luck. It should return array of image urls, if all images were uploaded successfully.
Here is my function with single image upload:
const { admin, db } = require("./admin");
const config = require("./config");
exports.uploadImage = (req, res, url, folder) => {
const BusBoy = require("busboy");
const path = require("path");
const os = require("os");
const fs = require("fs");
const busboy = new BusBoy({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res
.status(400)
.json({ error: "Wrong file type submitted!" });
}
// Getting extension of any image
const imageExtension = filename.split(".")[
filename.split(".").length - 1
];
// Setting filename
imageFileName = `${Math.round(
Math.random() * 1000000000
)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = { filepath, mimetype };
file.pipe(fs.createWriteStream(filepath));
});
busboy.on("finish", () => {
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
destination: `${folder}/${imageFileName}`,
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl = `https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o${folder}%2F${imageFileName}?alt=media`;
if (url === `/users/${req.user.alias}`) {
return db.doc(`${url}`).update({ imageUrl });
} else {
return res.json({ imageUrl });
}
})
.then(() => {
return res.json({
message: "Image uploaded successfully!"
});
})
.catch(err => {
console.log(err);
return res.status(500).json({ error: err.code });
});
});
busboy.end(req.rawBody);
};
Any suggestions how to move on?
You've the code almost done, all you've got to do is to create an array of promises and wait for all to resolve.
let imageFileName = {}
let imagesToUpload = []
let imageToAdd = {}
//This triggers for each file type that comes in the form data
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res
.status(400)
.json({ error: "Wrong file type submitted!" });
}
// Getting extension of any image
const imageExtension = filename.split(".")[
filename.split(".").length - 1
];
// Setting filename
imageFileName = `${Math.round(
Math.random() * 1000000000
)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToAdd = {
imageFileName
filepath,
mimetype };
file.pipe(fs.createWriteStream(filepath));
//Add the image to the array
imagesToUpload.push(imageToAdd);
});
busboy.on("finish", () => {
let promises = []
let imageUrls = []
imagesToUpload.forEach(imageToBeUploaded => {
imageUrls.push(`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o${folder}%2F${imageFileName}?alt=media`)
promises.push(admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
destination: `${folder}/${imageFileName}`,
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
}))
})
try{
await Promises.all(resolve)
res.status(200).json({msg: 'Successfully uploaded all images', imageUrls})
}catch(err){ res.status(500).json(err) }
});
busboy.end(req.rawBody);
With that you should be able to upload them all, it's just a matter of putting all promises inside an array and use the Promise.all method to wait for them to resolve. I made it with async/await because that's how I've been doing it but I suppose you would have no problem in doing it with the callbacks.
Also the code is messy but that's mostly because I dont know how to use this text editor, I hope you can still understand it 👀
Samuel Vera's answer is almost correct. There are some typos and a logic error when push to imageUrls array.
Here, the complete code fixed:
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs');
let fields = {};
const busboy = new BusBoy({ headers: request.headers });
let imageFileName = {};
let imagesToUpload = [];
let imageToAdd = {};
let imageUrls = [];
busboy.on('field', (fieldname, fieldvalue) => {
fields[fieldname] = fieldvalue;
});
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== 'image/jpeg' && mimetype !== 'image/png') {
return res
.status(400)
.json({ error: 'Wrong file type submitted!' });
}
// Getting extension of any image
const imageExtension = filename.split('.')[
filename.split('.').length - 1
];
// Setting filename
imageFileName = `${Math.round(Math.random() * 1000000000)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToAdd = {
imageFileName,
filepath,
mimetype,
};
file.pipe(fs.createWriteStream(filepath));
//Add the image to the array
imagesToUpload.push(imageToAdd);
});
busboy.on('finish', async () => {
let promises = [];
imagesToUpload.forEach((imageToBeUploaded) => {
imageUrls.push(
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageToBeUploaded.imageFileName}?alt=media`
);
promises.push(
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype,
},
},
})
);
});
try {
await Promise.all(promises);
return response.json({
message: `Images URL: ${imageUrls}`,
});
} catch (err) {
console.log(err);
response.status(500).json(err);
}
});
busboy.end(request.rawBody);
Anyway, thank you Samuel :)

Upload images from iOS is too slow

My React-Native iOS front-end can not upload images to my Node.JS (Express + Multer) back-end.
My front-end is React Native Android & iOS. The Android version works fine with no issues, however, uploading images from and iOS device doesn't work most of the time.
Once the upload request is sent, I can see the image file is added in FTP, however, very slowly, like a few KB every second. An image of 500 KB may take 3 minutes or more till the request times out. The file is added to the server partially and I can see change in size with each refresh.
Some [iOS] devices had no issues at all, uploads fast, however, the vast majority of devices are running into this issue.
No connectivity issues. The same host and network work perfectly with Android. Same with some iOS devices.
This is not limited to a specific iOS version or device. However, the devices who had the issue always have it, and those that don't, never have it.
How can I troubleshoot this?
POST request:
router.post('/image', (req, res) => {
console.log('image')
upload(req, res, (error) => {
if (error) {
console.log(error)
return res.send(JSON.stringify({
data: [],
state: 400,
message: 'Invalid file type. Only JPG, PNG or GIF file are allowed.'
}));
} else {
if (req.file == undefined) {
console.log('un')
return res.send(JSON.stringify({
data: [],
state: 400,
message: 'File size too large'
}));
} else {
var CaseID = req.body._case; // || new mongoose.Types.ObjectId(); //for testing
console.log(req.body._case + 'case')
var fullPath = "uploads/images/" + req.file.filename;
console.log(fullPath);
var document = {
_case: CaseID,
path: fullPath
}
var image = new Image(document);
image.save(function(error) {
if (error) {
console.log(error)
return res.send(JSON.stringify({
data: [],
state: 400,
message: 'bad request error'
}));
}
return res.send(JSON.stringify({
data: image,
state: 200,
message: 'success'
}));
});
}
}
});
});
Upload.js:
const multer = require('multer');
const path = require('path');
//image upload module
const storageEngine = multer.diskStorage({
destination: appRoot + '/uploads/images/',
filename: function (req, file, fn) {
fn(null, new Date().getTime().toString() + '-' + file.fieldname + path.extname(file.originalname));
}
});
const upload = multer({
storage: storageEngine,
// limits: {
// fileSize: 1024 * 1024 * 15 // 15 MB
// },
fileFilter: function (req, file, callback) {
validateFile(file, callback);
}
}).single('image');
var validateFile = function (file, cb) {
// allowedFileTypes = /jpeg|jpg|png|gif/;
// const extension = allowedFileTypes.test(path.extname(file.originalname).toLowerCase());
// const mimeType = allowedFileTypes.test(file.mimetype);
// if (extension && mimeType) {
// return cb(null, true);
// } else {
// cb("Invalid file type. Only JPEG, PNG and GIF file are allowed.")
// }
var type = file.mimetype;
var typeArray = type.split("/");
if (typeArray[0] == "image") {
cb(null, true);
}else {
cb(null, false);
}
};
module.exports = upload;
React Native Upload function:
pickImageHandler = () => {
ImagePicker.showImagePicker(this.options1, res => {
if (res.didCancel) {
} else if (res.error) {
} else {
this.setState({upLoadImage:true})
var data = new FormData();
data.append('image', {
uri: res.uri,
name: 'my_photo.jpg',
type: 'image/jpg'
})
data.append('_case',this.state.caseID)
fetch(url+'/image'
, {method:'POST',
body:data
}
)
.then((response) => response.json())
.then((responseJson) =>
{
this.setState(prevState => {
return {
images: prevState.images.concat({
key: responseJson._id,
src: res.uri
})
}
}
)
this.setState({upLoadImage:false})
})
.catch((error) =>
{
alert(error);
});
}
}
)
}
Any suggestions?
Thanks
I saw your answer from UpWork
please try this way,
I'm using API Sauce for API calls
export const addPartRedux = (data) => {
return (dispatch, getState) => {
console.log('addPArtREdux', data);
const values = {
json_email: data.token.username,
json_password: data.token.password,
name: data.text ? data.text : '',
car: data.selected.ID,
model: data.selectedSub.ID,
make_year: data.selectedYear,
type: data.type,
ImportCountry: data.import_image ? data.import_image : '',
FormNumber: data.number ? data.number : '',
do: 'insert'
};
const val = new FormData();
Object.keys(values).map((key) =>
val.append(key, values[key])
);
if (data.imageok) {
val.append('image', {
uri: data.image.uri,
type: data.image.type,
name: data.image.name
});
}
dispatch(loading());
api
.setHeader('Content-Type', 'multipart/form-data;charset=UTF-8');
api
.post('/partRequest-edit-1.html?json=true&ajax_page=true&app=IOS',
val,
{
onUploadProgress: (e) => {
console.log(e);
const prog = e.loaded / e.total;
console.log(prog);
dispatch(progress(prog));
}
})
.then((r) => {
console.log('Response form addPartRedux', r.data);
if (r.ok === true) {
const setting = qs.parse(r.data);
dispatch(addpart(setting));
} else {
dispatch(resetLoading());
dispatch(partstError('Error Loading '));
}
})
.catch(
(e) => {
console.log('submitting form Error ', e);
dispatch(resetLoading());
dispatch(partstError('Try Agin'));
}
);
};
};

How do I upload an image from a URL to Firebase Storage?

I am using Node.JS.
I can't seem to figure it out. I've updated the modules.
Old code:
var mimeType = 'image/jpeg';
var randomString = uuidv4();
var options = {
destination: 'test',
metadata: {
contentType: mimeType,
metadata: {
firebaseStorageDownloadTokens: randomString
}
}
};
return admin.storage().bucket().upload(userPhotoDownloadLink, options).then(()=>{
//
}).catch((err) => {
print(`Error Uploading Image:${err}`);
//
});
This code no longer works. How do I upload via an http link. Google Cloud 2.4.x only shows guides on local file uploads.
EDIT:
var uploadPhoto = async function(public_courier_id,userPhotoDownloadLink){
var mimeType = 'image/jpeg';
var randomString = "testkey123";
var options = {
//destination: 'courierPhotos/' + public_courier_id,
metadata: {
contentType: mimeType,
metadata: {
firebaseStorageDownloadTokens: randomString
}
}
};
const file = admin.storage().bucket().file(`courierPhotos/${public_courier_id}`)
console.log(`Fetching File...`);
return fetch(userPhotoDownloadLink).then(res=>{
console.log(`Successfully Fetched, Piping File...`)
return new Promise((resolve,reject)=>{
res.body.pipe(file.createWriteStream(options))
.on('error', function(err) {
reject(`err: ${err}`)
})
.on('finish', function() {
resolve(`complete`);
});
})
});
}
return uploadPhoto("TESTUID","https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png")
.then(res => console.log(`Printing after function ${res}`))
.catch(e => console.log(e));

Resources