I'm having hard time for the last few day on how to upload multiples images on firebase with busboy.
I want to use 3 fields with 3 different images. so I can store it in one folder.
I also want the image to have the field name
I found one topic that helped me use Promise.all and forEach but it didn't worked out for me
storing all files in an array
var Promise = require('promise');
const Busboy = require("busboy");
const fs = require("fs");
const os = require("os");
const path = require("path");
const busboy = new Busboy({ headers: req.headers });
let imageToAdd = {};
let imagesToUpload = []
let newFileName;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const imageExtension = filename.split('.')[filename.split('.').length - 1];
filename = `${fieldname}.${imageExtension} `
newFileName = filename
const filepath = path.join(os.tmpdir(), filename);
imageToAdd = { file: filepath, type: mimetype };
file.pipe(fs.createWriteStream(filepath));
imagesToUpload = [...imagesToUpload, imageToAdd]
});
loop over the files array and store the promises in a new array
then wait all the promise to resolve with Promise.all
busboy.on("finish", () => {
let promises = []
imagesToUpload.forEach((imageToBeUploaded) => {
promises.push(
admin
.storage()
.bucket(`${config.storageBucket}`)
.upload(imageToBeUploaded.file, {
resumable: false,
destination: `projectname/${newFileName}`,
metadata: {
metadata: {
contentType: imageToBeUploaded.type,
}
}
})
)
})
Promise.all(promises)
.then(res => {
res.status(200).json({msg: 'Successfully uploaded all images')
})
.catch(err => {
res.status(500).json({error: err.code})
})
})
busboy.end(req.rawBody);
})
Only the last image is stored in my firebase storage.
Is someone can help me with this ?
thanks
You only have one newFileName in your code, while you have an array of imagesToUpload. So you're uploading each of those images to the same newFileName and end up with whichever of the uploads completes last.
You'll want to keep an array of newFileNames, to match up with the array of imagesToUpload.
Something like:
const busboy = new Busboy({ headers: req.headers });
let imageToAdd = {};
let imagesToUpload = []
let newFileNames = [];
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const imageExtension = filename.split('.')[filename.split('.').length - 1];
filename = `${fieldname}.${imageExtension} `
const filepath = path.join(os.tmpdir(), filename);
imageToAdd = { file: filepath, type: mimetype };
file.pipe(fs.createWriteStream(filepath));
imagesToUpload.push(imageToAdd);
newFileNames.push(filename);
});
...
busboy.on("finish", () => {
let promises = imagesToUpload.map((imageToBeUploaded, index) => {
admin
.storage()
.bucket(`${config.storageBucket}`)
.upload(imageToBeUploaded.file, {
resumable: false,
destination: `projectname/${newFileNames[index]}`,
metadata: {
metadata: {
contentType: imageToBeUploaded.type,
}
}
})
})
Promise.all(promises)
...
Related
I can upload one file, but I can't seem to upload multiple, it gets stuck on the first file.
Client
async function uploadFiles() {
let formData = new FormData();
formData.append("recordUid", recordUid);
formData.append("fieldUid", fieldUid);
[...pendingUploadFiles].forEach((file) => {
formData.append("uploaded_files", file);
});
await fetchPostFiles("/api/files", formData);
}
Server
const busboy = require("busboy");
const path = require("path");
const fs = require("fs");
router.post("/api/files", async (req, res, next) => {
try {
let bb = busboy({
headers: req.headers,
limits: {
fileSize: 10 * 1024 * 1024, // 10 mb
},
});
bb.on("file", (fieldname, file, filename, encoding, mimetype) => {
let parts = filename.filename.split(".");
let name = parts[0];
let extension = parts[parts.length - 1]; // without the . from .jpeg
let finalName = `${name}-${+new Date()}.${extension}`;
let saveTo = `${filesFolderPath}${finalName}`;
// Open writeable stream to path
let writeStream = fs.createWriteStream(saveTo);
// Pipe the file to the opened stream
file.pipe(writeStream);
// Check for errors
writeStream.on("error", (err) => {
console.log(err);
});
});
bb.on("finish", function () {
console.log("uploaded");
});
return req.pipe(bb);
});
I have a NodeJS function that writes several small svg files locally and then is attempting to upload those files to cloud bucket.
in the function log, i am only seeing the message that file written to local disk. now will upload. But there is no file in the bucket and no error logged anywhere. i have made sure the timeout is set to 9 min (max) so i am sure its not timing out. what else shoudl i check?
any pointers will be appreciated.
exports.createQRCode = functions.storage.object().onFinalize(async (object) =>{
const qrcodeMonkeyKey = functions.config().qrcodemonkey.key;
//console.log(`key for qrcode monkey is ${qrcodeMonkeyKey}`);
const fileBucket = object.bucket; // The Storage bucket that contains the file.
const filePath = object.name; // File path in the bucket.
const contentType = object.contentType; // File content type.
const metageneration = object.metageneration; // Number of times metadata has been generated. New objects have a value of 1.
console.log(fileBucket);
console.log(filePath);
if(!filePath.toLowerCase().endsWith('.csv'))
return console.log('not a csv so no need to anything fancy');
const bucket = admin.storage().bucket(fileBucket);
const filePathComps = filePath.split('/');
const folderName = filePathComps[filePathComps.length-3];
if(folderName !== "qrcode")
return console.log('not a qr code csv so no need to anything fancy');
const fileName = filePathComps[filePathComps.length-1];
console.log(fileName);
const path = require('path');
const os = require('os');
const fs = require('fs');
const tempFilePath = path.join(os.tmpdir(), fileName);
const metadata = {
contentType: contentType,
};
await bucket.file(filePath).download({destination: tempFilePath});
const csv = require('csv-parser')
const results = [];
fs.createReadStream(tempFilePath)
.pipe(csv({headers:
['uri','filename','foldername']
,skipLines:1
}))
.on('data', async (data) => {
const x = data;
results.push(data);
//results.push({id:x.id,phoneNumber:x.phoneNumber,isInternational:x.isInternational,message:x.messageText,respStatus:resp.status,responsedata:resp.data});
})
.on('end',async () => {
pArray = [];
results.forEach(x =>{
pArray.push(createQRCodeAndUpload(qrcodeMonkeyKey,x.filename,x.uri,x.foldername));
});
const finaloutput = await Promise.all(pArray);
console.log(JSON.stringify(finaloutput));
return;
});
});
const createQRCodeAndUpload = async (qrcodeMonkeyKey,fileName, url,foldername) =>{
const bucket = admin.storage().bucket('vmallapp.appspot.com');
const path = require('path');
const os = require('os');
const fs = require('fs');
var axios = require("axios").default;
console.log('processing ' + url);
if(url !==""){
const dataToSend = {
data : url,
config :{
body:'circle',
eye:'frame14',
eyeBall:'ball16',
bodyColor:"#032b5c",
bgColor:"#84d4e2",
"logo":"ae600e1267b9e477f0b635b60ffaec1d1c18d93b.png"
},
size:1200,
download:false,
file:'svg',
gradientOnEyes:true
}
var options = {
method: 'POST',
url: 'https://qrcode-monkey.p.rapidapi.com/qr/custom',
headers: {
'content-type': 'application/json',
'x-rapidapi-host': 'qrcode-monkey.p.rapidapi.com',
'x-rapidapi-key': qrcodeMonkeyKey
},
data: dataToSend
};
var response = await axios.request(options);
console.log('qrcode monkey returned status' + response.status);
const outputFilePath = path.join(os.tmpdir(), `${fileName}.svg`);
fs.writeFileSync(outputFilePath, response.data);
console.log(`${fileName}.svg written to local disk. now will upload`);
try{
await bucket.upload(outputFilePath, {
destination: `qrcode/output/${fileName}.svg`
});
}catch(error){
console.log('error in uploding ' + error);
}
console.log('lets delete the file now and clean up local storage');
fs.unlinkSync(outputFilePath);
return 'all done';
}
}
I can't understand why this function exit before image is downloaded and saved.
I need to wait image will be saved before exiting.
function downloadImagefromRemote(url_immagine, filename) {
console.log('[2] Salvo l\'immagine remota in locale');
const downloadFile = async (fileUrl, downloadFolder, filename) => {
// Get the file name
const fileName = path.basename(fileUrl);
// The path of the downloaded file on our machine
const localFilePath = path.resolve(__dirname, downloadFolder, filename);
try {
const response = await axios({ // <----- here jump out
method: "GET",
url: fileUrl,
responseType: "stream",
});
await response.data.pipe(fs.createWriteStream(localFilePath));
console.log("Successfully downloaded file!");
} catch (err) {
throw new Error(err);
}
};
const IMAGE_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg';
let rel_path = __dirname + '/../../public/images/';
downloadFile(IMAGE_URL, rel_path, filename);
}
The main problem is that using
axios({...})
you need to specify
url: imgUrl
More generally speaking:
avoid unnecessary nested functions
clean your unused and unnecessary params
Here a functioning example:
'use strict'
const fs = require('fs')
const path = require('path')
const axios = require('axios')
async function downloadImage(imgUrl, saveRelPath, saveName) {
const _path = path.resolve(__dirname, saveRelPath, saveName)
const writer = fs.createWriteStream(_path)
const response = await axios({
url: imgUrl,
method: 'GET',
responseType: 'stream',
})
response.data.pipe(writer)
return new Promise((resolve, reject) => {
writer.on('finish', resolve)
writer.on('error', reject)
})
}
async function anotherFunction() {
const IMG_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg'
const REL_PATH = '.'
const NAME = 'test.jpg'
console.log('Get and save image')
await downloadImage(IMG_URL, REL_PATH, NAME)
console.log('Image saving done, other stuff here')
}
anotherFunction()
EDIT: as response to
I can't understand why this function exit before image is downloaded and saved
the stream piping doesn't return a promise, so you can't await it.
You pipe a stream into a write stream that is an EventEmitter, so you should use a listener.
You can try that, it should work. for the pipeline() explaination, see this answer Promise completed before file is written. I promisified your flow to be able to resolve as soon as the stream process ends (note that i don t know about the other parts of your code, like if axios does not return a stream it will not work)
const { pipeline } = require('stream')
async function downloadImagefromRemote(url_immagine, filename) {
console.log('[2] Salvo l\'immagine remota in locale');
const downloadFile = (fileUrl, downloadFolder, filename) => {
return new Promise(async(resolve, reject) => {
// Get the file name
const fileName = path.basename(fileUrl);
// The path of the downloaded file on our machine
const localFilePath = path.resolve(__dirname, downloadFolder, filename);
try {
const response = await axios({ // <----- here jump out
method: "GET",
url: fileUrl,
responseType: "stream",
});
pipeline(
response.data,
fs.createWriteStream(localFilePath),
e => {
if(e) reject(e)
else resolve("Successfully downloaded file!")
}
)
} catch (err) {
reject(Error(err))
}
})
};
const IMAGE_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg';
let rel_path = __dirname + '/../../public/images/';
try{
const confirmDownload = await downloadFile(IMAGE_URL, rel_path, filename);
console.log(confirmDownload)
} catch(e) {
console.log('catched err:', e)
}
}
I'm trying to get the progress of a 1 minute video uploading to firebase bucket storage using the admin sdk. I've seen a lot about using firebase.storage().ref.child..... but I'm unable to do that with the admin sdk since they don't have the same functions. This is my file upload:
exports.uploadMedia = (req, res) => {
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs');
const busboy = new BusBoy({ headers: req.headers, limits: { files: 1, fileSize: 200000000 } });
let mediaFileName;
let mediaToBeUploaded = {};
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
if(mimetype !== 'image/jpeg' && mimetype !== 'image/png' && mimetype !== 'video/quicktime' && mimetype !== 'video/mp4') {
console.log(mimetype);
return res.status(400).json({ error: 'Wrong file type submitted, only .png, .jpeg, .mov, and .mp4 files allowed'})
}
// my.image.png
const imageExtension = filename.split('.')[filename.split('.').length - 1];
//43523451452345231234.png
mediaFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), mediaFileName);
mediaToBeUploaded = { filepath, mimetype };
file.pipe(fs.createWriteStream(filepath));
file.on('limit', function(){
fs.unlink(filepath, function(){
return res.json({'Error': 'Max file size is 200 Mb, file size too large'});
});
});
});
busboy.on('finish', () => {
admin
.storage()
.bucket()
.upload(mediaToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: mediaToBeUploaded.mimetype
}
}
})
.then(() => {
const meadiaUrl = `https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${mediaFileName}?alt=media`;
return res.json({mediaUrl: meadiaUrl});
})
.catch((err) => {
console.error(err);
return res.json({'Error': 'Error uploading media'});
});
});
req.pipe(busboy);
}
This works okay right now, but the only problem is that the user can't see where their 1 or 2 minute video upload is at. Currently, it's just a activity indicator and the user just sits their waiting without any notice. I'm using react native on the frontend if that helps with anything. Would appreciate any help!
I was able to implement on the client side a lot easier... but it works perfect with image and video upload progress. On the backend, I was using the admin sdk, but frontend I was originally using the firebase sdk.
this.uploadingMedia = true;
const imageExtension = this.mediaFile.split('.')[this.mediaFile.split('.').length - 1];
const mediaFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const response = await fetch(this.mediaFile);
const blob = await response.blob();
const storageRef = storage.ref(`${mediaFileName}`).put(blob);
storageRef.on(`state_changed`,snapshot=>{
this.uploadProgress = (snapshot.bytesTransferred/snapshot.totalBytes);
}, error=>{
this.error = error.message;
this.submitting = false;
this.uploadingMedia = false;
return;
},
async () => {
storageRef.snapshot.ref.getDownloadURL().then(async (url)=>{
imageUrl = [];
videoUrl = [url];
this.uploadingMedia = false;
this.submitPost(imageUrl, videoUrl);
});
});
export const uploadFile = (
folderPath,
fileName,
file,
generateDownloadURL = true,
updateInformationUploadProgress
) => {
return new Promise((resolve, reject) => {
try {
const storageRef = firebaseApp.storage().ref(`${folderPath}/${fileName}`)
const uploadTask = storageRef.put(file)
uploadTask.on(
'state_changed',
snapshot => {
if (updateInformationUploadProgress) {
const progress =
(snapshot.bytesTransferred / snapshot.totalBytes) * 100
updateInformationUploadProgress({
name: fileName,
progress: progress,
})
}
},
error => {
console.log('upload error: ', error)
reject(error)
},
() => {
if (generateDownloadURL) {
uploadTask.snapshot.ref
.getDownloadURL()
.then(url => {
resolve(url)
})
.catch(error => {
console.log('url error: ', error.message)
reject(error)
})
} else {
resolve(uploadTask.snapshot.metadata.fullPath)
}
}
)
} catch (error) {
reject(error)
}
})
}
I created function for uploading a single image on Firebase using NodeJS and Busboy, which returns image url. Allowed image extensions are only .jpg and .png. It will generate random filename and create filepath with storageBucket.
However, I am struggling to refactor this function, so I could upload multiple images. I tried several attempts, but no luck. It should return array of image urls, if all images were uploaded successfully.
Here is my function with single image upload:
const { admin, db } = require("./admin");
const config = require("./config");
exports.uploadImage = (req, res, url, folder) => {
const BusBoy = require("busboy");
const path = require("path");
const os = require("os");
const fs = require("fs");
const busboy = new BusBoy({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res
.status(400)
.json({ error: "Wrong file type submitted!" });
}
// Getting extension of any image
const imageExtension = filename.split(".")[
filename.split(".").length - 1
];
// Setting filename
imageFileName = `${Math.round(
Math.random() * 1000000000
)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = { filepath, mimetype };
file.pipe(fs.createWriteStream(filepath));
});
busboy.on("finish", () => {
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
destination: `${folder}/${imageFileName}`,
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl = `https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o${folder}%2F${imageFileName}?alt=media`;
if (url === `/users/${req.user.alias}`) {
return db.doc(`${url}`).update({ imageUrl });
} else {
return res.json({ imageUrl });
}
})
.then(() => {
return res.json({
message: "Image uploaded successfully!"
});
})
.catch(err => {
console.log(err);
return res.status(500).json({ error: err.code });
});
});
busboy.end(req.rawBody);
};
Any suggestions how to move on?
You've the code almost done, all you've got to do is to create an array of promises and wait for all to resolve.
let imageFileName = {}
let imagesToUpload = []
let imageToAdd = {}
//This triggers for each file type that comes in the form data
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res
.status(400)
.json({ error: "Wrong file type submitted!" });
}
// Getting extension of any image
const imageExtension = filename.split(".")[
filename.split(".").length - 1
];
// Setting filename
imageFileName = `${Math.round(
Math.random() * 1000000000
)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToAdd = {
imageFileName
filepath,
mimetype };
file.pipe(fs.createWriteStream(filepath));
//Add the image to the array
imagesToUpload.push(imageToAdd);
});
busboy.on("finish", () => {
let promises = []
let imageUrls = []
imagesToUpload.forEach(imageToBeUploaded => {
imageUrls.push(`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o${folder}%2F${imageFileName}?alt=media`)
promises.push(admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
destination: `${folder}/${imageFileName}`,
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
}))
})
try{
await Promises.all(resolve)
res.status(200).json({msg: 'Successfully uploaded all images', imageUrls})
}catch(err){ res.status(500).json(err) }
});
busboy.end(req.rawBody);
With that you should be able to upload them all, it's just a matter of putting all promises inside an array and use the Promise.all method to wait for them to resolve. I made it with async/await because that's how I've been doing it but I suppose you would have no problem in doing it with the callbacks.
Also the code is messy but that's mostly because I dont know how to use this text editor, I hope you can still understand it 👀
Samuel Vera's answer is almost correct. There are some typos and a logic error when push to imageUrls array.
Here, the complete code fixed:
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs');
let fields = {};
const busboy = new BusBoy({ headers: request.headers });
let imageFileName = {};
let imagesToUpload = [];
let imageToAdd = {};
let imageUrls = [];
busboy.on('field', (fieldname, fieldvalue) => {
fields[fieldname] = fieldvalue;
});
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== 'image/jpeg' && mimetype !== 'image/png') {
return res
.status(400)
.json({ error: 'Wrong file type submitted!' });
}
// Getting extension of any image
const imageExtension = filename.split('.')[
filename.split('.').length - 1
];
// Setting filename
imageFileName = `${Math.round(Math.random() * 1000000000)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToAdd = {
imageFileName,
filepath,
mimetype,
};
file.pipe(fs.createWriteStream(filepath));
//Add the image to the array
imagesToUpload.push(imageToAdd);
});
busboy.on('finish', async () => {
let promises = [];
imagesToUpload.forEach((imageToBeUploaded) => {
imageUrls.push(
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageToBeUploaded.imageFileName}?alt=media`
);
promises.push(
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype,
},
},
})
);
});
try {
await Promise.all(promises);
return response.json({
message: `Images URL: ${imageUrls}`,
});
} catch (err) {
console.log(err);
response.status(500).json(err);
}
});
busboy.end(request.rawBody);
Anyway, thank you Samuel :)