Related
How can I access the files in the API?
I read the other solutions and all of them require npm packages to solve this. I want to understand why I can't do it vanilla. Also, the answers are old and recommend using body-parser, which now comes bundled with Express.
I would like the solution to be vanilla JS in order to understand the process better.
client
async function uploadFile(file) {
let formData = new FormData();
formData.append("file", file);
let res = await fetchPostFile("/api/files", formData);
}
fetch
export async function fetchPostFile(url, formData) {
try {
let result = await (
await fetch(url, {
method: "POST",
withCredentials: true,
credentials: "include",
headers: {
Authorization: localStorage.getItem("token"),
Accept: "application/json",
"Content-type": "multipart/form-data",
},
body: formData,
})
).json();
return result;
} catch (err) {
return err;
}
}
api
router.post("/api/files", async function (req, res, next) {
try {
console.log(req.file); // undefined
console.log(req.files); // undefined
console.log(req.body); // {}
} catch (err) {
next(err);
} finally {
req.connection.release();
}
});
Why is the req.body empty? Please help me understand what I'm doing wrong.
First you need to use multer package to handle multipart/form-data in express. You must use it as a middleware to set the field name for the file. The name passed as an argument to the single() function must match to the appended name on client side.
const multer = require('multer')
router.post("/api/files", multer().single('file'), async function (req, res, next) {
try {
console.log(req.file)
} catch (err) {
next(err)
} finally {
req.connection.release()
}
});
Here's the whole file management API.
I am using busboy which is what multer uses under the hood. I found it easier to use.
const express = require("express");
const router = express.Router();
const config = require("../../config");
const busboy = require("busboy");
const fs = require("fs");
const SHA256 = require("crypto-js/sha256");
let filesFolderPath = config.paths.files;
router.get("/api/records/:recordUid/files/:fieldUid", async (req, res, next) => {
try {
let { recordUid, fieldUid } = req.params;
let query = `
select
rdf.*,
r.uid recordUid,
round(sizeBytes / 1024, 0) sizeKb,
round(sizeBytes / 1024 / 1024, 0) sizeMb
from recordDataFile rdf
left join record r on r.id = rdf.recordId
left join field f on f.id = rdf.fieldId
where
r.uid = ?
and f.uid = ?;
`;
let rows = await req.pool.query(query, [recordUid, fieldUid]);
res.status(200).send(rows);
} catch (err) {
next(err);
}
});
router.get("/api/files/:hash", async (req, res, next) => {
try {
let { hash } = req.params;
let query = `
select *
from recordDataFile
where hash = ?
`;
let rows = await req.pool.query(query, [hash]);
let fileData = rows[0];
res.download(fileData.path);
} catch (err) {
next(err);
}
});
router.post("/api/files", async (req, res, next) => {
try {
let bb = busboy({
headers: req.headers,
defCharset: "utf8",
limits: {
fileSize: 20 * 1024 * 1024, // 20 mb
files: 5,
},
});
let fields = {};
// Get any text values
bb.on("field", (fieldname, val, fieldnameTruncated, valTruncated) => {
console.log(fieldname, val);
fields[fieldname] = val;
});
// Read file stream
bb.on("file", (fieldname, fileStream, filename, encoding, mimetype) => {
// Prevents hieroglyphs from cyrillic
let originalName = Buffer.from(filename.filename, "latin1").toString("utf8");
let nameParts = originalName.split(".");
let extension = nameParts[nameParts.length - 1]; // without the . from .jpeg
// IMPORTANT!!! FILE NAME CAN'T HAVE SPACES, it won't save properly!!!
let hash = SHA256(`${+new Date()}${originalName}`).toString();
// Absolute path to file
let filePath = `${filesFolderPath}${hash}`;
// Open writeable stream to path
let writeStream = fs.createWriteStream(filePath);
// Pipe the file to the opened stream
fileStream.pipe(writeStream);
// Check for errors
writeStream.on("error", (err) => {
console.log("writeStream", err);
});
// Writing done, stream closed
writeStream.on("close", async (err) => {
// console.log("closing + SQL");
if (err) {
console.log("closing error");
return;
}
let query = `
insert into recordDataFile
(
recordId,
fieldId,
name,
extension,
hash,
path,
sizeBytes,
userId,
created
)
values
(
(select id from record where uid = ?),
(select id from field where uid = ?),
?,
?,
?,
?,
?,
?,
now()
);
`;
let sizeBytes = fs.statSync(filePath).size;
await req.pool.query(query, [fields.recordUid, fields.fieldUid, originalName, extension, hash, filePath, sizeBytes, req.userId]);
// record updated. send notification?
await req.pool.query(`update record set updated = now(), updatedByUserId = ? where uid = ?`, [req.userId, fields.recordUid]);
});
});
bb.on("finish", () => {
res.status(200).send({ success: true });
});
req.pipe(bb); // Hooks the streams together. Without it, you're not feeding busboy any data to parse.
} catch (err) {
console.log("file upload catch", err);
next(err);
}
});
router.delete("/api/files/:hash", async (req, res, next) => {
try {
let { hash } = req.params;
// get the file
let query = `
select * from recordDataFile where hash = ?
`;
let rows = await req.pool.query(query, [hash]);
let file = rows[0];
let filePath = file.path;
// remove file
fs.unlinkSync(filePath);
// delete the file metadata
await req.pool.query(` delete from recordDataFile where hash = ?`, [hash]);
res.status(200).send(rows);
} catch (err) {
next(err);
}
});
module.exports = router;
I'm having hard time for the last few day on how to upload multiples images on firebase with busboy.
I want to use 3 fields with 3 different images. so I can store it in one folder.
I also want the image to have the field name
I found one topic that helped me use Promise.all and forEach but it didn't worked out for me
storing all files in an array
var Promise = require('promise');
const Busboy = require("busboy");
const fs = require("fs");
const os = require("os");
const path = require("path");
const busboy = new Busboy({ headers: req.headers });
let imageToAdd = {};
let imagesToUpload = []
let newFileName;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const imageExtension = filename.split('.')[filename.split('.').length - 1];
filename = `${fieldname}.${imageExtension} `
newFileName = filename
const filepath = path.join(os.tmpdir(), filename);
imageToAdd = { file: filepath, type: mimetype };
file.pipe(fs.createWriteStream(filepath));
imagesToUpload = [...imagesToUpload, imageToAdd]
});
loop over the files array and store the promises in a new array
then wait all the promise to resolve with Promise.all
busboy.on("finish", () => {
let promises = []
imagesToUpload.forEach((imageToBeUploaded) => {
promises.push(
admin
.storage()
.bucket(`${config.storageBucket}`)
.upload(imageToBeUploaded.file, {
resumable: false,
destination: `projectname/${newFileName}`,
metadata: {
metadata: {
contentType: imageToBeUploaded.type,
}
}
})
)
})
Promise.all(promises)
.then(res => {
res.status(200).json({msg: 'Successfully uploaded all images')
})
.catch(err => {
res.status(500).json({error: err.code})
})
})
busboy.end(req.rawBody);
})
Only the last image is stored in my firebase storage.
Is someone can help me with this ?
thanks
You only have one newFileName in your code, while you have an array of imagesToUpload. So you're uploading each of those images to the same newFileName and end up with whichever of the uploads completes last.
You'll want to keep an array of newFileNames, to match up with the array of imagesToUpload.
Something like:
const busboy = new Busboy({ headers: req.headers });
let imageToAdd = {};
let imagesToUpload = []
let newFileNames = [];
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const imageExtension = filename.split('.')[filename.split('.').length - 1];
filename = `${fieldname}.${imageExtension} `
const filepath = path.join(os.tmpdir(), filename);
imageToAdd = { file: filepath, type: mimetype };
file.pipe(fs.createWriteStream(filepath));
imagesToUpload.push(imageToAdd);
newFileNames.push(filename);
});
...
busboy.on("finish", () => {
let promises = imagesToUpload.map((imageToBeUploaded, index) => {
admin
.storage()
.bucket(`${config.storageBucket}`)
.upload(imageToBeUploaded.file, {
resumable: false,
destination: `projectname/${newFileNames[index]}`,
metadata: {
metadata: {
contentType: imageToBeUploaded.type,
}
}
})
})
Promise.all(promises)
...
I created function for uploading a single image on Firebase using NodeJS and Busboy, which returns image url. Allowed image extensions are only .jpg and .png. It will generate random filename and create filepath with storageBucket.
However, I am struggling to refactor this function, so I could upload multiple images. I tried several attempts, but no luck. It should return array of image urls, if all images were uploaded successfully.
Here is my function with single image upload:
const { admin, db } = require("./admin");
const config = require("./config");
exports.uploadImage = (req, res, url, folder) => {
const BusBoy = require("busboy");
const path = require("path");
const os = require("os");
const fs = require("fs");
const busboy = new BusBoy({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res
.status(400)
.json({ error: "Wrong file type submitted!" });
}
// Getting extension of any image
const imageExtension = filename.split(".")[
filename.split(".").length - 1
];
// Setting filename
imageFileName = `${Math.round(
Math.random() * 1000000000
)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = { filepath, mimetype };
file.pipe(fs.createWriteStream(filepath));
});
busboy.on("finish", () => {
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
destination: `${folder}/${imageFileName}`,
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl = `https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o${folder}%2F${imageFileName}?alt=media`;
if (url === `/users/${req.user.alias}`) {
return db.doc(`${url}`).update({ imageUrl });
} else {
return res.json({ imageUrl });
}
})
.then(() => {
return res.json({
message: "Image uploaded successfully!"
});
})
.catch(err => {
console.log(err);
return res.status(500).json({ error: err.code });
});
});
busboy.end(req.rawBody);
};
Any suggestions how to move on?
You've the code almost done, all you've got to do is to create an array of promises and wait for all to resolve.
let imageFileName = {}
let imagesToUpload = []
let imageToAdd = {}
//This triggers for each file type that comes in the form data
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== "image/jpeg" && mimetype !== "image/png") {
return res
.status(400)
.json({ error: "Wrong file type submitted!" });
}
// Getting extension of any image
const imageExtension = filename.split(".")[
filename.split(".").length - 1
];
// Setting filename
imageFileName = `${Math.round(
Math.random() * 1000000000
)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToAdd = {
imageFileName
filepath,
mimetype };
file.pipe(fs.createWriteStream(filepath));
//Add the image to the array
imagesToUpload.push(imageToAdd);
});
busboy.on("finish", () => {
let promises = []
let imageUrls = []
imagesToUpload.forEach(imageToBeUploaded => {
imageUrls.push(`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o${folder}%2F${imageFileName}?alt=media`)
promises.push(admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
destination: `${folder}/${imageFileName}`,
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
}))
})
try{
await Promises.all(resolve)
res.status(200).json({msg: 'Successfully uploaded all images', imageUrls})
}catch(err){ res.status(500).json(err) }
});
busboy.end(req.rawBody);
With that you should be able to upload them all, it's just a matter of putting all promises inside an array and use the Promise.all method to wait for them to resolve. I made it with async/await because that's how I've been doing it but I suppose you would have no problem in doing it with the callbacks.
Also the code is messy but that's mostly because I dont know how to use this text editor, I hope you can still understand it 👀
Samuel Vera's answer is almost correct. There are some typos and a logic error when push to imageUrls array.
Here, the complete code fixed:
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs');
let fields = {};
const busboy = new BusBoy({ headers: request.headers });
let imageFileName = {};
let imagesToUpload = [];
let imageToAdd = {};
let imageUrls = [];
busboy.on('field', (fieldname, fieldvalue) => {
fields[fieldname] = fieldvalue;
});
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
if (mimetype !== 'image/jpeg' && mimetype !== 'image/png') {
return res
.status(400)
.json({ error: 'Wrong file type submitted!' });
}
// Getting extension of any image
const imageExtension = filename.split('.')[
filename.split('.').length - 1
];
// Setting filename
imageFileName = `${Math.round(Math.random() * 1000000000)}.${imageExtension}`;
// Creating path
const filepath = path.join(os.tmpdir(), imageFileName);
imageToAdd = {
imageFileName,
filepath,
mimetype,
};
file.pipe(fs.createWriteStream(filepath));
//Add the image to the array
imagesToUpload.push(imageToAdd);
});
busboy.on('finish', async () => {
let promises = [];
imagesToUpload.forEach((imageToBeUploaded) => {
imageUrls.push(
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageToBeUploaded.imageFileName}?alt=media`
);
promises.push(
admin
.storage()
.bucket()
.upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype,
},
},
})
);
});
try {
await Promise.all(promises);
return response.json({
message: `Images URL: ${imageUrls}`,
});
} catch (err) {
console.log(err);
response.status(500).json(err);
}
});
busboy.end(request.rawBody);
Anyway, thank you Samuel :)
I am trying to upload chunks of base64 to node js server and save those chunks into one file
let chunks = [];
app.post('/api', (req, res) => {
let {blob} = req.body;
//converting chunks of base64 to buffer
chunks.push(Buffer.from(blob, 'base64'));
res.json({gotit:true})
});
app.post('/finish', (req, res) => {
let buf = Buffer.concat(chunks);
fs.writeFile('finalvideo.webm', buf, (err) => {
console.log('Ahh....', err)
});
console.log('SAVED')
res.json({save:true})
});
Problem with the above code is video is not playable I don't why Am I really doing something wrong and I've also tried writable streams it is not working either
UPDATE - I
Instead of sending blobs I've implemented to send binary but even though I am facing a problem like TypeError: First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.
client.js
postBlob = async blob => {
let arrayBuffer = await new Response(blob).arrayBuffer();
let binary = new Uint8Array(arrayBuffer)
console.log(binary) // logging typed Uint8Array
axios.post('/api',{binary})
.then(res => {
console.log(res)
})
};
server.js
let chunks = [];
app.post('/api', (req, res) => {
let {binary} = req.body;
let chunkBuff = Buffer.from(binary) // This code throwing Error
chunks.push(chunkBuff);
console.log(chunkBuff)
res.json({gotit:true})
});
//Somehow combine those chunks into one file
app.post('/finish', (req, res) => {
console.log('Combinig the files',chunks.length);
let buf = Buffer.concat(chunks);
console.log(buf) //empty buff
fs.writeFile('save.webm', buf, (err) => {
console.log('Ahh....', err)
});
res.json({save:true})
});
UPDATE - II
I am able to receive the binary chunk and append to a stream but in the final video only first chunk is playing I don't know what happened to other chunks and the video ends.
code
const writeMyStream = fs.createWriteStream(__dirname+'/APPENDED.webm', {flags:'a', encoding:null});
app.post('/api', (req, res) => {
let {binary} = req.body;
let chunkBuff = Buffer.from(new Uint8Array(binary));
writeMyStream.write(chunkBuff);
res.json({gotit:true})
});
UPDATE - III
my client code | Note: I've tried other ways to upload blobs I've commented out
customRecordStream = stream => {
let recorder = new MediaStreamRecorder(stream);
recorder.mimeType = 'video/webm;codecs=vp9';
recorder.ondataavailable = this.postBlob
recorder.start(INT_REC)
};
postBlob = async blob => {
let arrayBuffer = await new Response(blob).arrayBuffer();
let binary = new Uint8Array(arrayBuffer)
axios.post('/api',{binary})
.then(res => {
console.log(res)
})
// let binaryUi8 = new Uint8Array(arrayBuffer);
// let binArr = Array.from(binaryUi8);
// // console.log(new Uint8Array(arrayBuffer))
//
// console.log(blob);
// console.log(binArr)
// let formData = new FormData();
// formData.append('fname', 'test.webm')
// formData.append("file", blob);
//
// console.log(formData,'Checjk Me',blob)
// axios({
// method:'post',
// url:'/api',
// data:formData,
// config: { headers: {'Content-Type': 'multipart/form-data' }}
// }).then(res => {
// console.log(res,'FROM SERBER')
//
// })
//
//
// .then(res => {
// console.log(res)
// })
// this.blobToDataURL(blob, (blobURL) => {
//
// axios.post('/api',{blob:blobURL})
// .then(res => {
// console.log(res)
// })
// })
};
I was able to get this working by converting to base64 encoding on the front-end with the FileReader api. On the backend, create a new Buffer from the data chunk sent and write it to a file stream. Some key things with my code sample:
I'm using fetch because I didn't want to pull in axios.
When using fetch, you have to make sure you use bodyParser on the backend
I'm not sure how much data you're collecting in your chunks (i.e. the duration value passed to the start method on the MediaRecorder object), but you'll want to make sure your backend can handle the size of the data chunk coming in. I set mine really high to 50MB, but this may not be necessary.
I never close the write stream explicitly... you could potentially do this in your /final route. Otherwise, createWriteStream defaults to AutoClose, so the node process will do it automatically.
Full working example below:
Front End:
const mediaSource = new MediaSource();
mediaSource.addEventListener('sourceopen', handleSourceOpen, false);
let mediaRecorder;
let sourceBuffer;
function customRecordStream(stream) {
// should actually check to see if the given mimeType is supported on the browser here.
let options = { mimeType: 'video/webm;codecs=vp9' };
recorder = new MediaRecorder(window.stream, options);
recorder.ondataavailable = postBlob
recorder.start(INT_REC)
};
function postBlob(event){
if (event.data && event.data.size > 0) {
sendBlobAsBase64(event.data);
}
}
function handleSourceOpen(event) {
sourceBuffer = mediaSource.addSourceBuffer('video/webm; codecs="vp8"');
}
function sendBlobAsBase64(blob) {
const reader = new FileReader();
reader.addEventListener('load', () => {
const dataUrl = reader.result;
const base64EncodedData = dataUrl.split(',')[1];
console.log(base64EncodedData)
sendDataToBackend(base64EncodedData);
});
reader.readAsDataURL(blob);
};
function sendDataToBackend(base64EncodedData) {
const body = JSON.stringify({
data: base64EncodedData
});
fetch('/api', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body
}).then(res => {
return res.json()
}).then(json => console.log(json));
};
Back End:
const fs = require('fs');
const path = require('path');
const express = require('express');
const bodyParser = require('body-parser');
const app = express();
const server = require('http').createServer(app);
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json({ limit: "50MB", type:'application/json'}));
app.post('/api', (req, res) => {
try {
const { data } = req.body;
const dataBuffer = new Buffer(data, 'base64');
const fileStream = fs.createWriteStream('finalvideo.webm', {flags: 'a'});
fileStream.write(dataBuffer);
console.log(dataBuffer);
return res.json({gotit: true});
} catch (error) {
console.log(error);
return res.json({gotit: false});
}
});
Inspired by #willascend answer:
Backend-side:
app.use(express.raw());
app.post('/video-chunck', (req, res) => {
fs.createWriteStream('myvideo.webm', { flags: 'a' }).write(req.body);
res.sendStatus(200);
});
Frontend-side:
mediaRecorder.ondataavailable = event => {
if (event.data && event.data.size > 0) {
fetch(this.serverUrl + '/video-chunck', {
method: 'POST',
headers: {'Content-Type': 'application/octet-stream'},
body: event.data
});
}
};
My express version is 4.17.1
i faced the same problem today
as a solution in back-end i used fs.appendfile
fs.appendFile(Path, rawData, function (err) {
if (err) throw err;
console.log('Chunck Saved!');
})
I decided to post this after extensive searching here (1, 2, 3 ) and here (1, 2) and many, many other related posts. I am loosing hope, but will not give up that easily :)
I'm using multer to upload a PNG image to mongo database:
const storage = new GridFsStorage({
url: 'mongodb://my_database:thisIsfake#hostName/my_database',
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => { // generating unique names to avoid duplicates
if (err) {
return reject(err);
}
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: 'media',
metadata : {
clientId : req.body.client_id // added metadata to have a reference to the client to whom the image belongs
}
};
resolve(fileInfo);
});
});
}
});
const upload = multer({storage}).single('image');
Then I create a stream and pipe it to response:
loader: function (req, res) {
var conn = mongoose.createConnection('mongodb://my_database:thisIsfake#hostName/my_database');
conn.once('open', function () {
var gfs = Grid(conn.db, mongoose.mongo);
gfs.collection('media');
gfs.files.find({ metadata : {clientId : req.body.id}}).toArray(
(err, files) => {
if (err) throw err;
if (files) {
const readStream = gfs.createReadStream(files[0].filename); //testing only with the first file in the array
console.log(readStream);
res.set('Content-Type', files[0].contentType)
readStream.pipe(res);
}
});
});
}
Postman POST request to end point results in response body being displayed as an image file:
In the front end I pass the response in a File object, read it and save the result in a src attribute of img:
findAfile(){
let Data = {
id: this.$store.state.StorePatient._id,
};
console.log(this.$store.state.StorePatient._id);
visitAxios.post('http://localhost:3000/client/visits/findfile', Data )
.then(res => {
const reader = new FileReader();
let file = new File([res.data],"image.png", {type: "image/png"});
console.log('this is file: ',file);
reader.readAsDataURL(file); // encode a string
reader.onload = function() {
const img = new Image();
img.src = reader.result;
document.getElementById('imgContainer').appendChild(img);
};
})
.catch( err => console.error(err));
}
My File object is similar to the one I get when using input field only bigger:
This is original file:
When inspecting element I see this:
Looks like data URI is where it should be, but it's different from the original image on file input:
Again, when I want to display it through input element:
onFileSelected(event){
this.file = event.target.files[0];
this.fileName = event.target.files[0].name;
const reader = new FileReader();
console.log(this.file);
reader.onload = function() {
const img = new Image();
img.src = reader.result;
document.getElementById('imageContainer').appendChild(img);
};
reader.readAsDataURL(this.file);
}
I get this:
But when reading it from the response, it is corrupted:
Postman gets it right, so there must be something wrong with my front-end code, right? How do I pass this gfs stream to my html?
I managed to make a POST request to fetch an image from MongoDB and save it in the server dir:
const readStream = gfs.createReadStream(files[0].filename);
const wstream = fs.createWriteStream(path.join(__dirname,"uploads", "fileToGet.jpg"));
readStream.pipe(wstream);
Then, I just made a simple GET request by adding an absolute path to the and finally delete the file after successful response:
app.get('/image', function (req, res) {
var file = path.join(dir, 'fileToGet.jpg');
if (file.indexOf(dir + path.sep) !== 0) {
return res.status(403).end('Forbidden');
}
var type = mime[path.extname(file).slice(1)] || 'text/plain';
var s = fs.createReadStream(file);
s.on('open', function () {
res.set('Content-Type', type);
s.pipe(res);
});
s.on('end', function () {
fs.unlink(file, ()=>{
console.log("file deleted");
})
});
s.on('error', function () {
res.set('Content-Type', 'text/plain');
res.status(404).end('Not found');
});