Mutipart/form-data to JSON in Node.js using Busboy - node.js

I am working on an ios app which sends images and text to my firebase server using mutipart/form-data URLRequest. In order to process the data in my cloud function, I am using the method mentioned in documentation to parse the mutipart/form-data into JSON format, and here is my code:
const Busboy = require('busboy');
exports.test = functions.https.onRequest((req, res) => {
console.log("start");
console.log(req.rawBody.toString());
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers});
busboy.on('field', (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
console.log('field');
});
busboy.on('finish', function() {
console.log('finish');
res.json({
data: null,
error: null
});
});
req.pipe(busboy);
} else {
console.log('else...');
}
});
However, the above code doesn't seem to work, and here is the output from console:
Function execution started
start
--Boundary-43F22E06-B123-4575-A7A3-6C144C213D09
Content-Disposition: form-data; name="json"
{"name":"Alex","age":"24","friends":["John","Tom","Sam"]}
--Boundary-43F22E06-B123-4575-A7A3-6C144C213D09--
finish
Function execution took 517 ms, finished with status code: 200
As you can see, the on('field') function never execute. What did I miss?
Also, here is the code in swift for sending httpRequest:
var request = URLRequest(url: myCloudFunctionURL)
request.httpMethod = "POST"
request.setValue("multipart/form-data; boundary=myBoundary", forHTTPHeaderField: "Content-Type")
request.addValue(userToken, forHTTPHeaderField: "Authorization")
request.httpBody = myHttpBody
let session = URLSession.shared
session.dataTask(with: request) { (data, response, requestError) in
// callback
}.resume()

You will have to call busboy.end(req.rawBody); instead of req.pipe(busboy) as described in the example of the documentation. I dont know why .pipe doesnt work. Calling .end will produce the same result but with a different way.
const Busboy = require('busboy');
exports.helloWorld = functions.https.onRequest((req, res) => {
const busboy = new Busboy({ headers: req.headers });
let formData = {};
busboy.on('field', (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
// We're just going to capture the form data in a JSON document.
formData[fieldname] = val;
console.log('Field [' + fieldname + ']: value: ' + val)
});
busboy.on('finish', () => {
res.send(formData);
});
// The raw bytes of the upload will be in req.rawBody.
busboy.end(req.rawBody);
});

Enjoy this simple express middleware which converts all the Content-Type: multipart/form-data into you req.body in json format :)
const Busboy = require('busboy');
const expressJsMiddleware = (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
let formData = {};
busboy.on(
"field",
(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
formData = { ...formData, [fieldname]: val };
},
);
busboy.on("finish", () => {
req.body = formData;
next();
});
req.pipe(busboy);
};

Related

React: Multer image and data upload with using firebase function, node multer

Basically I am using firebase function and hosting with node and react. I can upload image courtesy of How to perform an HTTP file upload using express on Cloud Functions for Firebase (multer, busboy)
but how do you upload image and data at the same time?
export const addProduct = (product, imageUrl) => {
return (dispatch) => {
return new Promise((resolve, reject) => {
const fileData = new FormData();
fileData.append("imageUrl", imageUrl);
fileData.append("productData", product);
axios({
method: "post",
url: "/api/products/add-product",
data: fileData,
headers: {
"Content-Type": "multipart/form-data",
},
});
});
};
};
NodeJS
const router = express.Router();
const Busboy = require("busboy");
router.post("/api/products/add-product", async (req, res, next) => {
if (req.method === "POST") {
const busboy = new Busboy({ headers: req.headers });
const uploads = {};
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
console.log(
`File [${fieldname}] filename: ${filename}, encoding: ${encoding}, mimetype: ${mimetype}`
);
});
}
});
Your client-side code looks OK.
On the server-side you can tell busboy to extract fields as well as files:
const fields = {};
const files = [];
const busboy = new Busboy({headers: req.headers});
busboy.on("field", (key, value) => (fields[key] = value));
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {/*...*/});
busboy.end(req.rawBoy);
This way you can access fields["productData"] later in your code.
Note that you need to use rawBody to access the unparsed body in Cloud Functions: https://firebase.google.com/docs/functions/http-events#read_values_from_the_request

Node js file upload issue

I'm trying to upload a file to nanosets API. I uploaded the following node js function to firebase and trying to excess it with following URL for example with a file in body (trying to test this with postman)
Node js function looks like:
exports.uploadFile = functions.https.onRequest((req, res) => {
cors(req, res, () => {
if (req.method !== "POST") {
return res.status(500).json({
message: "Not allowed"
});
}
const busboy = new Busboy({headers: req.headers});
let uploadData = null;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const filepath = path.join(os.tmpdir(), filename);
uploadData = {modelId: '4bc54977-60cf-4415-a417-c39f1c18b83f', file: fs.createReadStream(filename), type: mimetype};
const options = {
url: 'https://app.nanonets.com/api/v2/OCR/Model/XXXXXXX-60cf-4415-a417-c39f1c18b83f/LabelFile/',
formData: uploadData,
headers: {
'Authorization': 'Basic ' + Buffer.from('tiOJNxuDbdl40lXXXXXXXXXXFTYbY' + ':').toString('base64')
}
};
request.post(options, function (err, httpResponse, body) {
if (err) {
console.log(err);
}
console.log(body)
});
});
busboy.on("finish", () => {
res.status(200).json({
message: "It worked!"
});
});
busboy.end(req.rawBody);
});
});
Why I check the logs with firebase functions:log I get the following results:
2020-06-06T09:35:06.168774140Z D uploadFile: Function execution started
2020-06-06T09:35:06.344Z I uploadFile: invoice_4.pdf
2020-06-06T09:35:06.432Z I uploadFile: FileStream {
2020-06-06T09:35:06.439Z E uploadFile: TypeError: source.pause is not a function
Anyone an idea? How to pass the file to nanosets?
In order to make outgoing requests from Cloud Functions, your project must be on the Blaze payment plan.

busboy not emitting field event

In the below code, I could see only the finish event emitting and not the field event.
const Busboy = require('busboy');
module.exports.controller = function(app) {
app.post('/api', function(req, res) {
var busboy = new Busboy({ headers: req.headers });
var formdata = {};
busboy.on('field', function (fieldname, val) {
console.log("fieldname : " + fieldname);
console.log("value : " + val);
formdata[fieldname] = val;
});
busboy.on('finish', function() {
res.send(formdata);
});
req.pipe(busboy);
});
};
As response, I receive {} back in postman. What could be possibly going wrong here?
I know this is long overdue, but I had this issue too and then I realized I was missing the name="" field in my HTML inputs.
Working snippet:
// handle form submissions
router.post('/submit', (req, res) => {
var busboy = new Busboy({ headers: req.headers });
busboy.on('field', function (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
console.log('Field [' + fieldname + ']: value: ' + val);
});
busboy.on('finish', () => {
res.writeHead(303, { Connection: 'close', Location: '/' });
res.end();
});
req.pipe(busboy);
});

writing file from post request in ExpressJS

Working with express to create a file transfer tool, and I've almost gotten everything completed. Just need to figure out how to get the data from the request written to file.
My issue appears to be stemming from not knowing where the file contents are placed in the request object.
My code to process sending the request
let file = watcher.getOneFile(config.thisLocation);
console.dir(file);
let contents = fs.readFileSync(file.fullPath, 'utf-8');
console.log(contents);
let form = {
attachments: [
contents
]
}
rq.post({
url: `http://${homeAddress}:${port}/files/?loc=${config.thisLocation}&file=${file.fileName}`,
headers: {'content-type':'application/x-www-form-urlencoded'},
formData: form
}, (err, res, body) => {
// body = JSON.parse(body);
console.log(body);
});
and when I get the request on the server, I'm not sure where the file contents actually are.
Code for handling the request
app.post('/files', (req, res) => {
console.log(req.query.loc);
// console.dir(req);
let incoming = watcher.getOutputPath(req.query.loc, config.locations);
console.log(incoming);
console.dir(req.body);
// console.log(req.body);
// let body = JSON.parse(req.body);
console.log(req.query);
let filename = path.join(incoming, req.query.file);
console.log(filename);
fs.writeFile(filename, req.body, (err) => {
if(err){
console.error(err);
}
console.log(`Successfully wrote file: ${path.join(incoming, req.query.file)}`);
});
res.sendStatus(200);
});
Where on the Request Object is the file contents?
Unfortunately you can't access the file content in any straightforward way. I recommend you to use busboy or similar package to parse form-data requests.
Here is how can you read file content using busboy and write it to the file system:
const Busboy = require('busboy');
app.post('/files', (req, res) => {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldname, file, filename, encoding, mime) => {
const newFilename = `${Date.now()}_${filename}`,
newFile = fs.createWriteStream(newFilename);
file.pipe(newFile);
file.on('end', () => {
console.log(`Finished reading ${filename}`);
});
});
busboy.on('finish', () => {
console.log('Finished parsing form');
res.sendStatus(200);
});
req.pipe(busboy);
});

Parse multipart/form-data from body as string on AWS Lambda

I'm glad to see AWS now supports multipart/form-data on AWS Lambda, but now that the raw data is in my lambda function how do I process it?
I see multiparty is a good multipart library in Node for multipart processing, but its constructor expects a request, not a raw string.
The input message I am receiving on my Lambda function (after the body mapping template has been applied) is:
{ "rawBody": "--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"Content-Type\"\r\n\r\nmultipart/mixed; boundary=\"------------020601070403020003080006\"\r\n--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"Date\"\r\n\r\nFri, 26 Apr 2013 11:50:29 -0700\r\n--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"From\"\r\n\r\nBob <bob#mg.mydomain.io>\r\n--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"In-Reply-To\"\r...
etc and some file data.
The body mapping template I'm using is
{
"rawBody" : "$util.escapeJavaScript($input.body).replaceAll("\\'", "'")"
}
How can I parse this data to acecss the fields and files posted to my Lambda function?
busboy doesn't work for me in the "file" case. It didn't throw an exception so I couldn't handle exception in lambda at all.
I'm using aws-lambda-multipart-parser lib wasn't hard like so. It just parses data from event.body and returns data as Buffer or text.
Usage:
const multipart = require('aws-lambda-multipart-parser');
const result = multipart.parse(event, spotText) // spotText === true response file will be Buffer and spotText === false: String
Response data:
{
"file": {
"type": "file",
"filename": "lorem.txt",
"contentType": "text/plain",
"content": {
"type": "Buffer",
"data": [ ... byte array ... ]
} or String
},
"field": "value"
}
This worked for me - using busboy
credits owed to Parse multipart/form-data from Buffer in Node.js which I copied most of this from.
const busboy = require('busboy');
const headers = {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'OPTIONS, POST',
'Access-Control-Allow-Headers': 'Content-Type'
};
function handler(event, context) {
var contentType = event.headers['Content-Type'] || event.headers['content-type'];
var bb = new busboy({ headers: { 'content-type': contentType }});
bb.on('file', function (fieldname, file, filename, encoding, mimetype) {
console.log('File [%s]: filename=%j; encoding=%j; mimetype=%j', fieldname, filename, encoding, mimetype);
file
.on('data', data => console.log('File [%s] got %d bytes', fieldname, data.length))
.on('end', () => console.log('File [%s] Finished', fieldname));
})
.on('field', (fieldname, val) =>console.log('Field [%s]: value: %j', fieldname, val))
.on('finish', () => {
console.log('Done parsing form!');
context.succeed({ statusCode: 200, body: 'all done', headers });
})
.on('error', err => {
console.log('failed', err);
context.fail({ statusCode: 500, body: err, headers });
});
bb.end(event.body);
}
module.exports = { handler };
Building on #AvnerSo :s answer, here's a simpler version of a function that gets the request body and headers as parameters and returns a promise of an object containing the form fields and values (skipping files):
const parseForm = (body, headers) => new Promise((resolve, reject) => {
const contentType = headers['Content-Type'] || headers['content-type'];
const bb = new busboy({ headers: { 'content-type': contentType }});
var data = {};
bb.on('field', (fieldname, val) => {
data[fieldname] = val;
}).on('finish', () => {
resolve(data);
}).on('error', err => {
reject(err);
});
bb.end(body);
});
If you want to get a ready to use object, here is the function I use. It returns a promise of it and handle errors:
import Busboy from 'busboy';
import YError from 'yerror';
import getRawBody from 'raw-body';
const getBody = (content, headers) =>
new Promise((resolve, reject) => {
const filePromises = [];
const data = {};
const parser = new Busboy({
headers,
},
});
parser.on('field', (name, value) => {
data[name] = value;
});
parser.on('file', (name, file, filename, encoding, mimetype) => {
data[name] = {
filename,
encoding,
mimetype,
};
filePromises.push(
getRawBody(file).then(rawFile => (data[name].content = rawFile))
);
});
parser.on('error', err => reject(YError.wrap(err)));
parser.on('finish', () =>
resolve(Promise.all(filePromises).then(() => data))
);
parser.write(content);
parser.end();
})

Resources