I'm glad to see AWS now supports multipart/form-data on AWS Lambda, but now that the raw data is in my lambda function how do I process it?
I see multiparty is a good multipart library in Node for multipart processing, but its constructor expects a request, not a raw string.
The input message I am receiving on my Lambda function (after the body mapping template has been applied) is:
{ "rawBody": "--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"Content-Type\"\r\n\r\nmultipart/mixed; boundary=\"------------020601070403020003080006\"\r\n--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"Date\"\r\n\r\nFri, 26 Apr 2013 11:50:29 -0700\r\n--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"From\"\r\n\r\nBob <bob#mg.mydomain.io>\r\n--ce0741b2-93d4-4865-a7d6-20ca51fe2689\r\nContent-Disposition: form-data; name=\"In-Reply-To\"\r...
etc and some file data.
The body mapping template I'm using is
{
"rawBody" : "$util.escapeJavaScript($input.body).replaceAll("\\'", "'")"
}
How can I parse this data to acecss the fields and files posted to my Lambda function?
busboy doesn't work for me in the "file" case. It didn't throw an exception so I couldn't handle exception in lambda at all.
I'm using aws-lambda-multipart-parser lib wasn't hard like so. It just parses data from event.body and returns data as Buffer or text.
Usage:
const multipart = require('aws-lambda-multipart-parser');
const result = multipart.parse(event, spotText) // spotText === true response file will be Buffer and spotText === false: String
Response data:
{
"file": {
"type": "file",
"filename": "lorem.txt",
"contentType": "text/plain",
"content": {
"type": "Buffer",
"data": [ ... byte array ... ]
} or String
},
"field": "value"
}
This worked for me - using busboy
credits owed to Parse multipart/form-data from Buffer in Node.js which I copied most of this from.
const busboy = require('busboy');
const headers = {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'OPTIONS, POST',
'Access-Control-Allow-Headers': 'Content-Type'
};
function handler(event, context) {
var contentType = event.headers['Content-Type'] || event.headers['content-type'];
var bb = new busboy({ headers: { 'content-type': contentType }});
bb.on('file', function (fieldname, file, filename, encoding, mimetype) {
console.log('File [%s]: filename=%j; encoding=%j; mimetype=%j', fieldname, filename, encoding, mimetype);
file
.on('data', data => console.log('File [%s] got %d bytes', fieldname, data.length))
.on('end', () => console.log('File [%s] Finished', fieldname));
})
.on('field', (fieldname, val) =>console.log('Field [%s]: value: %j', fieldname, val))
.on('finish', () => {
console.log('Done parsing form!');
context.succeed({ statusCode: 200, body: 'all done', headers });
})
.on('error', err => {
console.log('failed', err);
context.fail({ statusCode: 500, body: err, headers });
});
bb.end(event.body);
}
module.exports = { handler };
Building on #AvnerSo :s answer, here's a simpler version of a function that gets the request body and headers as parameters and returns a promise of an object containing the form fields and values (skipping files):
const parseForm = (body, headers) => new Promise((resolve, reject) => {
const contentType = headers['Content-Type'] || headers['content-type'];
const bb = new busboy({ headers: { 'content-type': contentType }});
var data = {};
bb.on('field', (fieldname, val) => {
data[fieldname] = val;
}).on('finish', () => {
resolve(data);
}).on('error', err => {
reject(err);
});
bb.end(body);
});
If you want to get a ready to use object, here is the function I use. It returns a promise of it and handle errors:
import Busboy from 'busboy';
import YError from 'yerror';
import getRawBody from 'raw-body';
const getBody = (content, headers) =>
new Promise((resolve, reject) => {
const filePromises = [];
const data = {};
const parser = new Busboy({
headers,
},
});
parser.on('field', (name, value) => {
data[name] = value;
});
parser.on('file', (name, file, filename, encoding, mimetype) => {
data[name] = {
filename,
encoding,
mimetype,
};
filePromises.push(
getRawBody(file).then(rawFile => (data[name].content = rawFile))
);
});
parser.on('error', err => reject(YError.wrap(err)));
parser.on('finish', () =>
resolve(Promise.all(filePromises).then(() => data))
);
parser.write(content);
parser.end();
})
Related
I'm trying to implement an API endpoint that allows for multiple file uploads.
I don't want to write any file to disk, but to buffer them and pipe to S3.
Here's my code for uploading a single file. Once I attempt to post multiple files to the the endpoint in route.js, it doesn't work.
route.js - I'll keep this as framework agnostic as possible
import Busboy from 'busboy'
// or const Busboy = require('busboy')
const parseForm = async req => {
return new Promise((resolve, reject) => {
const form = new Busboy({ headers: req.headers })
let chunks = []
form.on('file', (field, file, filename, enc, mime) => {
file.on('data', data => {
chunks.push(data)
})
})
form.on('error', err => {
reject(err)
})
form.on('finish', () => {
const buf = Buffer.concat(chunks)
resolve({
fileBuffer: buf,
fileType: mime,
fileName: filename,
fileEnc: enc,
})
})
req.pipe(form)
})
}
export default async (req, res) => {
// or module.exports = async (req, res) => {
try {
const { fileBuffer, ...fileParams } = await parseForm(req)
const result = uploadFile(fileBuffer, fileParams)
res.status(200).json({ success: true, fileUrl: result.Location })
} catch (err) {
console.error(err)
res.status(500).json({ success: false, error: err.message })
}
}
upload.js
import S3 from 'aws-sdk/clients/s3'
// or const S3 = require('aws-sdk/clients/s3')
export default (buffer, fileParams) => {
// or module.exports = (buffer, fileParams) => {
const params = {
Bucket: 'my-s3-bucket',
Key: fileParams.fileName,
Body: buffer,
ContentType: fileParams.fileType,
ContentEncoding: fileParams.fileEnc,
}
return s3.upload(params).promise()
}
I couldn't find a lot of documentation for this but I think I've patched together a solution.
Most implementations appear to write the file to disk before uploading it to S3, but I wanted to be able to buffer the files and upload to S3 without writing to disk.
I created this implementation that could handle a single file upload, but when I attempted to provide multiple files, it merged the buffers together into one file.
The one limitation I can't seem to overcome is the field name. For example, you could setup the FormData() like this:
const formData = new FormData()
fileData.append('file[]', form.firstFile[0])
fileData.append('file[]', form.secondFile[0])
fileData.append('file[]', form.thirdFile[0])
await fetch('/api/upload', {
method: 'POST',
body: formData,
}
This structure is laid out in the FormData.append() MDN example. However, I'm not certain how to process that in. In the end, I setup my FormData() like this:
Form Data
const formData = new FormData()
fileData.append('file1', form.firstFile[0])
fileData.append('file2', form.secondFile[0])
fileData.append('file3', form.thirdFile[0])
await fetch('/api/upload', {
method: 'POST',
body: formData,
}
As far as I can tell, this isn't explicitly wrong, but it's not the preferred method.
Here's my updated code
route.js
import Busboy from 'busboy'
// or const Busboy = require('busboy')
const parseForm = async req => {
return new Promise((resolve, reject) => {
const form = new Busboy({ headers: req.headers })
const files = [] // create an empty array to hold the processed files
const buffers = {} // create an empty object to contain the buffers
form.on('file', (field, file, filename, enc, mime) => {
buffers[field] = [] // add a new key to the buffers object
file.on('data', data => {
buffers[field].push(data)
})
file.on('end', () => {
files.push({
fileBuffer: Buffer.concat(buffers[field]),
fileType: mime,
fileName: filename,
fileEnc: enc,
})
})
})
form.on('error', err => {
reject(err)
})
form.on('finish', () => {
resolve(files)
})
req.pipe(form) // pipe the request to the form handler
})
}
export default async (req, res) => {
// or module.exports = async (req, res) => {
try {
const files = await parseForm(req)
const fileUrls = []
for (const file of files) {
const { fileBuffer, ...fileParams } = file
const result = uploadFile(fileBuffer, fileParams)
urls.push({ filename: result.key, url: result.Location })
}
res.status(200).json({ success: true, fileUrls: urls })
} catch (err) {
console.error(err)
res.status(500).json({ success: false, error: err.message })
}
}
upload.js
import S3 from 'aws-sdk/clients/s3'
// or const S3 = require('aws-sdk/clients/s3')
export default (buffer, fileParams) => {
// or module.exports = (buffer, fileParams) => {
const params = {
Bucket: 'my-s3-bucket',
Key: fileParams.fileName,
Body: buffer,
ContentType: fileParams.fileType,
ContentEncoding: fileParams.fileEnc,
}
return s3.upload(params).promise()
}
I am writing a handler as AWS Lambda Function which is supposed to wrap the content into the docx file. I am using the Content-Disposition in response headers in order to achieve the objective. Below is the code that I have written so far:
//Process Array Buffer and extract the plain content out of it
export const extractContent = async (data: Buffer) => {
return new Promise((resolve, reject) =>
textract.fromBufferWithMime(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
data,
{ preserveLineBreaks: true },
(err, content) => {
if (err) {
reject(err);
} else {
resolve(content);
}
}
)
);
};
export const handlerName = async (
event: APIGatewayProxyEvent
): Promise<APIGatewayProxyResult> => {
if (event.body === null) {
return ErrorResponse;
}
const result = await class.func(JSON.parse(event.body)); //returns an array buffer object
if (result instanceof Error) {
return ErrorResponse;
}
const content = await extractContent(result.data)
.then(res => res)
.catch(err => err);
const headers = {
"Access-Control-Allow-Origin": "*",
"Strict-Transport-Security": "'max-age=31536000'",
"Access-Control-Expose-Headers": "Content-Disposition",
"Content-Disposition": attachment; filename=fn.docx; filename*=UTF-8''fn.docx,
"Content-Type": "application/vnd.ms-word.document"
};
return {
body: content,
headers,
statusCode: 200,
};
};
Now only string can be returned in APIGatewayResponse; hence I am not able to return byte object | stream | buffer instead.
I am able to download the docx file but it is not getting opened in MS-Word; I am always getting a detailed error that either file is corrupt or some part is invalid or missing.
I have already tried the base64 encoded string but still not able to get what is desired. Please suggest a solution to it.
I writing the handler in typescript Node using the SLS framework.
I have a REST API that upload images to s3 and returns the response. The API works perfectly using Postman.
The problem arrises when calling the API from frontend. I am using Angular 6.
I am getting Error: Unsupported content type: application/json error. Although I am setting the headers properly.
Here is my Angular 6 code.
export class UploadComponent {
percentDone: number;
uploadSuccess: boolean;
constructor(private http: HttpClient) {}
upload(file: File) {
this.singleBasicUpload(file);
}
singleBasicUpload(file: File) {
const headers = new HttpHeaders({
'Content-Type': 'multipart/form-data',
});
const options = { headers: headers };
this.http.post(`${BASE_URL}/upload`, file, options).subscribe(response => {
console.log('response', response);
});
}
}
And here is my S3 code in backend Node.js
AWS.config.update({
accessKeyId: constants.IAM_USER_KEY,
secretAccessKey: constants.IAM_USER_SECRET,
});
const BUCKET_NAME = constants.BUCKET_NAME;
const ACL = 'public-read';
const S3 = new AWS.S3();
export async function S3Upload(req, res) {
const chunks = [];
let fname;
let fileType;
let fileEncodingType;
const busboy = new Busboy({
headers: req.headers,
});
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
fname = filename.replace(/ /g, '_');
fileType = mimetype;
fileEncodingType = encoding;
file.on('data', data => {
// you will get chunks here will pull all chunk to an array and later concat it.
console.log(chunks.length);
chunks.push(data);
});
file.on('end', () => {
console.log(`File [${filename}] Finished`);
});
});
busboy.on('finish', () => {
const userId = UUID();
const params = {
Bucket: BUCKET_NAME, // your s3 bucket name
Key: `${userId}-${fname}`,
Body: Buffer.concat(chunks), // concatinating all chunks
ACL,
ContentEncoding: fileEncodingType, // optional
ContentType: fileType, // required
};
// we are sending buffer data to s3.
S3.upload(params, (err, s3res) => {
if (err) {
res.send({
err,
status: 'error',
});
} else {
return res.send({
data: s3res,
message: 'Image successfully uploaded.',
});
}
});
});
req.pipe(busboy);
}
Network description
I am working on an ios app which sends images and text to my firebase server using mutipart/form-data URLRequest. In order to process the data in my cloud function, I am using the method mentioned in documentation to parse the mutipart/form-data into JSON format, and here is my code:
const Busboy = require('busboy');
exports.test = functions.https.onRequest((req, res) => {
console.log("start");
console.log(req.rawBody.toString());
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers});
busboy.on('field', (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
console.log('field');
});
busboy.on('finish', function() {
console.log('finish');
res.json({
data: null,
error: null
});
});
req.pipe(busboy);
} else {
console.log('else...');
}
});
However, the above code doesn't seem to work, and here is the output from console:
Function execution started
start
--Boundary-43F22E06-B123-4575-A7A3-6C144C213D09
Content-Disposition: form-data; name="json"
{"name":"Alex","age":"24","friends":["John","Tom","Sam"]}
--Boundary-43F22E06-B123-4575-A7A3-6C144C213D09--
finish
Function execution took 517 ms, finished with status code: 200
As you can see, the on('field') function never execute. What did I miss?
Also, here is the code in swift for sending httpRequest:
var request = URLRequest(url: myCloudFunctionURL)
request.httpMethod = "POST"
request.setValue("multipart/form-data; boundary=myBoundary", forHTTPHeaderField: "Content-Type")
request.addValue(userToken, forHTTPHeaderField: "Authorization")
request.httpBody = myHttpBody
let session = URLSession.shared
session.dataTask(with: request) { (data, response, requestError) in
// callback
}.resume()
You will have to call busboy.end(req.rawBody); instead of req.pipe(busboy) as described in the example of the documentation. I dont know why .pipe doesnt work. Calling .end will produce the same result but with a different way.
const Busboy = require('busboy');
exports.helloWorld = functions.https.onRequest((req, res) => {
const busboy = new Busboy({ headers: req.headers });
let formData = {};
busboy.on('field', (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
// We're just going to capture the form data in a JSON document.
formData[fieldname] = val;
console.log('Field [' + fieldname + ']: value: ' + val)
});
busboy.on('finish', () => {
res.send(formData);
});
// The raw bytes of the upload will be in req.rawBody.
busboy.end(req.rawBody);
});
Enjoy this simple express middleware which converts all the Content-Type: multipart/form-data into you req.body in json format :)
const Busboy = require('busboy');
const expressJsMiddleware = (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
let formData = {};
busboy.on(
"field",
(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
formData = { ...formData, [fieldname]: val };
},
);
busboy.on("finish", () => {
req.body = formData;
next();
});
req.pipe(busboy);
};
I'm trying to send a file to another node.js service. So for that i'm using http and form-data modules.
This is the code i wrote
function uintToString(uintArray) {
return String.fromCharCode.apply(null, new Uint8Array(uintArray));
}
function (file) {
var data = uintToString(file.buffer);
//
var crlf = "\r\n",
boundaryKey = Math.random().toString(16),
boundary = `--${boundaryKey}`;
delimeter = `${crlf}--${boundary}`,
preamble = "", // ignored. a good place for non-standard mime info
epilogue = "",
headers = [
'Content-Disposition: form-data; name="file"; filename="' + name + '"' + crlf
],
closeDelimeter = `${delimeter}--`,
multipartBody = Buffer.concat(
new Buffer(preamble + delimeter + crlf + headers.join('') + crlf),
data,
new Buffer(closeDelimeter + epilogue)
);
var options = {
host: 'localhost',
port: 3000,
method: 'POST',
path: '/data/get',
headers: {
'Content-Type': 'multipart/form-data; boundary=' + boundary,
'Content-Length': formData._valueLength
}
};
//make request
return httpsRequest(formData, options)
.then((result) => {
console.log(result);
}, (err) => {
console.log(err);
});
};
function httpsRequest(data, options) {
return new Promise(function (resolve, reject) {
// request object
var req = https.request(options, function (res) {
var result = '';
res.on('data', function (chunk) {
result += chunk;
});
res.on('end', function () {
console.log("https end result - " + result);
resolve(result);
});
res.on('error', function (err) {
reject(err);
})
});
// req error
req.on('error', function (err) {
reject(err);
});
//send request witht the postData form
req.write(data);
req.end();
});
}
It is giving ""list" argument must be an Array of Buffers" this error. It looks like something is wrong on the httpsRequest function.
Don't reinvent the wheel, needle/request can do this for you. If you want to promisify things, use bluebird
const Promise = require('bluebird')
const needle = Promise.promisifyAll(require('needle'))
function send (file) {
let url = 'https://localhost/data/get'
let data = {
zip_file: {
buffer : file.buffer,
filename : name,
content_type : 'application/octet-stream'
}
}
return needle.postAsync(url, data, { multipart: true })
}