Save video file received from multipart form in Serverless Offline - node.js

I have an website running Angular4 with a simple form uploading data using ng2-file-upload. I'm sending those files to a Node.js-based serverless offline server where my intention is to simply write those files received from the form to disk.
I tried to do in many different ways, and in the end I found this right here that parses that form from the event into a json. The resulting json contains a buffer in one of the fields with the video data like so:
{ Host: 'localhost:3000',
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0',
Accept: '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate',
Referer: 'http://localhost:4200/myupload',
'Content-Length': 2391623,
'Content-Type': 'multipart/form-data; boundary=---------------------------2125290100942661667805976894',
Origin: 'http://localhost:4200',
Connection: 'keep-alive' }
{ file:
{ type: 'file',
filename: 'y9K18CGEeiI.webm',
contentType: 'video/webm',
content: <Buffer 1a 45 e3 01 00 00 00 00 00 00 1f 42 fd fd 01 42 fd fd 01 42 fd 04 42 fd 08 42 fd fd 77 65 62 6d 42 fd fd 02 42 fd fd 02 18 53 fd 67 01 00 00 00 00 14 ... > } }
Now what I'm trying to do is to save the file in the buffer using fs:
module.exports.handler = (event, context, callback) => {
let data = multipart.parse(event, false);
fs.writeFile('meme.webm', data.file.content, 'binary', function(err) {
if(err) {
console.log(err);
} else {
console.log('saved!');
}
});
// etc ...
};
The file saves to the disk with the correct size (1.3MB), same as the original file. Unfortunately, I can't seem to open it on the other side and I assume it's either because othe encoding or because of the way I'm writing it to disk. Any ideas?

For anyone with this problem, check this issue right here. It's a problem with serverless offline converting file data and there's not much that can be done it seems other than applying the fork.

Related

Send multipart/form-data with axios in AWS Lambda function: source.on is not a function

I have a lambda function written in node js. My front end application is sending formdata in the body as a POST request to my lambda function. The goal of the lambda is to send the formData to a backend service. My issue is I don't know how to process the formdata within the lambda function.
I have the following modules packages: lambda-multipart-parser and form-data.
This is the output of the lambda-parser:
const parser = require('lambda-multipart-parser');
// inside of handler
const result = await parser.parse(event);
console.log('result',result)
Result:
result {
files: [
{
content: <Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 01 00 01 2c 01 2c 00 00 ff e1 03 66 45 78 69 66 00 00 4d 4d 00 2a 00 00 00 08 00 09 01 0f 00 02 00 00 00 06 00 00 ... 141038 more bytes>,
filename: 'IMG_0037 - Copy.JPG',
contentType: 'image/jpeg',
encoding: '7bit',
fieldname: 'files'
},
{
content: <Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 01 00 01 2c 01 2c 00 00 ff e1 03 66 45 78 69 66 00 00 4d 4d 00 2a 00 00 00 08 00 09 01 0f 00 02 00 00 00 06 00 00 ... 141038 more bytes>,
filename: 'IMG_0037.JPG',
contentType: 'image/jpeg',
encoding: '7bit',
fieldname: 'files'
}
]
Then I append the results to form data and try and make the request:
// Add files to formData
const formData = new FormData();
result.files.forEach(file => {
formData.append('files', file);
});
const formHeaders = formData.getHeaders();
formData.submit({
host: www.example.com,
headers: formHeaders
})
I then get the following error: source.on is not a function
I've looked online and tried quite a few different things but I can't seem to get this to work.
Any help would be much appreciated!
Thank You
EDIT
Complete stack trace
TypeError: source.on is not a function
at Function.DelayedStream.create (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\delayed-stream\lib\delayed_stream.js:33:10)
at FormData.CombinedStream.append (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\combined-stream\lib\combined_stream.js:45:37)
at FormData.append (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\form-data\lib\form_data.js:75:3)
at C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\.webpack\service\src\upload\handler.js:1:3543
at Array.forEach (<anonymous>)
at r.handler (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\.webpack\service\src\upload\handler.js:1:3529)
at processTicksAndRejections (internal/process/task_queues.js:97:5)
at async InProcessRunner.run (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\serverless-offline\dist\lambda\handler-runner\in-process-runner\InProcessRunner.js:211:24)
at async LambdaFunction.runHandler (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\serverless-offline\dist\lambda\LambdaFunction.js:355:20)
at async hapiHandler (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\serverless-offline\dist\events\http\HttpServer.js:533:18)
at async module.exports.internals.Manager.execute (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\#hapi\hapi\lib\toolkit.js:45:28)
at async Object.internals.handler (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\#hapi\hapi\lib\handler.js:46:20)
at async exports.execute (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\#hapi\hapi\lib\handler.js:31:20)
at async Request._lifecycle (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\#hapi\hapi\lib\request.js:312:32)
at async Request._execute (C:\Users\206591256\Documents\Personal\Sage-Projects\Eagle\football\eagle-api\node_modules\#hapi\hapi\lib\request.js:221:9)
offline: (λ: upload) RequestId: ckxajn1hn0002i4nwegzrbsx5 Duration: 283.18 ms Billed Duration: 284 ms

How to convert image buffer to form-data in node.js?

I have a buffer of an image like:
<Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 01 00 00 01 00 01 00 00 ff db 00 43 00 04 03 03 03 03 02 04 03 03 03 04 04 04 05 06 0a 06 06 05 05 06 0c 08 09 07 ... 231835 more bytes>
and I have installed form-data module from npm.
I need to send this image file as form-data, I tried:
const form = new FormData();
form.append("image", buffer, { filename: "london.jpg" });
But it didn't work, how can I solve this problem?
I finally found a way to resolve the issue by using request module.
https://www.npmjs.com/package/request
request.post({
url,
formData: {
image: {
value: file.buffer, // Give your node.js buffer to here
options: {
filename: file.originalname, // filename
contentType: file.mimetype // file content-type
}
}
}
},(error, http_response, body) => {
});

The "chunk" argument must be of type string or an instance of Buffer. Received an instance of ArrayBuffer

I have passed an image obtained from Microsoft Graph API to pictureURL() function.
But getting this error:
TypeError [ERR_INVALID_ARG_TYPE]: The "chunk" argument must be of type string or an instance of Buffer. Received an instance of ArrayBuffer
at ClientRequest.end (_http_outgoing.js:768:13)
at features.constructor.writeBody (K:\Project\project-backend\node_modules\aws-sdk\lib\http\node.js:137:14)
at features.constructor.handleRequest (K:\Project\project-backend\node_modules\aws-sdk\lib\http\node.js:105:12)
at executeSend (K:\Project\project-backend\node_modules\aws-sdk\lib\event_listeners.js:342:29)
at Request.SEND (K:\Project\project-backend\node_modules\aws-sdk\lib\event_listeners.js:356:9)
I want to upload the image obtained from Graph API to s3 bucket.
Microsoft Graph API for getting the image of a user-
https://graph.microsoft.com/v1.0/users/${id | userPrincipalName}/photo/$value
Function in which I am passing the obtained image is -
async pictureURL(image, userID) {
try {
const uploadParams = { Bucket: config.s3bucket.name };
const bufferData = await image.arrayBuffer();
uploadParams.Body = bufferData;
uploadParams.Key = 'example.jpg'// path.basename(image);
s3.putObject(uploadParams, (err, data) => {
if (err) {
console.log('Error', err);
return;
}
console.log('Upload Success', data.Location);
return data.Location;
});
} catch (error) {
console.log(error)
}
}
image format returned by the graph-api -
image: Blob {
[Symbol(type)]: 'image/jpeg',
[Symbol(buffer)]: <Buffer ff d8 ff e0 00 10 4a 46 49 46 00 01 01 01 00 60 00 60 00 00 ff db 00 43 00 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 ... 4355 more bytes>
}

How to change axios content from "text/html" to application/json type in Nestjs

I discovered that axios is returning a string instead of a valid json.
headers:Object {date: "Tue, 02 Jun 2020 08:44:06 GMT", server: "Apache", connection: "close", …}
connection:"close"
content-type:"text/html; charset=UTF-8"
date:"Tue, 02 Jun 2020 08:44:06 GMT"
server:"Apache"
transfer-encoding:"chunked"
how do I change the content-type to application/json in NestJs application?
I tried this but didnt not work
const meterInfo = await this.httpService.get(url, { headers: { "Content-Type": "application/json" } }).toPromise();
Here is the invalid json returned.
"{"status":"00","message":"OK","access_token":"2347682423567","customer":{"name":"John Doe","address":"Mr. John Doe 34 Tokai, leaflet. 7999.","util":"Demo Utility","minimumAmount":"13897"},"response_hash":"c43c9d74480f340f55156f6r5c56487v8w"}"
Instead of sending a Content-Type header, you should send an Accept header with the same MIME type. This tells the server what you are expecting to receive, and if Content Negotiation is set up properly, it will allow you to get a JSON back instead of that weird string.
this.httpService.get(
url,
{
headers: {
'Accept': 'application/json',
},
},
).toPromise();
If that doesn't work, you'll need to provide your own serializer to take the string from that wonky format to JSON, or get in touch with the server admins and see if they can provide you better documentation about how to consume their API.

Node.js how to ungzip GET response made through TCP socket

When i send a GET request to Hearthstone website, and once i've concatenated all the sent frames, i still get a compressed string
My request:
GET /hearthstone/en/ HTTP/1.1
Host: eu.battle.net
Connection: keep-alive
Cache-Control: max-age=0
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
Referer: https://www.google.fr/
Accept-Encoding: gzip, deflate
Accept-Language: en-US,en;q=0.8,fr;q=0.6
Cookie: eu-cookie-compliance-agreed=1; _ga=GA1.3.780909635.1489783325; _gat_bnetgtm=1
My logs:
[Socket] > Connected !
Connected to 185.60.115.40
[Socket - Data] > Received data !
--- Header ---
HTTP/1.1 200 OK
Date: Sat, 18 Mar 2017 10:56:12 GMT
Server: Apache
X-Frame-Options: SAMEORIGIN
Retry-After: 600
Content-Language: en-GB
Vary: Accept-Encoding
Content-Encoding: gzip
Keep-Alive: timeout=5, max=4000
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html;charset=UTF-8
4331
▼ ♥��{�∟W�'��
�1�4_`�4→������s㑑�*V��H��(�2����s�=��8w��▼v���▲�3���d���[�k��N�rw�_U��������↔��xd�q<zIe♠T�Ƅe�݁��1▼졲b��|#▼w���l8U���ɠ�L�z�(�?�6��b♠e���y�bw�N�XUy>��↑↑"�*�U���MJ50F♦�Z�_�_�6���↓�R>Q�j☼��*9V;b�ȧj�nZ��j��~�XQ��U�)��$IE�&�'l��'i�n��R[�&iv`¶j☻�����/TRcT☻��E+&�♥6e��∟f�→i\�◄�☻��♣=������+]�Ŗ�|)�§)���r<e�v⸉♀}���Ȳm�EI�Ca{�u_�♫z,���↑���:,[��X�T6J�����§�����♂"?◄▲w�g♣Q �(♠Fu4S
l�-��l�
������~K�>*�Leu�T�(�↓�ָ-p ���q��'j(����▲$M�9����*tz►d`����T↔��[=�?Le��+§ȯL��c�YZ���R�
♀�zY�DYBL��♦�� ‼��n�y������gh�x²r�f����ady1e�&�P������Y�ͭ�►7�y1��}
v�����,�◄‼-��
,�X�$n∟Z>�=�$��1���k���'*rc�☼bK��§$�r�ͅ�����+�X�§♂/f�↔s♂§mΤ�#ԳC�ZV↑� g☺�l7�<♠f��$t���*����H►��'�Jҗ�׍�z�����hU*i↕��
��w�΃f�↕�FA�8,f�=�4QQ�]�♫-� �#8♥��V■♂� �ȋ�������g♥♣▼�!�B˶D↕A↑E<H,♂� dQ1�N ��sm'♫�L☻m()� f��<I֯����\♠Y�H�TH�8�\�Cԡ7cGz�↔p x Ӄ,��P��k�>wb�8�#� i�‼~∟+t�▼♦�+"W��‼:��#�Z�Q� �0r�↔8"�↔?N<��aU�Q��������8�↓w��B��VB�Z♫�☺↨◄�9���l��§��P▼H1?☻Nv¶J.l
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket - Data] > Received data !
[Socket] > End message received !
--- HTML ---
<Buffer 48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d 0a 44 61 74 65 3a 20 53 61 74 2c 20 31 38 20 4d 61 72 20 32 30 31 37 20 31 30 3a 35 36 3a 31 32 20 47 ... >
My code:
send ()
{
let chuncks = []
this.client.socket.once('data', data =>
{
/* Header */
chuncks.push(data)
console.log('--- Header ---')
console.log(data.toString())
this.client.socket.on('data', data =>
{
chuncks.push(data)
})
})
this.client.socket.on('end', () =>
{
let html = Buffer.concat(chuncks)
console.log('--- HTML ---')
console.log(html)
let decoded = require('zlib').gunzipSync(html)
})
this.client.write( this.request ) // Above request
}
And, what i don't understand, is why things goes wrong, Am i make this properly ?
Thanks !

Resources