I'm downloading a file from remote server using axios post method,then saving to the disk.
Once the file is saved to disk I need to send it to client in response.
let response = await axios.post(url, body, {headers :headers, responseType:'stream'})
let filePath = "reports/" + response.headers['content-disposition'].split("filename=")[1].replace(/\"/g, "");
await response.data.pipe(fs.createWriteStream(filePath))
res.download(filePath);
The problem I'm facing is, Response is sent when the file writing is still in progress.
If there is any alternative, Please suggest.
EDIT:-
The problem was with the write stream.
Solution:-
let response = await axios.post(url, body, {headers :headers, responseType:'stream'});
let filePath = "reports/" + response.headers['content-disposition'].split("filename=")[1].replace(/\"/g, "");
let ws = fs.createWriteStream(filePath);
new Promise((resolve, reject) => {
ws.on('open', () => {
response.data.pipe(ws);
}).on('error', function (err) {
reject(res.send({error: "message"}));
}).on('finish', function () {
ws.end();
resolve(res.download(filePath));
});
});
Link:- fs.createWriteStream does not immediately create file?
I think you should use Promise.All to resolve all promises before moving on.
As much as i know response.data.pipe is not a promise so await is not going to work on it.
Try :-
response.data.pipe(fs.createWriteStream(filePath), {end: false})
response.data.on('end', ()=> res.download(filePath))
Related
My API end point returns an excel file streamed from S3. It works on the local but when testing on API gateway the file is corrupted. Here is my API code:
const downloadContentFromS3 = async function(bucket, file) {
return new Promise((resolve, reject) => {
streamFileFromS3(bucket, file, (error, s3buffer) => {
if (error) return reject(error);
return resolve(s3buffer);
});
});
};
const streamFileFromS3 = async function(bucket, fileName, callback) {
const params = {
Bucket: bucket,
Key: fileName,
};
const buffers = [];
const stream = s3.getObject(params).createReadStream();
stream.on('data', data => buffers.push(data));
stream.on('end', () => callback(null, Buffer.concat(buffers)));
stream.on('error', error => callback(error));
};
downloadExcelFile: async (req, res) => {
try {
const fileName = 'myFilename';
const workbook = await downloadContentFromS3(
'bucket-name'
fileName
);
const workbook = xlsx.read(buffer);
res.setHeader('Content-disposition', `attachment; filename=${fileName}`);
res.setHeader(
'Content-type',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
);
const wbout = xlsx.write(workbook, { bookType: 'xlsx', type: 'buffer' });
res.status(200).send(Buffer.from(wbout));
} catch (error) {
throw new OriolaError(error.message);
}
},
What I have tried so far: Setup the binary media types as shown in the picture:
In addition tried to set the RESPONSE HEADER to Content-Type and Content-Disposition but to no avail. The problem seems to persist. Any ideas and help is appreciated.
EDIT: I have also tried to set binary type */* in the settings but this does not help as well.
API Gateway and Lambda send files between themselves as base64. This is regardless of whether you set a binary media type on API Gateway, as API Gateway does the conversion between base64 and binary.
S3 getObject gets a binary file, but that is a moot point for your problem, as you are still creating a binary file with xlsx.
What you are currently doing is sending binary data untransformed as base64 data.
All you need to do is return the file as a base64 buffer instead of a binary one.
So
res.status(200).send(Buffer.from(wbout));
becomes
res.status(200).send(Buffer.from(wbout).toString('base64'));
https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-payload-encodings.html
So either
setup passthrough behaviour and send binary response body from your function as you are already doing,
or set the contentHandling property of the IntegrationResponse resource to CONVERT_TO_BINARY in addition to binaryMediaTypes setting already done and then send base64 response body.
I want to get binary from image to rotate then, using sharp.rotate();
I try to do this content += chunk; but dosent work.
let Client = require('ftp');
let fs = require('fs');
let sharp = require('sharp');
let path = 'users/'+userId+'/headerImage/header';
let Ftp = new Client();//create new istance of Ftp
//Start. Here we get image from server
await Ftp.on('ready', function(){
Ftp.get(path, async function(err, stream){
if(err){
res.status(400).send(err);
};
var content = '';
await stream.on('data', async (chunk) => {
content += chunk;
});
await stream.on('end', async function(){
console.log(content);
let image = await sharp(content);
await image
.rotate(90)
.toBuffer()
.then(async data => {
console.log(data);
})
.catch(error => {
console.log(error);
});
Ftp.end();
});
});
});
await Ftp.connect({
host: fileTransferProtocol.host,
port: fileTransferProtocol.port,
user: fileTransferProtocol.user,
password: fileTransferProtocol.pass
});
console: Error: [Error: Input file is missing]
I believe the problem you are having is that you are not handling the incoming data as a buffer. The stream variable inside the Ftp.get callback is of type ReadableStream. By default, stream data will be returned as Buffer objects unless you specify an encoding for the data, using the readable.setEncoding() method.
For your specific purpose, you want to handle the data as a Buffer object, since that is what the sharp function is expecting. To store the incoming data into a Buffer modify what happens on the data event.
var content = new Buffer(0);
stream.on("data", async chunk => {
content = Buffer.concat([content, chunk]);
});
Also, I don't think you are using async/await duly. The ftp module runs with callbacks and events, not promises. Appending those functions with await won't make them run synchronously.
Please check the following link to find more information about this feature:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
If you want to us async/await to handle your ftp requests try this module:
https://www.npmjs.com/package/promise-ftp
It provides an asynchronous interface for communicating with an FTP server.
I am trying to use busboy inside a lambda function to process a post request which is supposed to upload an image file. I notice that the whole file content is not making it to be parsed by busboy.
I tried changing the call to busboy.write to just use 'base64' since it looks like the file arrives in binary, but that didn't work either.
my client code
const formData = new FormData();
formData.append("file", params.file, params.file.name);
const request = new XMLHttpRequest();
request.open("POST", "https://myapi/uploadphoto");
request.setRequestHeader('Authorization', this.props.idToken);
request.send(formData);
my lambda code
function getFile(event) {
const busboy = new Busboy({headers: event.headers});
const result = {};
return new Promise((resolve, reject) => {
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
file.on('data', data => {
result.content = data;
console.log("got data... " + data.length + ' bytes');
});
file.on('end', () => {
result.filename = filename;
result.contentType = mimetype;
resolve(result);
});
});
busboy.on('error', error => reject(error));
busboy.write(event.body, event.isBase64Encoded ? 'base64' : 'binary');
busboy.end();
});
}
When trying with an example photo, I notice that the "got data" console log is showing me that I am not receiving the whole file. The file I am using is 229707 bytes but the console log says that it received 217351 bytes.
I am wondering if I am using busboy wrong or if this is some quirk of lambda + api gateway. Any ideas or help troubleshooting is much appreciated.
I was struggling with this issue too, but in the end it was a problem with API Gateway.
I was able to solve the problem by adding multipart/form-data as a binary media type inside the settings of API Gateway.
To do it go to API Gateway > "Your API" > Settings > Add Binary Media Type and add multipart/form-data.
After that, deploy again your API and it should work.
I hope this helps anyone!
I'm trying to POST a raw body with restify. I have the receive side correct, when using POSTman I can send a raw zip file, and the file is correctly created on the server's file system. However, I'm struggling to write my test in mocha. Here is the code I have, any help would be greatly appreciated.
I've tried this approach.
const should = require('should');
const restify = require('restify');
const fs = require('fs');
const port = 8080;
const url = 'http://localhost:' + port;
const client = restify.createJsonClient({
url: url,
version: '~1.0'
});
const testPath = 'test/assets/test.zip';
fs.existsSync(testPath).should.equal(true);
const readStream = fs.createReadStream(testPath);
client.post('/v1/deploy', readStream, function(err, req, res, data) {
if (err) {
throw new Error(err);
}
should(res).not.null();
should(res.statusCode).not.null();
should(res.statusCode).not.undefined();
res.statusCode.should.equal(200);
should(data).not.null();
should(data.endpoint).not.undefined();
data.endpoint.should.equal('http://endpointyouhit:8080');
done();
});
Yet the file size on the file system is always 0. I'm not using my readStream correctly, but I'm not sure how to correct it. Any help would be greatly appreciated.
Note that I want to stream the file, not load it in memory on transmit and receive, the file can potentially be too large for an in memory operation.
Thanks,
Todd
One thing is that you would need to specify a content-type of multi-part/form-data. However, it looks like restify doesn't support that content type, so you're probably out of luck using the restify client to post a file.
To answer my own question, it doesn't appear to be possible to do this with the restify client. I also tried the request module, which claims to have this capability. However, when using their streaming examples, I always had a file size of 0 on the server. Below is a functional mocha integration test.
const testPath = 'test/assets/test.zip';
fs.existsSync(testPath).should.equal(true);
const readStream = fs.createReadStream(testPath);
var options = {
host: 'localhost'
, port: port
, path: '/v1/deploy/testvalue'
, method: 'PUT'
};
var req = http.request(options, function (res) {
//this feels a bit backwards, but these are evaluated AFTER the read stream has closed
var buffer = '';
//pipe body to a buffer
res.on('data', function(data){
buffer+= data;
});
res.on('end', function () {
should(res).not.null();
should(res.statusCode).not.null();
should(res.statusCode).not.undefined();
res.statusCode.should.equal(200);
const json = JSON.parse(buffer);
should(json).not.null();
should(json.endpoint).not.undefined();
json.endpoint.should.equal('http://endpointyouhit:8080');
done();
});
});
req.on('error', function (err) {
if (err) {
throw new Error(err);
}
});
//pipe the readstream into the request
readStream.pipe(req);
/**
* Close the request on the close of the read stream
*/
readStream.on('close', function () {
req.end();
console.log('I finished.');
});
//note that if we end up with larger files, we may want to support the continue, much as S3 does
//https://nodejs.org/api/http.html#http_event_continue
I'm using SendGrid for receiving files via email. SendGrid parses the incoming emails and sends the files in a multipart form to an endpoint I have set up.
I don't want the files on my local disk so I stream them straight to Amazon S3. This works perfect.
But before I can stream to S3 I need to get hold of the destination mail address so I can work out the correct s3 folder. This is sent in a field named "to" in the form post. Unfortunately this field sometimes arrives after the files are arriving, hence I need a way to await the to-field before I'm ready to take the stream.
I thought I could wrap the onField in a promise and await the to-field from within the onFile. But this concept seems to lock it self up when the field arrives after the file.
I'm new to booth streams and promises. I would really appreciate if someone could tell me how to do this.
This is the non working pseudoish code:
function sendGridUpload(req, res, next) {
var busboy = new Busboy({ headers: req.headers });
var awaitEmailAddress = new Promise(function(resolve, reject) {
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
if(fieldname === 'to') {
resolve(val);
} else {
return;
}
});
});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
function findInbox(emailAddress) {
console.log('Got email address: ' + emailAddress);
..find the inbox and generate an s3Key
return s3Key;
}
function saveFileStream(s3Key) {
..pipe the file directly to S3
}
awaitEmailAddress.then(findInbox)
.then(saveFileStream)
.catch(function(err) {
log.error(err)
});
});
req.pipe(busboy);
}
I finally got this working. The solution is not very pretty, and I have actually switched to another concept (described at the end of the post).
To buffer the incoming data until the "to"-field arrives I used stream-buffers by #samcday. When I get hold of the to-field I release the readable stream to the pipes lined up for the data.
Here is the code (some parts omitted, but essential parts are there).
var streamBuffers = require('stream-buffers');
function postInboundMail(req, res, next) {
var busboy = new Busboy({ headers: req.headers});
//Sometimes the fields arrives after the files are streamed.
//We need the "to"-field before we are ready for the files
//Therefore the onField is wrapped in a promise which gets
//resolved when the to field arrives
var awaitEmailAddress = new Promise(function(resolve, reject) {
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
var emailAddress;
if(fieldname === 'to') {
try {
emailAddress = emailRegexp.exec(val)[1]
resolve(emailAddress)
} catch(err) {
return reject(err);
}
} else {
return;
}
});
});
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
var inbox;
//I'm using readableStreamBuffer to accumulate the data before
//I get the email field so I can send the stream through to S3
var readBuf = new streamBuffers.ReadableStreamBuffer();
//I have to pause readBuf immediately. Otherwise stream-buffers starts
//sending as soon as I put data in in with put().
readBuf.pause();
function getInbox(emailAddress) {
return model.inbox.findOne({email: emailAddress})
.then(function(result) {
if(!result) return Promise.reject(new Error(`Inbox not found for ${emailAddress}`))
inbox = result;
return Promise.resolve();
});
}
function saveFileStream() {
console.log('=========== starting stream to S3 ========= ' + filename)
//Have to resume readBuf since we paused it before
readBuf.resume();
//file.save will approximately do the following:
// readBuf.pipe(gzip).pipe(encrypt).pipe(S3)
return model.file.save({
inbox: inbox,
fileStream: readBuf
});
}
awaitEmailAddress.then(getInbox)
.then(saveFileStream)
.catch(function(err) {
log.error(err)
});
file.on('data', function(data) {
//Fill readBuf with data as it arrives
readBuf.put(data);
});
file.on('end', function() {
//This was the only way I found to get the S3 streaming finished.
//Destroysoon will let the pipes finish the reading bot no more writes are allowed
readBuf.destroySoon()
});
});
busboy.on('finish', function() {
res.writeHead(202, { Connection: 'close', Location: '/' });
res.end();
});
req.pipe(busboy);
}
I would really much like feedback on this solution, even though I'm not using it. I have a feeling that this can be done much more simple and elegant.
New solution:
Instead of waiting for the to-field I send the stream directly to S3. I figured, the more stuff I put in between the incoming stream and the S3 saving, the higher the risk of loosing the incoming file due to a bug in my code. (SendGrid will eventually resend the file if I'm not responding with 200, but it will take some time.)
This is how I do it:
Save a placeholder for the file in the database
Pipe the stream to S3
Update the placeholder with more information as it arrives
This solution also gives me the opportunity to easily get hold of unsuccessful uploads since the placeholders for unsuccessful uploads will be incomplete.
//Michael