AWS S3 Lambda WriteFile from stream - node.js

Using an AWS Lambda, I just want to get an archive (lo.tar.gz) from my s3, copy it to the tmp folder and extract it to the tmp folder.
I have this code in local who work perfectly :
var AWS = require('aws-sdk');
var fs = require('fs');
var targz = require('targz');
var s3 = new AWS.S3({ apiVersion: '2006-03-01' });
var fileStream = fs.createWriteStream('./tmp/lo.tar.gz');
var s3Stream = s3.getObject({ Bucket: 'mybucketname', Key: 'lo.tar.gz' }).createReadStream();
//console.log("Here the last console.log who appear");
// Listen for errors returned by the service
s3Stream.on('error', function (err) {
// NoSuchKey: The specified key does not exist
console.error(err);
});
s3Stream.pipe(fileStream).on('error', function (err) {
// capture any errors that occur when writing data to the file
console.error('File Stream:', err);
}).on('close', function () {
console.log('Stream Done.');
// decompress files from tar.gz archive
targz.decompress({
src: './tmp/lo.tar.gz',
dest: './tmp'
}, function (err) {
if (err) {
console.log(err);
} else {
console.log("Extract Done!");
}
});
});
But when I put it in a lambda, nothing append.
It creates an empty file "lo.tar.gz" in tmp and it closes saying "statut:succeeded" and without any error message ... I put "console.log" between each line and the last to appear is the one I left ("Here the last console.log who appear") ...
Does anyone have any idea why the code is not being executed to the end?
Thank you :D!
EDIT : here the exact code for the lambda (the same with exports.handler) :
var AWS = require('aws-sdk');
var fs = require('fs');
var targz = require('targz');
exports.handler = async (event, context, callback) => {
var s3 = new AWS.S3({ apiVersion: '2006-03-01' });
var fileStream = fs.createWriteStream('./tmp/lo.tar.gz');
var s3Stream = s3.getObject({ Bucket: 'mybucketname', Key: 'lo.tar.gz'
}).createReadStream();
//console.log("Here the last console.log who appear");
// Listen for errors returned by the service
s3Stream.on('error', function (err) {
// NoSuchKey: The specified key does not exist
console.error(err);
});
s3Stream.pipe(fileStream).on('error', function (err) {
// capture any errors that occur when writing data to the file
console.error('File Stream:', err);
}).on('close', function () {
console.log('Stream Done.');
// decompress files from tar.gz archive
targz.decompress({
src: './tmp/lo.tar.gz',
dest: './tmp'
}, function (err) {
if (err) {
console.log(err);
} else {
console.log("Extract Done!");
}
});
});
}

Related

SFTP createReadStream not writing into s3 bucket

I'm trying to read a file from an sftp server and stream that file into an s3 bucket. I'm not able to stream the file into the s3 bucket. Yes the file path is exactly correct. I'm not sure what I am doing wrong. When I run the code, it doesn't even try to upload the stream into the bucket because I don't get any upload console logs.
const aws = require('aws-sdk');
const s3 = new aws.S3();
const Client = require('ssh2').Client;
const conn = new Client();
const connSettings = {
host: event.serverHost,
port: event.port,
username: event.username,
password: event.password
};
exports.handler = function(event) {
conn.on('ready', function() {
conn.sftp(function(err, sftp) {
if (err) {
console.log("Errror in connection", err);
conn.end()
} else {
console.log("Connection established");
let readStream = sftp.createReadStream(remoteFilePath);
console.log(`Read Stream ${readStream}`)
// readStream outputs [object Object] to the console
const uploadParams = {
Bucket: s3Bucket,
Key: 'fileName',
Body: readStream
}
s3.upload (uploadParams, function (err, data) {
if (err) {
console.log("Error", err);
} if (data) {
console.log("Upload Success", data.Location);
}
});
conn.end()
}
});
}).connect(connSettings);
}
I want to be able to stream the readStream from sftp server into s3 bucket.
conn.end() ends the connection immediately. Move that to inside your s3.upload() callback so that your data actually gets transferred before the connection is closed.
This is a working Node 12 example of what I believe you are trying to accomplish:
const aws = require('aws-sdk');
const s3 = new aws.S3();
const Client = require('ssh2').Client;
const conn = new Client();
const { PassThrough } = require('stream');
conn.on('ready', () => {
conn.sftp((err, sftp) => {
const transferStream = new PassThrough();
s3.upload({
Bucket: s3Bucket,
Key: 'test_file.txt',
Body: transferStream
}, (err, data) => {
if (err) {
console.log(`Upload error: ${err}`);
}
if (data) {
console.log(`Uploaded to [${data.Location}].`);
}
});
sftp.createReadStream(remoteFilePath)
.pipe(transferStream)
.on('end', () => {
transferStream.end();
conn.end();
});
});
}).connect(connectionSettings);

How does child_process behave in Nodejs

I have a working Nodejs code, however, the child_process library behaves strange, I am just wondering how this library works.
My code is trying to download the SSL certificates from S3, then create the two new files based on the exiting ones using child_process library.
const AWS = require('aws-sdk');
const fs = require('fs');
const child_process = require("child_process");
const exec = require('child_process').exec;
var s3 = new AWS.S3();
var filePath = '../Desktop/Certs/'
var bucketName = 'neb.certificates' //replace example bucket with your s3 bucket name
var params = {
Bucket: bucketName,
Prefix: 'dev/jenkins.secure.care/',
};
s3.listObjectsV2(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else {
// console.log(data.Contents)
var len = data.Contents.length
for(var i=0; i<len; i++){
var key = data.Contents[i]["Key"]
var newPath = filePath.concat(key.substring(31))
const downloadFile = (newPath, bucketName, key) => {
//construct getParam
var getParams = {
Bucket: bucketName,
Key: key
};
s3.getObject(getParams, (err, data) => {
if (err) console.error(err)
fs.writeFileSync(newPath, data.Body.toString())
// console.log(`${newPath} has been created!`)
})
}
downloadFile(newPath, bucketName, key)
}
}
});
exec('mv ../Desktop/Certs/cert.pem ../Desktop/Certs/jenkins.crt', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
exec('mv ../Desktop/Certs/privkey.pem ../Desktop/Certs/jenkins.key', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
So when I run the code the first time, it only downloads the certificates from S3 to the local folder, it did not create the other 2 files. Then I have to run it the second time to create additional files.
However, I just want to run it once, and it has everything what I expect.
I have added a code which to sleep 5 seconds, then create the additional files, But it did not solve my problem, which means I still run the code twice to get everything.
child_process.execSync("sleep 5");
Please helps
It looks like the problem here is the async execution of your code. In this example here's what's happening:
Run s3.listObjectsV2() and when finished, fire callback (but not now, in the future)
Run first exec() and when finished, fire callback (but not now, in the future)
Run second exec() and when finished, fire callback (but not now, in the future)
And those three steps are fired immediately, one by one. And each of them has its own callback which fires in the future. Ok, but when is the future? - exactly! You don't know. In your case probably those two callbacks in exec()'s are fired before the callback from s3 and this is why it does not work.
The solution here is to make sure that those exec()s are fired after s3.listObjects. So you have two options: first is to make a promise out of the s3, like this: s3.listObjectsV2(params).promise() and await for it, then in .then((data) => {}) you have your data and in .catch((error) => {}) you have your error. Or you can simply put those exec()s in the callback of the s3 call.
Your code should look like this according to the solution 2 (from the comments):
const AWS = require('aws-sdk');
const fs = require('fs');
const child_process = require("child_process");
const exec = require('child_process').exec;
var s3 = new AWS.S3();
var filePath = '../Desktop/Certs/'
var bucketName = 'neb.certificates' //replace example bucket with your s3 bucket name
var params = {
Bucket: bucketName,
Prefix: 'dev/jenkins.secure.care/',
};
s3.listObjectsV2(params, async function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else {
// console.log(data.Contents)
var len = data.Contents.length
for(var i=0; i<len; i++){
var key = data.Contents[i]["Key"]
var newPath = filePath.concat(key.substring(31))
const downloadFile = (newPath, bucketName, key) => {
//construct getParam
var getParams = {
Bucket: bucketName,
Key: key
};
return s3.getObject(getParams).promise();
};
const downloadData = await downloadFile(newPath, bucketName, key).catch(console.error);
fs.writeFileSync(newPath, downloadData.Body.toString());
console.log(newPath, 'created');
}
//rename files
console.log('renaming first cert.pem');
exec('mv ../Desktop/Certs/cert.pem ../Desktop/Certs/jenkins.crt', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
console.log('renaming second privkey.pem');
exec('mv ../Desktop/Certs/privkey.pem ../Desktop/Certs/jenkins.key', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
}
});

Trying to send readable stream (audio) to amazon s3 - runtime error

I am trying to write a function that takes the mp3 url of the recording and then uploads that to S3. However, I keep getting a runtime error and the callback is never reached. If I move the callback below s3.upload(...) then the statement "attempting to upload mp3 is never logged.
exports.handler = function(context, event, callback) {
const twiml = new Twilio.twiml.VoiceResponse();
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
var getUri = require('get-uri');
AWS.config.update({
accessKeyId: "...",
secretAccessKey: "..."
});
var client = context.getTwilioClient();
const recording_id = event.RecordingSid;
const uri = event.RecordingUrl + ".mp3";
getUri(uri, function (err, rs) {
if (err) {
console.log(err.message);
throw err;
}
var params = {
ACL: "public-read",
Body: rs,
Bucket: "...",
Key: "audio.mp3",
ContentType: 'audio/mp3'
};
s3.upload(params, function(err,data) {
console.log("attempting to upload mp3");
if (err) {
console.log("there is an error");
console.log(err.status);
throw err.message;
}
else {
console.log("Your upload has been successful.");
}
callback(null, twiml);
});
});
console.log("at the end");
};
Is there any other way to access the recording and put them in my public s3 bucket? Why is this never executing s3.upload(...).
Any insights into this is helpful! Thanks in advance!
app.get('/uploadsong',function(req,res){
console.log("Hi there")
var URI = 'http://sensongsmp3download.info/Kaala%20(2018)%20-%20Sensongsmp3.info/Thanga%20Sela%20--%20Sensongsmp3.Info.mp3';
var buffer = [];
request
.get(URI)
.on('error', function(err) {
console.log("error")
}).on('data',function(data){
buffer.push(data);
}).on('end',function(){
var completeSong = Buffer.concat(buffer);
var data = {
Body:completeSong,
Key: 'sample.mp3',
ContentType: 'audio/mp3'
}
s3Bucket.putObject(data, function(err, data){
if (err)
{
console.log('Error uploading data: ', data);
} else
{
console.log('upload successfull')
res.send('done');
}
})
})
})
here are the modules i have used
var request = require('request');
I contacted Twilio regarding this they responded that Twilio Functions have a strict 5 second time-out and the upload from the Twilio Function to S3 Bucket takes more than 5 seconds. My workaround was sending a string with all mp3 URLs separated by comma and a space. The lambda function would then parse through all the links and store all links in an array which would be used for audio playback.

azure fileservice createFileFromText with nodejs. is only uploading 15 B I have 16mb file

I am trying to send file from s3 to azure using lambda. but it is timing out and uploading only 15B.
'use strict';
var AWS = require('aws-sdk');
var azure = require('azure-storage');
var s3 = new AWS.S3();
var xml2js = require('xml2js');
var parser = new xml2js.Parser({explicitArray : false});
var extractedData = "";
exports.handler = function(event, context) {
var bucketName = event.Records[0].s3.bucket.name;
var keyName = event.Records[0].s3.object.key;
readFile(bucketName, keyName, readFileContent, onError);
};
function readFile (bucketName, filename, onFileContent, onError) {
var params = { Bucket: bucketName, Key: filename };
s3.getObject(params, function (err, data) {
if (!err)
onFileContent(filename, data.Body);
else
console.log(err);
});
}
function readFileContent(filename, content) {
parser.parseString(content, function(err,result){
console.log(result);
var fileService = azure.createFileService('fgdgdfg','dfgdfgdfgdfgdfg');
fileService.createFileFromText('xfiles', '', 'rros.json', result, function(error, result, response) {
if (!error) {
console.log("file created....")
}else{
console.log(error)
}
});
});
}
function onError (err) {
console.log('error: ' + err);
}
ERROR:
TypeError: First argument must be a string or Buffer
at ClientRequest.OutgoingMessage.write (_http_outgoing.js:458:11)
at Request.write (/var/task/node_modules/request/request.js:1501:27)
The fourth parameter of the function createFileFromText(...) doesn't accept a JavaScript object. You should use JSON.stringify() to covert it to a string like this:
fileService.createFileFromText('xfiles', '', 'rros.json', JSON.stringify(result), function(error, result, response) {
if (!error) {
console.log("file created....")
}else{
console.log(error)
}
});

Read file from aws s3 bucket using node fs

I am attempting to read a file that is in a aws s3 bucket using
fs.readFile(file, function (err, contents) {
var myLines = contents.Body.toString().split('\n')
})
I've been able to download and upload a file using the node aws-sdk, but I am at a loss as to how to simply read it and parse the contents.
Here is an example of how I am reading the file from s3:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myKey.csv'}
var s3file = s3.getObject(params)
You have a couple options. You can include a callback as a second argument, which will be invoked with any error message and the object. This example is straight from the AWS documentation:
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
Alternatively, you can convert the output to a stream. There's also an example in the AWS documentation:
var s3 = new AWS.S3({apiVersion: '2006-03-01'});
var params = {Bucket: 'myBucket', Key: 'myImageFile.jpg'};
var file = require('fs').createWriteStream('/path/to/file.jpg');
s3.getObject(params).createReadStream().pipe(file);
This will do it:
new AWS.S3().getObject({ Bucket: this.awsBucketName, Key: keyName }, function(err, data)
{
if (!err)
console.log(data.Body.toString());
});
Since you seem to want to process an S3 text file line-by-line. Here is a Node version that uses the standard readline module and AWS' createReadStream()
const readline = require('readline');
const rl = readline.createInterface({
input: s3.getObject(params).createReadStream()
});
rl.on('line', function(line) {
console.log(line);
})
.on('close', function() {
});
If you are looking to avoid the callbacks you can take advantage of the sdk .promise() function like this:
const s3 = new AWS.S3();
const params = {Bucket: 'myBucket', Key: 'myKey.csv'}
const response = await s3.getObject(params).promise() // await the promise
const fileContent = response.Body.toString('utf-8'); // can also do 'base64' here if desired
I'm sure the other ways mentioned here have their advantages but this works great for me. Sourced from this thread (see the last response from AWS): https://forums.aws.amazon.com/thread.jspa?threadID=116788
here is the example which i used to retrive and parse json data from s3.
var params = {Bucket: BUCKET_NAME, Key: KEY_NAME};
new AWS.S3().getObject(params, function(err, json_data)
{
if (!err) {
var json = JSON.parse(new Buffer(json_data.Body).toString("utf8"));
// PROCESS JSON DATA
......
}
});
I couldn't figure why yet, but the createReadStream/pipe approach didn't work for me. I was trying to download a large CSV file (300MB+) and I got duplicated lines. It seemed a random issue. The final file size varied in each attempt to download it.
I ended up using another way, based on AWS JS SDK examples:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myImageFile.jpg'};
var file = require('fs').createWriteStream('/path/to/file.jpg');
s3.getObject(params).
on('httpData', function(chunk) { file.write(chunk); }).
on('httpDone', function() { file.end(); }).
send();
This way, it worked like a charm.
I prefer Buffer.from(data.Body).toString('utf8'). It supports encoding parameters. With other AWS services (ex. Kinesis Streams) someone may want to replace 'utf8' encoding with 'base64'.
new AWS.S3().getObject(
{ Bucket: this.awsBucketName, Key: keyName },
function(err, data) {
if (!err) {
const body = Buffer.from(data.Body).toString('utf8');
console.log(body);
}
}
);
I had exactly the same issue when downloading from S3 very large files.
The example solution from AWS docs just does not work:
var file = fs.createWriteStream(options.filePath);
file.on('close', function(){
if(self.logger) self.logger.info("S3Dataset file download saved to %s", options.filePath );
return callback(null,done);
});
s3.getObject({ Key: documentKey }).createReadStream().on('error', function(err) {
if(self.logger) self.logger.error("S3Dataset download error key:%s error:%#", options.fileName, error);
return callback(error);
}).pipe(file);
While this solution will work:
var file = fs.createWriteStream(options.filePath);
s3.getObject({ Bucket: this._options.s3.Bucket, Key: documentKey })
.on('error', function(err) {
if(self.logger) self.logger.error("S3Dataset download error key:%s error:%#", options.fileName, error);
return callback(error);
})
.on('httpData', function(chunk) { file.write(chunk); })
.on('httpDone', function() {
file.end();
if(self.logger) self.logger.info("S3Dataset file download saved to %s", options.filePath );
return callback(null,done);
})
.send();
The createReadStream attempt just does not fire the end, close or error callback for some reason. See here about this.
I'm using that solution also for writing down archives to gzip, since the first one (AWS example) does not work in this case either:
var gunzip = zlib.createGunzip();
var file = fs.createWriteStream( options.filePath );
s3.getObject({ Bucket: this._options.s3.Bucket, Key: documentKey })
.on('error', function (error) {
if(self.logger) self.logger.error("%#",error);
return callback(error);
})
.on('httpData', function (chunk) {
file.write(chunk);
})
.on('httpDone', function () {
file.end();
if(self.logger) self.logger.info("downloadArchive downloaded %s", options.filePath);
fs.createReadStream( options.filePath )
.on('error', (error) => {
return callback(error);
})
.on('end', () => {
if(self.logger) self.logger.info("downloadArchive unarchived %s", options.fileDest);
return callback(null, options.fileDest);
})
.pipe(gunzip)
.pipe(fs.createWriteStream(options.fileDest))
})
.send();
With the new version of sdk, the accepted answer does not work - it does not wait for the object to be downloaded. The following code snippet will help with the new version:
// dependencies
const AWS = require('aws-sdk');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
var bucket = "TestBucket"
var key = "TestKey"
try {
const params = {
Bucket: Bucket,
Key: Key
};
var theObject = await s3.getObject(params).promise();
} catch (error) {
console.log(error);
return;
}
}
If you want to save memory and want to obtain each row as a json object, then you can use fast-csv to create readstream and can read each row as a json object as follows:
const csv = require('fast-csv');
const AWS = require('aws-sdk');
const credentials = new AWS.Credentials("ACCESSKEY", "SECRETEKEY", "SESSIONTOKEN");
AWS.config.update({
credentials: credentials, // credentials required for local execution
region: 'your_region'
});
const dynamoS3Bucket = new AWS.S3();
const stream = dynamoS3Bucket.getObject({ Bucket: 'your_bucket', Key: 'example.csv' }).createReadStream();
var parser = csv.fromStream(stream, { headers: true }).on("data", function (data) {
parser.pause(); //can pause reading using this at a particular row
parser.resume(); // to continue reading
console.log(data);
}).on("end", function () {
console.log('process finished');
});
var fileStream = fs.createWriteStream('/path/to/file.jpg');
var s3Stream = s3.getObject({Bucket: 'myBucket', Key: 'myImageFile.jpg'}).createReadStream();
// Listen for errors returned by the service
s3Stream.on('error', function(err) {
// NoSuchKey: The specified key does not exist
console.error(err);
});
s3Stream.pipe(fileStream).on('error', function(err) {
// capture any errors that occur when writing data to the file
console.error('File Stream:', err);
}).on('close', function() {
console.log('Done.');
});
Reference: https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/requests-using-stream-objects.html

Resources