I am attempting to read a file that is in a aws s3 bucket using
fs.readFile(file, function (err, contents) {
var myLines = contents.Body.toString().split('\n')
})
I've been able to download and upload a file using the node aws-sdk, but I am at a loss as to how to simply read it and parse the contents.
Here is an example of how I am reading the file from s3:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myKey.csv'}
var s3file = s3.getObject(params)
You have a couple options. You can include a callback as a second argument, which will be invoked with any error message and the object. This example is straight from the AWS documentation:
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
Alternatively, you can convert the output to a stream. There's also an example in the AWS documentation:
var s3 = new AWS.S3({apiVersion: '2006-03-01'});
var params = {Bucket: 'myBucket', Key: 'myImageFile.jpg'};
var file = require('fs').createWriteStream('/path/to/file.jpg');
s3.getObject(params).createReadStream().pipe(file);
This will do it:
new AWS.S3().getObject({ Bucket: this.awsBucketName, Key: keyName }, function(err, data)
{
if (!err)
console.log(data.Body.toString());
});
Since you seem to want to process an S3 text file line-by-line. Here is a Node version that uses the standard readline module and AWS' createReadStream()
const readline = require('readline');
const rl = readline.createInterface({
input: s3.getObject(params).createReadStream()
});
rl.on('line', function(line) {
console.log(line);
})
.on('close', function() {
});
If you are looking to avoid the callbacks you can take advantage of the sdk .promise() function like this:
const s3 = new AWS.S3();
const params = {Bucket: 'myBucket', Key: 'myKey.csv'}
const response = await s3.getObject(params).promise() // await the promise
const fileContent = response.Body.toString('utf-8'); // can also do 'base64' here if desired
I'm sure the other ways mentioned here have their advantages but this works great for me. Sourced from this thread (see the last response from AWS): https://forums.aws.amazon.com/thread.jspa?threadID=116788
here is the example which i used to retrive and parse json data from s3.
var params = {Bucket: BUCKET_NAME, Key: KEY_NAME};
new AWS.S3().getObject(params, function(err, json_data)
{
if (!err) {
var json = JSON.parse(new Buffer(json_data.Body).toString("utf8"));
// PROCESS JSON DATA
......
}
});
I couldn't figure why yet, but the createReadStream/pipe approach didn't work for me. I was trying to download a large CSV file (300MB+) and I got duplicated lines. It seemed a random issue. The final file size varied in each attempt to download it.
I ended up using another way, based on AWS JS SDK examples:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'myImageFile.jpg'};
var file = require('fs').createWriteStream('/path/to/file.jpg');
s3.getObject(params).
on('httpData', function(chunk) { file.write(chunk); }).
on('httpDone', function() { file.end(); }).
send();
This way, it worked like a charm.
I prefer Buffer.from(data.Body).toString('utf8'). It supports encoding parameters. With other AWS services (ex. Kinesis Streams) someone may want to replace 'utf8' encoding with 'base64'.
new AWS.S3().getObject(
{ Bucket: this.awsBucketName, Key: keyName },
function(err, data) {
if (!err) {
const body = Buffer.from(data.Body).toString('utf8');
console.log(body);
}
}
);
I had exactly the same issue when downloading from S3 very large files.
The example solution from AWS docs just does not work:
var file = fs.createWriteStream(options.filePath);
file.on('close', function(){
if(self.logger) self.logger.info("S3Dataset file download saved to %s", options.filePath );
return callback(null,done);
});
s3.getObject({ Key: documentKey }).createReadStream().on('error', function(err) {
if(self.logger) self.logger.error("S3Dataset download error key:%s error:%#", options.fileName, error);
return callback(error);
}).pipe(file);
While this solution will work:
var file = fs.createWriteStream(options.filePath);
s3.getObject({ Bucket: this._options.s3.Bucket, Key: documentKey })
.on('error', function(err) {
if(self.logger) self.logger.error("S3Dataset download error key:%s error:%#", options.fileName, error);
return callback(error);
})
.on('httpData', function(chunk) { file.write(chunk); })
.on('httpDone', function() {
file.end();
if(self.logger) self.logger.info("S3Dataset file download saved to %s", options.filePath );
return callback(null,done);
})
.send();
The createReadStream attempt just does not fire the end, close or error callback for some reason. See here about this.
I'm using that solution also for writing down archives to gzip, since the first one (AWS example) does not work in this case either:
var gunzip = zlib.createGunzip();
var file = fs.createWriteStream( options.filePath );
s3.getObject({ Bucket: this._options.s3.Bucket, Key: documentKey })
.on('error', function (error) {
if(self.logger) self.logger.error("%#",error);
return callback(error);
})
.on('httpData', function (chunk) {
file.write(chunk);
})
.on('httpDone', function () {
file.end();
if(self.logger) self.logger.info("downloadArchive downloaded %s", options.filePath);
fs.createReadStream( options.filePath )
.on('error', (error) => {
return callback(error);
})
.on('end', () => {
if(self.logger) self.logger.info("downloadArchive unarchived %s", options.fileDest);
return callback(null, options.fileDest);
})
.pipe(gunzip)
.pipe(fs.createWriteStream(options.fileDest))
})
.send();
With the new version of sdk, the accepted answer does not work - it does not wait for the object to be downloaded. The following code snippet will help with the new version:
// dependencies
const AWS = require('aws-sdk');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
var bucket = "TestBucket"
var key = "TestKey"
try {
const params = {
Bucket: Bucket,
Key: Key
};
var theObject = await s3.getObject(params).promise();
} catch (error) {
console.log(error);
return;
}
}
If you want to save memory and want to obtain each row as a json object, then you can use fast-csv to create readstream and can read each row as a json object as follows:
const csv = require('fast-csv');
const AWS = require('aws-sdk');
const credentials = new AWS.Credentials("ACCESSKEY", "SECRETEKEY", "SESSIONTOKEN");
AWS.config.update({
credentials: credentials, // credentials required for local execution
region: 'your_region'
});
const dynamoS3Bucket = new AWS.S3();
const stream = dynamoS3Bucket.getObject({ Bucket: 'your_bucket', Key: 'example.csv' }).createReadStream();
var parser = csv.fromStream(stream, { headers: true }).on("data", function (data) {
parser.pause(); //can pause reading using this at a particular row
parser.resume(); // to continue reading
console.log(data);
}).on("end", function () {
console.log('process finished');
});
var fileStream = fs.createWriteStream('/path/to/file.jpg');
var s3Stream = s3.getObject({Bucket: 'myBucket', Key: 'myImageFile.jpg'}).createReadStream();
// Listen for errors returned by the service
s3Stream.on('error', function(err) {
// NoSuchKey: The specified key does not exist
console.error(err);
});
s3Stream.pipe(fileStream).on('error', function(err) {
// capture any errors that occur when writing data to the file
console.error('File Stream:', err);
}).on('close', function() {
console.log('Done.');
});
Reference: https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/requests-using-stream-objects.html
Related
Using an AWS Lambda, I just want to get an archive (lo.tar.gz) from my s3, copy it to the tmp folder and extract it to the tmp folder.
I have this code in local who work perfectly :
var AWS = require('aws-sdk');
var fs = require('fs');
var targz = require('targz');
var s3 = new AWS.S3({ apiVersion: '2006-03-01' });
var fileStream = fs.createWriteStream('./tmp/lo.tar.gz');
var s3Stream = s3.getObject({ Bucket: 'mybucketname', Key: 'lo.tar.gz' }).createReadStream();
//console.log("Here the last console.log who appear");
// Listen for errors returned by the service
s3Stream.on('error', function (err) {
// NoSuchKey: The specified key does not exist
console.error(err);
});
s3Stream.pipe(fileStream).on('error', function (err) {
// capture any errors that occur when writing data to the file
console.error('File Stream:', err);
}).on('close', function () {
console.log('Stream Done.');
// decompress files from tar.gz archive
targz.decompress({
src: './tmp/lo.tar.gz',
dest: './tmp'
}, function (err) {
if (err) {
console.log(err);
} else {
console.log("Extract Done!");
}
});
});
But when I put it in a lambda, nothing append.
It creates an empty file "lo.tar.gz" in tmp and it closes saying "statut:succeeded" and without any error message ... I put "console.log" between each line and the last to appear is the one I left ("Here the last console.log who appear") ...
Does anyone have any idea why the code is not being executed to the end?
Thank you :D!
EDIT : here the exact code for the lambda (the same with exports.handler) :
var AWS = require('aws-sdk');
var fs = require('fs');
var targz = require('targz');
exports.handler = async (event, context, callback) => {
var s3 = new AWS.S3({ apiVersion: '2006-03-01' });
var fileStream = fs.createWriteStream('./tmp/lo.tar.gz');
var s3Stream = s3.getObject({ Bucket: 'mybucketname', Key: 'lo.tar.gz'
}).createReadStream();
//console.log("Here the last console.log who appear");
// Listen for errors returned by the service
s3Stream.on('error', function (err) {
// NoSuchKey: The specified key does not exist
console.error(err);
});
s3Stream.pipe(fileStream).on('error', function (err) {
// capture any errors that occur when writing data to the file
console.error('File Stream:', err);
}).on('close', function () {
console.log('Stream Done.');
// decompress files from tar.gz archive
targz.decompress({
src: './tmp/lo.tar.gz',
dest: './tmp'
}, function (err) {
if (err) {
console.log(err);
} else {
console.log("Extract Done!");
}
});
});
}
I've been following some examples here and on Google of how to write a Nodejs Lambda function to fetch an (audio) file from a URL and save it to S3. I've arrived at this so far:
var AWS = require('aws-sdk');
var https = require('https');
var s3 = new AWS.S3();
const querystring = require('querystring');
exports.handler = function(event, context) {
const params = querystring.parse(event.body);
const audioUrl = params['audioUrl'];
https.get(audioUrl, function(res) {
var body = '';
res.on('data', function(chunk) {
// Agregates chunks
body += chunk;
});
res.on('end', function() {
// Once you received all chunks, send to S3
var params = {
Bucket: 'bucket_name',
Key: 'filename.wav',
Body: body
};
s3.putObject(params, function(err, data) {
if (err) {
console.error("ERROR: ", err, err.stack);
} else {
console.log("DATA:", data);
}
});
});
});
};
This results in file with filename filename.wav being saved to S3 bucket but the file is not a WAV audio file which is what I want. It seems that in the process of retrieving the file from audioUrl and saving it to S3 the format/encoding of the file is lost.
Any hints or solutions to this would be appreciated!
Thanks!
You're converting a binary file, into a string, that's why your uploaded file is corrupted. What you need to do is work with Buffer instead of string.
const chunks = [];
res.on('data', function(chunk) {
// Agregates chunks
chunks.push(chunk)
});
res.on('end', function() {
// Once you received all chunks, send to S3
var params = {
Bucket: 'bucket_name',
Key: 'filename.wav',
Body: Buffer.concat(chunks)
};
s3.putObject(params, function(err, data) {
if (err) {
console.error("ERROR: ", err, err.stack);
} else {
console.log("DATA:", data);
}
});
});
In any case, instead of buffering the chunks, is better to just pass the stream directly, and let the S3 SDK handle it.
var params = {
Bucket: 'bucket_name',
Key: 'filename.wav',
Body: res // pass the readable stream directly
};
s3.putObject(params, function(err, data) {});
The only caveat of passing a stream directly, is that the library can only work with streams that it can determine the length of. If that request sets the content-length correctly, you won't have any issue.
S3.putObject only accepts streams that it can determine the length of
streaming images doesnt work
You can bypass this limitation, by using s3.upload instead.
Note: Haven't worked with the S3 SDK in a while, but they didn't support pipe last time I used it, to do directly: res.pipe(s3.putObject())
I'm trying to read a file from an sftp server and stream that file into an s3 bucket. I'm not able to stream the file into the s3 bucket. Yes the file path is exactly correct. I'm not sure what I am doing wrong. When I run the code, it doesn't even try to upload the stream into the bucket because I don't get any upload console logs.
const aws = require('aws-sdk');
const s3 = new aws.S3();
const Client = require('ssh2').Client;
const conn = new Client();
const connSettings = {
host: event.serverHost,
port: event.port,
username: event.username,
password: event.password
};
exports.handler = function(event) {
conn.on('ready', function() {
conn.sftp(function(err, sftp) {
if (err) {
console.log("Errror in connection", err);
conn.end()
} else {
console.log("Connection established");
let readStream = sftp.createReadStream(remoteFilePath);
console.log(`Read Stream ${readStream}`)
// readStream outputs [object Object] to the console
const uploadParams = {
Bucket: s3Bucket,
Key: 'fileName',
Body: readStream
}
s3.upload (uploadParams, function (err, data) {
if (err) {
console.log("Error", err);
} if (data) {
console.log("Upload Success", data.Location);
}
});
conn.end()
}
});
}).connect(connSettings);
}
I want to be able to stream the readStream from sftp server into s3 bucket.
conn.end() ends the connection immediately. Move that to inside your s3.upload() callback so that your data actually gets transferred before the connection is closed.
This is a working Node 12 example of what I believe you are trying to accomplish:
const aws = require('aws-sdk');
const s3 = new aws.S3();
const Client = require('ssh2').Client;
const conn = new Client();
const { PassThrough } = require('stream');
conn.on('ready', () => {
conn.sftp((err, sftp) => {
const transferStream = new PassThrough();
s3.upload({
Bucket: s3Bucket,
Key: 'test_file.txt',
Body: transferStream
}, (err, data) => {
if (err) {
console.log(`Upload error: ${err}`);
}
if (data) {
console.log(`Uploaded to [${data.Location}].`);
}
});
sftp.createReadStream(remoteFilePath)
.pipe(transferStream)
.on('end', () => {
transferStream.end();
conn.end();
});
});
}).connect(connectionSettings);
I am trying to write a function that takes the mp3 url of the recording and then uploads that to S3. However, I keep getting a runtime error and the callback is never reached. If I move the callback below s3.upload(...) then the statement "attempting to upload mp3 is never logged.
exports.handler = function(context, event, callback) {
const twiml = new Twilio.twiml.VoiceResponse();
var AWS = require('aws-sdk');
var s3 = new AWS.S3();
var getUri = require('get-uri');
AWS.config.update({
accessKeyId: "...",
secretAccessKey: "..."
});
var client = context.getTwilioClient();
const recording_id = event.RecordingSid;
const uri = event.RecordingUrl + ".mp3";
getUri(uri, function (err, rs) {
if (err) {
console.log(err.message);
throw err;
}
var params = {
ACL: "public-read",
Body: rs,
Bucket: "...",
Key: "audio.mp3",
ContentType: 'audio/mp3'
};
s3.upload(params, function(err,data) {
console.log("attempting to upload mp3");
if (err) {
console.log("there is an error");
console.log(err.status);
throw err.message;
}
else {
console.log("Your upload has been successful.");
}
callback(null, twiml);
});
});
console.log("at the end");
};
Is there any other way to access the recording and put them in my public s3 bucket? Why is this never executing s3.upload(...).
Any insights into this is helpful! Thanks in advance!
app.get('/uploadsong',function(req,res){
console.log("Hi there")
var URI = 'http://sensongsmp3download.info/Kaala%20(2018)%20-%20Sensongsmp3.info/Thanga%20Sela%20--%20Sensongsmp3.Info.mp3';
var buffer = [];
request
.get(URI)
.on('error', function(err) {
console.log("error")
}).on('data',function(data){
buffer.push(data);
}).on('end',function(){
var completeSong = Buffer.concat(buffer);
var data = {
Body:completeSong,
Key: 'sample.mp3',
ContentType: 'audio/mp3'
}
s3Bucket.putObject(data, function(err, data){
if (err)
{
console.log('Error uploading data: ', data);
} else
{
console.log('upload successfull')
res.send('done');
}
})
})
})
here are the modules i have used
var request = require('request');
I contacted Twilio regarding this they responded that Twilio Functions have a strict 5 second time-out and the upload from the Twilio Function to S3 Bucket takes more than 5 seconds. My workaround was sending a string with all mp3 URLs separated by comma and a space. The lambda function would then parse through all the links and store all links in an array which would be used for audio playback.
I am trying to download a file from s3 and directly put into into a file on the filesystem using a writeStream in nodejs. This is my code:
downloadFile = function(bucketName, fileName, localFileName) {
//Donwload the file
var bucket = new AWS.S3({
params: { Bucket: bucketName },
signatureVersion: 'v4'
});
var file = require('fs').createWriteStream(localFileName);
var request = bucket.getObject({ Key: fileName });
request.createReadStream().pipe(file);
request.send();
return request.promise();
}
Running this function I get this error:
Uncaught Error: write after end
What is happening? Is the file closed before the write is finished? Why?
var s3 = new AWS.S3({
accessKeyId: accessKeyId,
secretAccessKey: secretAccessKey
}),
file = fs.createWriteStream(localFileName);
s3
.getObject({
Bucket: bucketName,
Key: fileName
})
.on('error', function (err) {
console.log(err);
})
.on('httpData', function (chunk) {
file.write(chunk);
})
.on('httpDone', function () {
file.end();
})
.send();
Also AWS notes an example of using promises like this:
const s3 = new aws.S3({apiVersion: '2006-03-01'});
const params = { Bucket: 'yourBucket', Key: 'yourKey' };
const file = require('fs').createWriteStream('./local_file_path');
const s3Promise = s3.getObject(params).promise();
s3Promise.then((data) => {
file.write(data.Body, () => {
file.end();
fooCallbackFunction();
});
}).catch((err) => {
console.log(err);
});
This works perfect for me.
https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/using-promises.html
EDIT: (15 Feb 2018) Updated the code, as you have to end the write stream (file.end()).
I have combined the above response with a typical gunzip operation in pipe:
var s3DestFile = "./archive.gz";
var s3UnzippedFile = './deflated.csv';
var gunzip = zlib.createGunzip();
var file = fs.createWriteStream( s3DestFile );
s3.getObject({ Bucket: Bucket, Key: Key })
.on('error', function (err) {
console.log(err);
})
.on('httpData', function (chunk) {
file.write(chunk);
})
.on('httpDone', function () {
file.end();
console.log("downloaded file to" + s3DestFile);
fs.createReadStream( s3DestFile )
.on('error', console.error)
.on('end', () => {
console.log("deflated to "+s3UnzippedFile)
})
.pipe(gunzip)
.pipe(fs.createWriteStream( s3UnzippedFile ))
})
.send();