Im trying to upload a folder from a local directory to an AWS S3 Bucket.
I have the following code.
var s3 = require('s3');
var awsS3Client = new AWS.S3({
accessKeyId: 'XXXXXXX',
secretAccessKey: 'XXXXXXX'
});
var options = {
s3Client: awsS3Client
};
var client = s3.createClient(options);
var params = {
localDir: "./zips",
deleteRemoved: true, // default false, whether to remove s3 objects
// that have no corresponding local file.
s3Params: {
Bucket: "node-files",
Prefix: "test/unzip/"
},
};
var uploader = client.uploadDir(params);
uploader.on('error', function (err) {
console.error("unable to sync:", err.stack);
});
uploader.on('progress', function () {
console.log("progress", uploader.progressAmount, uploader.progressTotal);
});
uploader.on('end', function () {
console.log("done uploading");
});
All works fine when uploading for the first time, the directory and all of its files are in tact and in the bucket.
However when i try a second time, the buffer just gets stuck and times out.
Im assuming i either need to set some kind of option to overwrite the existing files?
Related
I am trying to sync my local folder with files from s3 bucket(full bucket dir structure). I tried using node-s3-client npm package but the connection to the package is failing. The process just exits without any output.
here are the code snippets:
const s3Client = require('s3');
let client = s3Client.createClient({
s3Options: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
region: config.region,
},
});
let params = {
localDir: localdirName,
deleteRemoved: true,
s3Params: {
Bucket: Bname,
},
};
let uploader = client.downloadDir(params);
uploader.on('error', (err) => {
throw err;
});
uploader.on('progress', () =>
console.log('progress: ', uploader.progressAmount, uploader.progressTotal),
);
uploader.on('end', () => console.log('Upload completed!'));
If this can't be resolved, please help me through a workaround, Thanks!
If you send it with the right key, it will be deleted. I thought it was good because DeleteMarker got true, but if you send it with a key that is not in S3, DeleteMarker true comes out. What should we do with the validation?
const s3 = new AWS.S3({
accessKeyId:awsConfig.accessKeyId,
secretAccessKey: awsConfig.secretAccessKey,
region: awsConfig.region,
});
(async function () {
const params = {
Bucket: awsConfig.bucket,
Key: '17f316af9b011f08b68538a8ac76b73f.jpg',
};
const deleteS3Data = await s3.deleteObject(params).promise();
if(deleteS3Data.DeleteMarker !== true){
throw new Error("I don't have the key.");
}else{
return deleteS3Data;
}
}())
// deleteS3Data Whatever comes out true
I want to be able to copy files within the same bucket from the root directory to a subfolder within a subfolder however excluding that subfolder by using the aws-sdk.
i.e:
I want to use this AWS-CLI command in a gulp file task:
aws s3 cp s3://bucketName s3://bucketName/last_good/YYYYMMDD --recursive --exclude "last_good/*"
I've used the copy examples used from How to copy/move all objects in Amazon S3 from one prefix to other using the AWS SDK for Node.js
I am just not sure how to specify the folder to exclude. In my above example it would be the last_good folder.
var gulp = require('gulp');
var AWS = require('aws-sdk');
var async = require('async');
var bucketName = 'bucketname';
var oldPrefix = '';
var newPrefix = 'last_good/20190817/';
var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});
gulp.task('publish', function() {
CopyToLastGood();
}
function CopyToLastGood() {
var done = function(err, data) {
if (err) console.log(err);
else console.log(data);
};
s3.listObjects({Prefix: oldPrefix}, function(err, data) {
if (data.Contents.length) {
async.each(data.Contents, function(file, cb) {
var params = {
CopySource: bucketName + '/' + file.Key,
Key: file.Key.replace(oldPrefix, newPrefix)
};
s3.copyObject(params, function(copyErr, copyData){
if (copyErr) { // an error occured
console.log(err);
}
else {
console.log('Copied: ', params.Key); //successful response
cb();
}
});
}, done);
}
});
}
I expect contents of root to update last_good/20190817/ however not copying the last_good folder itself.
I've solved my solution using a delimiter option on the s3.listObjects params.
i.e:
s3.listObjects({Prefix: oldPrefix, Delimiter:'/'}
this only lists files within the root.
For the following code, I was asked to check, which runs on AWS Lambda and performs the simple task of uploading files to S3, timesout randomly when run manually from the aws console,but at the same time when run on a schedule, will always timeout at the first attempt and then run successfully the second time(it is run immediately after failing the first time).The code always seems to hang after calling s3.getBucketLocation, never reaching the getBucketLocation's callback(on the first attempt!).
All VPC's and NAT are set-up correctly, such that, when I remove s3.getBucketLocation from the code and call s3Upload.upload(params, cb); directly, the code runs without timeouts.
I would really like to understand why the first function fails alternatively when trying to retrieve the buckets region from AWS.
"use strict";
var urlHelper = require('url');
var fs = require('fs');
var AWS = require('aws-sdk');
var exp = {};
function saveToFile(path, content, cb) {
fs.writeFile(path, content, cb);
}
function saveToS3(bucket, key, content, cb) {
var s3 = new AWS.S3({apiVersion: '2006-03-01'});
var params = {
Bucket: bucket
};
s3.getBucketLocation(params, function(err, data) {
console.log('THIS IS NOT REACHED');
if (err) {
return cb(err);
}
var s3Upload = new AWS.S3({
apiVersion: '2006-03-01',
region: data.LocationConstraint
});
var params = {
Bucket: bucket,
Key: key,
Body: content,
ACL: 'public-read',
ContentType: 'text/xml',
CacheControl: 'max-age=60'
};
s3Upload.upload(params, cb);
});
}
exp.upload = function(path, content, cb) {
var url = urlHelper.parse(path);
switch(url.protocol) {
case 'file:':
case null:
saveToFile(path, content, cb);
break;
case 's3:':
saveToS3(url.host, url.pathname.substring(1), content, cb);
break;
default:
console.log('No matching protocol found for upload!');
cb(null, null);
}
};
module.exports = exp;
I want to concatenate the files uploaded on Amazon S3 server.
How can I do this.
Concatenation on local machine i can do using following code.
var fs = require('fs'),
files = fs.readdirSync('./files'),
clips = [],
stream,
currentfile,
dhh = fs.createWriteStream('./concatfile.mp3');
files.forEach(function (file) {
clips.push(file.substring(0, 6));
});
function main() {
if (!clips.length) {
dhh.end("Done");
return;
}
currentfile = './files/' + clips.shift() + '.mp3';
stream = fs.createReadStream(currentfile);
stream.pipe(dhh, {end: false});
stream.on("end", function() {
main();
});
}
main();
You can achieve what you want by breaking it into two steps:
Manipulating files on s3
Since s3 is a remote file storage, you can't run code on s3 server to do the operation locally (as #Andrey mentioned).
what you will need to do in your code is to fetch each input file, process them locally and upload the results back to s3. checkout the code examples from amazon:
var s3 = new AWS.S3();
var params = {Bucket: 'myBucket', Key: 'mp3-input1.mp3'};
var file = require('fs').createWriteStream('/path/to/input.mp3');
s3.getObject(params).createReadStream().pipe(file);
at this stage you'll run your concatenation code, and upload the results back:
var fs = require('fs');
var zlib = require('zlib');
var body = fs.createReadStream('bigfile.mp3').pipe(zlib.createGzip());
var s3obj = new AWS.S3({params: {Bucket: 'myBucket', Key: 'myKey'}});
s3obj.upload({Body: body}).
on('httpUploadProgress', function(evt) { console.log(evt); }).
send(function(err, data) { console.log(err, data) });
Merging two (or more) mp3 files
Since MP3 file include a header that specifies some information like bitrate, simply concatenating them together might introduce playback issues.
See: https://stackoverflow.com/a/5364985/1265980
what you want to use a tool to that. you can have one approach of saving your input mp3 files in tmp folder, and executing an external program like to change the bitrate, contcatenate files and fix the header.
alternatively you can use an library that allows you to use ffmpeg within node.js.
in their code example shown, you can see how their merge two files together within the node api.
ffmpeg('/path/to/part1.avi')
.input('/path/to/part2.avi')
.input('/path/to/part2.avi')
.on('error', function(err) {
console.log('An error occurred: ' + err.message);
})
.on('end', function() {
console.log('Merging finished !');
})
.mergeToFile('/path/to/merged.avi', '/path/to/tempDir');
Here's my quick take on the problem of downloading and processing S3 objects. My example is focused mostly on getting the data local and then processing it once it's all downloaded. I suggest you use one of the ffmpeg approaches mentioned above.
var RSVP = require('rsvp');
var s3 = new AWS.S3();
var bucket = '<your bucket name>';
var getFile = function(key, filePath) {
return new RSVP.Promise(function(resolve, reject) {
var file = require('fs').createWriteStream(filePath);
if(!file) {
reject('unable to open file');
}
s3.getObject({
Bucket: bucket,
Key: key
}).on('httpData', function(chunk) {
file.write(chunk);
}).on('httpDone', function() {
file.end();
resolve(filePath);
});
});
};
var tempFiles = ['<local temp filename 1>', '<local temp filename 2>'];
var keys = ['<s3 object key 1>', '<s3 object key 2>'];
var promises = [];
for(var i = 0; i < keys.length; ++i) {
var promise = getFile(keys[i], tempFiles[i]);
promises.push(promise);
}
RSVP.all(promises).then(function(data) {
//do something with your files
}).catch(function(error) {
//handle errors
});