s3 client.uploadFile() hangs with no error - node.js

My credentials work successfully with AWS cli to upload s3 files, the credentials are associated with an admin user on a bucket which is a static site. Update: Download works as expected. For some reason the node 's3' package fails, but no error is displayed, it just logs:
progress 0 4561 4561
progress 0 4561 4561
...
unable to upload: RequestTimeout: Your socket connection to the server was not read from or written to within the timeout period. Idle connections will be closed.
at Request.extractError (/Users/../node_modules/aws-sdk/lib/services/s3.js:700:35)
at Request.callListeners (/Users/../node_modules/aws-sdk/lib/sequential_executor.js:106:20)
at Request.emit (/Users/../node_modules/aws-sdk/lib/sequential_executor.js:78:10)
at Request.emit (/Users/../node_modules/aws-sdk/lib/request.js:688:14)
And I'm not sure what the problem would be? Any insights? The code is below, it's taken directly from the s3 package https://www.npmjs.com/package/s3
// RUNNING NODE 11.0 (due to un-updated dependencies in s3 or aws-sdk it must be under 11.15)
const { accessKeyId, secretAccessKey } = require('./tools/AWS.json')
const s3 = require('s3')
const AWS = require('aws-sdk')
const awsS3Client = new AWS.S3({
accessKeyId,
secretAccessKey,
signatureVersion: 'v4',
region: 'us-east-2',
})
const client = s3.createClient({
s3Client: awsS3Client,
})
const liveBucket = 'my-bucket'
const params = {
localFile: 'out/404.html',
s3Params: {
Bucket: liveBucket,
Key: '404.html',
},
}
const uploader = client.uploadFile(params)
uploader.on('error', function(err) {
// never runs ...
console.error('unable to upload:', err.stack)
})
uploader.on('progress', function() {
// runs like 5 times before failing after a long wait
console.log('progress', uploader.progressMd5Amount, uploader.progressAmount, uploader.progressTotal)
})
uploader.on('end', function() {
// never runs ...
console.log('done uploading')
})
Permissions:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "PublicReadGetObject",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::my-bucket/*"
}
]
}

It looks like your upload hangs because of a bucket permissions issue. Please try to add the following permissions to your bucket:
s3:PutObject
s3:PutObjectTagging
s3:ListBucketMultipartUploads
s3:ListMultipartUploadParts

Related

AWS lambda function delete files from S3 folder

I import some data from Funnel to S3 bucket. After that, Lambda function copy data to table in Redshift and I tried to delete all copied object from bucket folder but I keep getting timeout.
This is my code:
const Promise = require('bluebird');
const {Pool} = require('pg');
const AWS = require('aws-sdk');
async function emptyS3Directory(bucket, dir) {
const listParams = {
Bucket: bucket,
Prefix: dir
};
var s3 = new AWS.S3();
s3.listObjectsV2(listParams, function(err, data) // Here I always getting timeout{
});
.....
}
EDIT....
This is code of the function.
async function DeleteAllDataFromDir(bucket, dir) {
const listParams = {
Bucket: bucket,
Prefix: dir
};
var s3 = new AWS.S3();
const listedObjects = await s3.listObjects(listParams).promise();
console.log("reponse", listedObjects);
if (listedObjects.Contents.length === 0) return;
const deleteParams = {
Bucket: bucket,
Delete: { Objects: [] }
};
listedObjects.Contents.forEach(({ Key }) => {
deleteParams.Delete.Objects.push({ Key });
});
await s3.deleteObjects(deleteParams).promise();
if (listedObjects.IsTruncated) await DeleteAllDataFromDir(bucket, dir);
}
The first time I set the time out to 2 minutes, then I changed it to 10 minutes and I get the same error::
{
"errorType": "NetworkingError",
"errorMessage": "connect ETIMEDOUT IP:port",
"code": "NetworkingError",
"message": "connect ETIMEDOUT IP:port",
"errno": "ETIMEDOUT",
"syscall": "connect",
"address": "IP",
"port": port,
"region": "eu-west-2",
"hostname": "hostName",
"retryable": true,
"time": "2020-12-10T08:36:29.984Z",
"stack": [
"Error: connect ETIMEDOUT 52.95.148.74:443",
" at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1107:14)"
]
}
It appears that your bucket may reside in a different region than your lambda function based on the nature of the error.
Provide the region hash as an option when constructing your S3 client.
var s3 = new AWS.S3({region: 'bucket-region-hash'});
To figure the region hash, go to S3 Management Console.
Then from the sidebar, click "Buckets". In the resulting view, you'll find the region hash. It's the one marked in gold as shown in the picture below.

Problem with aws-sdk while creating S3 Buckets nodejs

I am currently using aws-sdk in order to list and create buckets.
My code is as follows
var AWS = require('aws-sdk');
const router = require('express').Router();
require('dotenv').config();
AWS.config.logger = console;
// AWS.config.update({region: process.env.AWS_REGION}); // AWS bucket region
s3 = new AWS.S3({
// apiVersion: '2006-03-01',
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: process.env.AWS_REGION
}); // s3 svc object
// Accepts Bucket Name, Bucket Created User/ Belonging Organization
router.post('/create', (req, res, next)=>{
var bucketParams = {
Bucket: req.body.bucket
}
console.log(process.env.AWS_ACCESS_KEY_ID);
s3.createBucket(bucketParams, (err, data)=>{
console.log("sample");
if (err) {
console.log("Error", err);
} else {
console.log("Success", data.Location);
}
})
});
router.get('/', (req, res, next)=>{
s3.listBuckets((err, data)=>{
if(err){
}else{
}
})
});
The AWS IAM user policy that I use for these are as follows.
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": "*"
}
]
}
The problem is, whenever I submit a http request, the request doesn't get terminated or return a valid response.
For an example.
I get the followings for creating a bucket named sample-1
[AWS s3 409 1.052s 0 retries] createBucket({
Bucket: 'sample1',
CreateBucketConfiguration: { LocationConstraint: 'eu-west-1' }
})
sample
Error BucketAlreadyExists: The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.
at Request.extractError (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/services/s3.js:837:35)
at Request.callListeners (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
at Request.emit (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/sequential_executor.js:78:10)
at Request.emit (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/request.js:688:14)
at Request.transition (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request.<anonymous> (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/request.js:38:9)
at Request.<anonymous> (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/request.js:690:12)
at Request.callListeners (/home/caesar/Workspace/res-s3/res-s3-backend/node_modules/aws-sdk/lib/sequential_executor.js:116:18) {
code: 'BucketAlreadyExists',
region: 'us-east-1',
time: 2020-08-18T15:24:22.974Z,
requestId: '3C1F8277A6CAD712',
extendedRequestId: 'bfRp36yH8Gh64zjM5VrUqCJi0V1AY5Sc5Snpf5yROPyV0HHgWTtE7gIEz70HRHb2JoOcO6jfLvQ=',
cfId: undefined,
statusCode: 409,
retryable: false,
retryDelay: 5.386495440613426
}
The strangest thing is that there is no bucket with the same name and the region which I used in this case had been eu-west-1. Anyone have any clue on why this happens and why the requests are not getting terminated.
Bucket names are global to AWS. see below
Amazon S3 bucket names must be unique globally. If you get the "Bucket name already exists" or "BucketAlreadyExists" error, then you must use a different bucket name to create the bucket. These error messages indicate that another AWS account owns a bucket with the same name.
You should try using a unique name.

Not able to connect to AWS through aws-sdk node js

I am new to AWS SDK and trying to follow the documents there. https://aws.amazon.com/developers/getting-started/nodejs/ I am trying to create a bucket and upload some text file there. But I am getting the below mentioned error.
{ Error: connect ETIMEDOUT 52.216.206.51:443
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1104:14)
message: 'connect ETIMEDOUT 52.216.206.51:443',
errno: 'ETIMEDOUT',
code: 'NetworkingError',
syscall: 'connect',
address: '52.216.206.51',
port: 443,
region: 'us-east-1',
hostname:
'node-sdk-sample-c58bc12a-70be-4578-9c18-1c815962653b.s3.amazonaws.com',
retryable: true,
time: 20
I have tried giving credential in credential file under userid/.aws/credentials
[default]
aws_access_key_id = my cred...
aws_secret_access_key = my cred....
Also I have tried giving the credential in the code itself
AWS.config.update({accessKeyId: 'my cred', secretAccessKey: 'my secret key', region: 'us-east-1'});
`// Load the SDK and UUID
var AWS = require('aws-sdk');
var uuid = require('node-uuid');
AWS.config.update({accessKeyId: '......', secretAccessKey: '.....', region: 'us-east-1'});
var s3 = new AWS.S3();
var bucketName = 'node-sdk-sample-' + uuid.v4();
var keyName = 'hello_world.txt';
s3.createBucket({Bucket: bucketName}, function() {
var params = {Bucket: bucketName, Key: keyName, Body: 'Hello World!'};
s3.putObject(params, function(err, data) {
if (err)
console.log(err)
else
console.log("Successfully uploaded data to " + bucketName + "/" + keyName);
});
});`
I am able to resolve this issue. The issue was related to proxy. I was trying to access this from behind a firewall. After configuring the proxy setting, I am able to connect to AWS.
var proxy = require('proxy-agent');
AWS.config.update({ httpOptions: { agent: proxy('ur proxy url') } });
const AWS = require('aws-sdk'); //require aws sdk library
var params = {
Bucket: awsBucketName,
Key: path + fileName,
Body: value.data
};
//s3 upload start
s3bucket.upload(params, (err, data) => {
if (err) {
return reject(err);
}
return resolve(data.key);
});
//get signed url for downloading the uploaded file
const url = await s3bucket.getSignedUrl('getObject', {
Bucket: awsBucketName,
Key: key,
Expires: 60*3
});

Why is my access denied on s3 (using the aws-sdk for Node.js)?

I'm trying to read an existing file from my s3 bucket, but I keep getting "Access Denied" with no explanation or instructions on what to do about it. Here is the code I am using:
'use strict'
var AWS = require('aws-sdk')
const options = {
apiVersion: '2006-03-01',
params: {
Bucket: process.env['IMAGINATOR_BUCKET']
},
accessKeyId: process.env['IMAGINATOR_AWS_ACCESS_KEY_ID'],
secretAccessKey: process.env['IMAGINATOR_AWS_SECRET_ACCESS_KEY'],
signatureVersion: 'v4'
}
console.log('options', options)
var s3 = new AWS.S3(options)
module.exports = exports = {
get (name, cb) {
const params = {
Key: name + '.json'
}
console.log('get params', params)
return s3.getObject(params, cb)
},
set (name, body, cb) {
const params = {
Key: name + '.json',
Body: body
}
console.log('set params', params)
return s3.putObject(params, cb)
}
}
And this is what I'm getting as output when using the get method and logging the error provided in the callback (with sensitive information censored out):
options { apiVersion: '2006-03-01',
params: { Bucket: CENSORED_BUT_CORRECT },
accessKeyId: CENSORED_BUT_CORRECT,
secretAccessKey: CENSORED_BUT_CORRECT,
signatureVersion: 'v4' }
get params { Key: 'whitelist.json' }
err { [AccessDenied: Access Denied]
message: 'Access Denied',
code: 'AccessDenied',
region: null,
time: Wed Sep 21 2016 11:17:50 GMT-0400 (EDT),
requestId: CENSORED,
extendedRequestId: CENSORED,
cfId: undefined,
statusCode: 403,
retryable: false,
retryDelay: 20.084538962692022 }
/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/request.js:31
throw err;
^
AccessDenied: Access Denied
at Request.extractError (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/services/s3.js:538:35)
at Request.callListeners (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at Request.emit (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
at Request.emit (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/request.js:668:14)
at Request.transition (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request.<anonymous> (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/request.js:38:9)
at Request.<anonymous> (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/request.js:670:12)
at Request.callListeners (/Users/shawn/git/vigour-io/imaginate/node_modules/aws-sdk/lib/sequential_executor.js:115:18)
Now I'm not sure what to do beacuse I think I'm doing things correctly according to the docs, but it's not working and the error message doesn't say why my access is denied... Any idea what the next step should be to get this working?
The problem was that my new IAM user didn't have a policy attached to it. I assigned it the AmazonS3FullAccess policy and now it works.
As pointed out in the comments, a more restrictive policy would be much safer
This could also happen if you're trying to set ACL to "public-read" but the bucket is blocking public access. For example if you mean to upload static assets to a misconfigured S3 bucket. You can change it in your bucket settings.
FullAccess in your policy is not required. You can try something like this:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"s3:Put*",
"s3:Get*",
"s3:List*",
"s3:Delete*"
],
"Resource": [
"arn:aws:s3:::bucket/*",
"arn:aws:s3:::bucket"
]
},
{
"Sid": "VisualEditor1",
"Effect": "Allow",
"Action": "s3:ListAllMyBuckets",
"Resource": "*"
}
]
}
These errors can occur when the object which you are trying to read does not exist. From what I understood the AWS errors are not so clear in these situations.
Validate if your key/bucket is correct and if you are sending the correct params on the API method.
I already got this problem two times:
when I was replacing the key param with the bucket param and vice versa and I was trying to read an s3 object using the getObject() method.
when I was trying to copy a file to a location that did not exist using the copyObject() method.
Steps
1: click on Users in IAM (in AWS)
2: click on permission tab
3: click on add permission then click on add group
4: search s3fullaccess in searchbar
5: select AmazonS3FullAccess and type any group name then click on create
6: perform action through your API again
7: done
"code":"AccessDenied","region":null,"time":"2020-05-24T05:20:56.219Z","requestId": ...
Applied the below policy in s3 aws console > Bucket policy editor of 'Permissions' tab to get rid of the above error,
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::<IAM-user-ID>:user/testuser"
},
"Action": [
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:GetBucketLocation",
"s3:Get*",
"s3:Put*"
],
"Resource": "arn:aws:s3:::srcbucket"
}
]
}
In my case, I updated the variable holding s3 bucket name in .env file but didn't update the variable in the program, thus the program was receiving undefined value for the bucket name variable which caused my program throw access denied error.
So do make sure you are using correct bucket name or correct variable name if you are storing bucket name in a variable
I had the same error and it was because the file I was trying to access was not in the bucket. So make sure you are using the right bucket name and that the name of the file you are looking for is exactly the same as the one that exists in that bucket.
https://www.diffchecker.com/diff is a good tool to look for differences in strings

Nodejs adm-zip fails in Lambda with Access Denied Error

My goal is to read a file inside a zip file from Lambda which is called by S3 Put, here is the code, but it fails Access Denied, even though Lambda has full access to S3:
var AWS = require('aws-sdk');
var fs = require('fs');
var s3 = new AWS.S3();
var AdmZip = require('adm-zip');
exports.handler = function(event, context, callback) {
const bucket = event.Records[0].s3.bucket.name;
const key = event.Records[0].s3.object.key;
var params = {Bucket: 'bucket', Key: 'key'};
console.log('Bucket Name:', bucket);
console.log('Object Name:', key);
var buffer = s3.getObject(params).createReadStream();
var zip = new AdmZip(buffer);
var zipEntries = zip.getEntries();
zipEntries.forEach(function(zipEntry) {
console.log(zipEntry.toString());
console.log('File Name: ',zipEntry.entryName);
if (zipEntry.entryName == "text.txt") {
console.log(zipEntry.data.toString('utf8'));
}
});
};
I am getting the following error:
START RequestId: fc524797-4d28-11e6-a092-5b3299c063a6 Version: $LATEST
2016-07-18T20:48:33.983Z fc524797-4d28-11e6-a092-5b3299c063a6 Bucket Name: myTestBucket123
2016-07-18T20:48:33.984Z fc524797-4d28-11e6-a092-5b3299c063a6 Object Name: test.zip
2016-07-18T20:48:35.145Z fc524797-4d28-11e6-a092-5b3299c063a6 AccessDenied: Access Denied
at Request.extractError (/var/runtime/node_modules/aws-sdk/lib/services/s3.js:524:35)
at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at Request.emit (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
at Request.emit (/var/runtime/node_modules/aws-sdk/lib/request.js:615:14)
at Request.transition (/var/runtime/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/var/runtime/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /var/runtime/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:38:9)
at Request.<anonymous> (/var/runtime/node_modules/aws-sdk/lib/request.js:617:12)
at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:115:18)
END RequestId: fc524797-4d28-11e6-a092-5b3299c063a6
REPORT RequestId: fc524797-4d28-11e6-a092-5b3299c063a6 Duration: 4318.87 ms Billed Duration: 4400 ms Memory Size: 128 MB Max Memory Used: 24 MB
Process exited before completing request
Here is the policy for the role that Lambda is using:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "arn:aws:logs:*:*:*"
},
{
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
"arn:aws:s3:::*"
]
}
]
}

Resources