How to use AWS KMS Encryption in the node js SDK - node.js

I cannot figure out how to upload files into AWS S3 using KMS encryption from the Node JS SDK. I keep getting a 403: Access Denied error. I am able to get files from AWS S3 using KMS.
I am reusing most of the code from https://github.com/gilt/node-s3-encryption-client
Main Class
var fs = require('fs'),
AWS = require('aws-sdk'),
crypt = require("./crypt"),
kms,
s3;
const metadataCipherAlgorithm = 'cipher-algorithm',
metadataDecryptedEncoding = 'decrypted-encoding'
metadataKmsKeyName = 'x-amz-key';
/**
* Constructor - Initializes S3 sdk connection
*/
function S3FileStreamer(key, secret, region) {
if (region) {
AWS.config.region = region;
}
//set credentials if passed in
if (key && secret) {
AWS.config.update({accessKeyId: key, secretAccessKey: secret})
}
s3 = new AWS.S3({signatureVersion: "v4"});
kms = new AWS.KMS({apiVersion: '2014-11-01'});
}
S3FileStreamer.prototype.uploadFile = function(bucket, key, kmsKey, filename, onComplete) {
var params = {
Bucket: bucket,
Key: key,
Body: fs.readFileSync(filename),
ContentType: getMimeType(filename)
};
params.KmsParams = {
KeyId: kmsKey,
KeySpec: 'AES_256'
}
kmsUpload(params, function(err, data) {
if (err) onComplete(err, null);
else {
onComplete(err, data);
}
});
};
function kmsUpload(params, callback) {
var kmsParams = params.KmsParams
if (kmsParams && kmsParams.KeyId) {
kms.generateDataKey(kmsParams, function(err, kmsData) {
if (err) {
callback(err, null);
} else {
var helper = new crypt.Helper(kmsData.Plaintext.toString('base64'), {algorithm: params.CipherAlgorithm, decryptedEncoding: params.DecryptedEncoding});
params.Body = helper.encrypt(params.Body);
params.Metadata = params.Metadata || {};
params.Metadata[metadataKmsKeyName] = kmsData.CiphertextBlob.toString('base64');
if (params.CipherAlgorithm) params.Metadata[metadataCipherAlgorithm] = params.CipherAlgorithm;
if (params.DecryptedEncoding) params.Metadata[metadataDecryptedEncoding] = params.DecryptedEncoding;
putObject(params, callback);
}
})
} else {
putObject(params, callback);
}
}
function putObject(params, callback) {
delete params.KmsParams;
delete params.CipherAlgorithm;
delete params.DecryptedEncoding;
s3.putObject(params, callback);
}
Crypt class
var crypto = require('crypto');
/*
options:
algorithm: Anything from crypto.getCiphers()
decryptedEncoding: 'utf8', 'ascii', or 'binary'
outputEncoding: 'binary', 'base64', or 'hex'
*/
exports.Helper = function(password, options) {
this.password = password;
options = options || {};
this.algorithm = options.algorithm || 'aes-256-cbc';
this.decryptedEncoding = options.decryptedEncoding || 'utf8';
this.encryptedEncoding = options.encryptedEncoding || 'base64';
}
exports.Helper.prototype.encrypt = function(unencrypted) {
var cipher = crypto.createCipher(this.algorithm, this.password);
return cipher.update(unencrypted, this.decryptedEncoding, this.encryptedEncoding) + cipher.final(this.encryptedEncoding);
}
exports.Helper.prototype.decrypt = function(encrypted) {
var decipher = crypto.createDecipher(this.algorithm, this.password);
return decipher.update(encrypted, this.encryptedEncoding, this.decryptedEncoding) + decipher.final(this.decryptedEncoding);
}
Is there something I am missing here, an extra metadata tag that needs to be set?
Is the keyId parameter that is passed to the kms generateDataKey method supposed to be in some sort of unique format? I am just simply passing in my key.

While the other answer about checking permissions is undoubtedly correct. I had a hard time finding a good example of how to decode S3 objects using the AES GCM encryption algorithm. I managed to get this code to work based on the aws ruby sdk (as I found the node-s3-encryption-client a little old).
/**
* Decrypt s3 file data
* #param {object} objectData result of s3 get call
* #param {Function} callback function(err, data) returns error or decrypted data
*/
function decrypt(objectData, callback) {
var metadata = objectData.Metadata || {};
var kmsKeyBase64 = metadata['x-amz-key-v2'];
var iv = metadata['x-amz-iv'];
var tagLen = (metadata['x-amz-tag-len'] || 0)/8;
var algo = metadata['x-amz-cek-alg'];
var encryptionContext = JSON.parse(metadata['x-amz-matdesc']);
switch (algo) {
case 'AES/GCM/NoPadding':
algo = 'aes-256-gcm';
break;
case 'AES/CBC/PKCS5Padding':
algo = 'aes-256-cbc';
break;
default:
callback(new Error('Unsupported algorithm: ' + algo), null);
return;
}
if (typeof (kmsKeyBase64) === 'undefined') {
callback(new Error('Missing key in metadata'), null);
return;
}
var kmsKeyBuffer = new Buffer(kmsKeyBase64, 'base64');
kms.decrypt({
CiphertextBlob: kmsKeyBuffer,
EncryptionContext: encryptionContext
}, function(err, kmsData) {
if (err) {
callback(err, null);
} else {
var decipher = crypto.createDecipheriv(algo,
kmsData.Plaintext,
new Buffer(iv, 'base64'));
if (tagLen !== 0) {
// the tag is appended to the data buffer
var tag = objectData.Body.slice(-tagLen);
decipher.setAuthTag(tag);
}
var data = objectData.Body.slice(0,-tagLen);
var dec = decipher.update(data, 'binary', 'utf8');
dec += decipher.final('utf8');
console.log("Decoded:", dec);
callback(null, dec);
}
});
}

Thanks for all the help. I figured out the solution to my question.
I went back to using just using the aws-sdk node module and took out all the code I got from the node-s3-encryption-client module.
All I needed to do in order to successfully upload a file into Amazon S3 using KMS encryption was to add two parameters before passing my params object to the putObject method. These parameters were ServerSideEncryption and SSEKMSKeyId as shown below. It now works!
var params = {
Bucket: bucket,
Key: key,
Body: fs.readFileSync(filename),
ContentType: getMimeType(filename),
ServerSideEncryption: 'aws:kms',
SSEKMSKeyId: kmsKey
};
s3.putObject(params, function(err, data) {
if (err) {
console.log(err);
} else {
console.log(data);
});

Related

Decrypt text with AWS KMS in NodeJs

I am trying to decrypt some text encrypted with AWS KMS using aws-sdk and NodeJs. I started to play today with NodeJs so I am a newbie with it.
I have this problem resolved with Java but I am trying to migrate an existing Alexa skill from Java to NodeJs.
The code to decrypt is:
function decrypt(buffer) {
const kms = new aws.KMS({
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
region: 'eu-west-1'
});
return new Promise((resolve, reject) => {
let params = {
"CiphertextBlob" : buffer,
};
kms.decrypt(params, (err, data) => {
if (err) {
reject(err);
} else {
resolve(data.Plaintext);
}
});
});
};
When I run this code with a correct CiphertextBlob, I get this error:
Promise {
<rejected> { MissingRequiredParameter: Missing required key 'CiphertextBlob' in params
at ParamValidator.fail (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\param_validator.js:50:37)
at ParamValidator.validateStructure (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\param_validator.js:61:14)
at ParamValidator.validateMember (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\param_validator.js:88:21)
at ParamValidator.validate (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\param_validator.js:34:10)
at Request.VALIDATE_PARAMETERS (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\event_listeners.js:126:42)
at Request.callListeners (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\sequential_executor.js:106:20)
at callNextListener (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\sequential_executor.js:96:12)
at D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\event_listeners.js:86:9
at finish (D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\config.js:349:7)
at D:\Developing\abono-transportes-js\node_modules\aws-sdk\lib\config.js:367:9
message: 'Missing required key \'CiphertextBlob\' in params',
code: 'MissingRequiredParameter',
time: 2019-06-30T20:29:18.890Z } }
I don't understand why I am receiving that if CiphertextBlob is in the params variable.
Anyone knows?
Thanks in advance!
EDIT 01/07
Test to code the feature:
First function:
const CheckExpirationDateHandler = {
canHandle(handlerInput) {
return handlerInput.requestEnvelope.request.type === 'IntentRequest'
&& handlerInput.requestEnvelope.request.intent.name === 'TtpConsultaIntent';
},
handle(handlerInput) {
var fecha = "";
var speech = "";
userData = handlerInput.attributesManager.getSessionAttributes();
if (Object.keys(userData).length === 0) {
speech = consts.No_Card_Registered;
} else {
console.log("Retrieving expiration date from 3rd API");
fecha = crtm.expirationDate(cipher.decrypt(userData.code.toString()));
speech = "Tu abono caducará el " + fecha;
}
return handlerInput.responseBuilder
.speak(speech)
.shouldEndSession(true)
.getResponse();
}
}
Decrypt function provided with a log:
// source is plaintext
async function decrypt(source) {
console.log("Decrypt func INPUT: " + source)
const params = {
CiphertextBlob: Buffer.from(source, 'base64'),
};
const { Plaintext } = await kms.decrypt(params).promise();
return Plaintext.toString();
};
Output:
2019-07-01T19:01:12.814Z 38b45272-809d-4c84-b155-928bee61a4f8 INFO Retrieving expiration date from 3rd API
2019-07-01T19:01:12.814Z 38b45272-809d-4c84-b155-928bee61a4f8 INFO Decrypt func INPUT:
AYADeHK9xoVE19u/3vBTiug3LuYAewACABVhd3MtY3J5cHRvLXB1YmxpYy1rZXkAREF4UW0rcW5PSElnY1ZnZ2l1bHQ2bzc3ZnFLZWZMM2J6YWJEdnFCNVNGNzEyZGVQZ1dXTDB3RkxsdDJ2dFlRaEY4UT09AA10dHBDYXJkTnVtYmVyAAt0aXRsZU51bWJlcgABAAdhd3Mta21zAEthcm46YXdzOmttczpldS13ZXN0LTE6MjQwMTE3MzU1MTg4OmtleS81YTRkNmFmZS03MzkxLTRkMDQtYmUwYi0zZDJlMWRhZTRkMmIAuAECAQB4sE8Iv75TZ0A9b/ila9Yi/3vTSja3wM7mN/B0ThqiHZEBxYsoWpX7jCqHMoeoYOkVtAAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDNnGIwghz+b42E07KAIBEIA76sV3Gmp5ib99S9H4MnY0d1l............
2019-07-01T19:01:12.925Z 38b45272-809d-4c84-b155-928bee61a4f8 INFO Error
handled: handlerInput.responseBuilder.speak(...).shouldEndSession is
not a function
2019-07-01T19:01:13.018Z 38b45272-809d-4c84-b155-928bee61a4f8 ERROR Unhandled Promise
Rejection {"errorType":"Runtime.UnhandledPromiseRejection","errorMessage":"InvalidCiphertextException:
null","stack":["Runtime.UnhandledPromiseRejection:
InvalidCiphertextException: null","...
That either means you're missing key 'CiphertextBlob' or its value is undefined.
Please checkout the value you're passing in as buffer.
For reference, I also added my working code example that I used.
import { KMS } from 'aws-sdk';
import config from '../config';
const kms = new KMS({
accessKeyId: config.aws.accessKeyId,
secretAccessKey: config.aws.secretAccessKey,
region: config.aws.region,
});
// source is plaintext
async function encrypt(source) {
const params = {
KeyId: config.aws.kmsKeyId,
Plaintext: source,
};
const { CiphertextBlob } = await kms.encrypt(params).promise();
// store encrypted data as base64 encoded string
return CiphertextBlob.toString('base64');
}
// source is plaintext
async function decrypt(source) {
const params = {
CiphertextBlob: Buffer.from(source, 'base64'),
};
const { Plaintext } = await kms.decrypt(params).promise();
return Plaintext.toString();
}
export default {
encrypt,
decrypt,
};
----- ADDED -----
I was able to reproduce your issue.
decrypt("this text has never been encrypted before!");
This code throws same error.
So if you pass plain text that has never been encrypted before or has been encrypted with different key, it throws InvalidCiphertextException: null.
Now I'll give you one usage example.
encrypt("hello world!") // this will return base64 encoded string
.then(decrypt) // this one accepts encrypted string
.then(decoded => console.log(decoded)); // hello world!
I kept on getting this error in my AWS lambda when trying the accepted solution, using AWS KMS over an environment variable I had encrypted by using AWS user interface.
It worked for me with this code adapted from the AWS official solution:
decrypt.js
const AWS = require('aws-sdk');
AWS.config.update({ region: 'us-east-1' });
module.exports = async (env) => {
const functionName = process.env.AWS_LAMBDA_FUNCTION_NAME;
const encrypted = process.env[env];
if (!process.env[env]) {
throw Error(`Environment variable ${env} not found`)
}
const kms = new AWS.KMS();
try {
const data = await kms.decrypt({
CiphertextBlob: Buffer.from(process.env[env], 'base64'),
EncryptionContext: { LambdaFunctionName: functionName },
}).promise();
console.info(`Environment variable ${env} decrypted`)
return data.Plaintext.toString('ascii');
} catch (err) {
console.log('Decryption error:', err);
throw err;
}
}
To be used like this:
index.js
const decrypt = require("./decrypt.js")
exports.handler = async (event, context, callback) => {
console.log(await decrypt("MY_CRYPTED_ENVIRONMENT_VARIABLE"))
}
EncryptionContext is a must for this to work.
Let's say the name of EnvironmentVariable is Secret
The code below reads the EnvironmentVariable called Secret and returns decrypted secret as plain text in the body.
Please see the function code posted below
'use strict';
const aws = require('aws-sdk');
var kms = new aws.KMS();
exports.handler = (event, context, callback) => {
const functionName = process.env.AWS_LAMBDA_FUNCTION_NAME;
const encryptedSecret = process.env.Secret;
kms.decrypt({
CiphertextBlob: new Buffer(encryptedSecret, 'base64'),
EncryptionContext: {
LambdaFunctionName: functionName /*Providing the name of the function as the Encryption Context is a must*/
},
},
(err, data) => {
if (err) {
/*Handle the error please*/
}
var decryptedSecret = data.Plaintext.toString('ascii');
callback(null, {
statusCode: 200,
body: decryptedSecret,
headers: {
'Content-Type': 'application/json',
},
});
});
};

How to execute my nodejs code in block in AWS Lambda

I am developing a code to validate openssl keys using NodeJs in AWS Lambda. I have a requirement to fetch the public key from the DynamoDB and to validate with the user given private key. As of now, I have two classes, one is to fetch the data from the dynamodb and second is to validate public key and private. But because of asynchronous NodeJS executing my Second class first and is not waiting until my first class gets the data from dynamoDB.
Please help me to execute the code in sequence.
I have tried Promise, bluebird and callback But I'm not able to understand them
const AWS = require('aws-sdk')
const dynamodb = new AWS.DynamoDB();
var promise = require('promise');
exports.handler = async (event) => {
var privatek = '-----BEGIN RSA PRIVATE KEY-----\n'+
'MIICWwIBAAKBgQC2rWURD7fK/3B0W7d36BJnv4ITzSd+K6o+itgHkqe+0EOvoOn2\n'+
'yHK3J11j2c+BkgAcdfwYaBFhH7Gubvyt0TLAKJvxi6cIbD4DVJqoTwJPzgdCczKZ\n'+
'AdhevoYam3t/Q454pW5N7IoF5IzMgPypRbPhi7JnkqcE1/CIXC3hrysMeQIDAQAB\n'+
'AoGANFPlEIcVGdQkDWC8ZF+Y7hkglLV+q5iscq/pA/pRjMoxqVyJyIRQwABJszGQ\n'+
'TEhbOcveQ8uDtvOSPSpTvSKgy4fxmH0/RuypTYcAD/BN76T1DDODSsyn+KuNOdko\n'+
'x6bo30wexmBL/itya9VJMBM49iMMYtYBtOuoJGamMc+vUQECQQDxaWk9alNa37Yb\n'+
'SAfQRGoU7xJvuVQ8qHBY0EgCzYwaMkWuWKkk8GA058PezUxEjwZN8ZRVsYO2YHG1\n'+
'3w3vcF+ZAkEAwbdf1ZVpPEsVyXeftnnu5uPxjN6SGqojV1M1/QXQJaFVd0SFAWMY\n'+
'LE1tqFI6KEfQ1huehvwXhja6HU5z4p+f4QJAce/xRpYvHx2koj2dynLvqk+nYOmU\n'+
'U0igNZqf0grXC+ocLwwTUKbOkUmtjTNRwq3KKPFStBsi8emU4WST/CUKSQJAE2Af\n'+
'AsLl+rTb4gHIBL1fatKjx14/qNEZpdNZ1AvvzMO9Q6ej0gayVUQNUseer4a3WaL7\n'+
'kS7Hv5HbvbCqIKGsoQJAS3LAPW5Wpc2rHFJQDryIGdVnZLW6YdFXGudzMRjHsIB0\n'+
'UMKUywmjRf45ugQMBKJ+iesDwNLmXtOjcB9AdGRz5w==\n'+
'-----END RSA PRIVATE KEY-----';
var publick = '-----BEGIN PUBLIC KEY-----\n'+
'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC2rWURD7fK/3B0W7d36BJnv4IT\n'+
'zSd+K6o+itgHkqe+0EOvoOn2yHK3J11j2c+BkgAcdfwYaBFhH7Gubvyt0TLAKJvx\n'+
'i6cIbD4DVJqoTwJPzgdCczKZAdhevoYam3t/Q454pW5N7IoF5IzMgPypRbPhi7Jn\n'+
'kqcE1/CIXC3hrysMeQIDAQAB\n'+
'-----END PUBLIC KEY-----';
console.log(event["pk"]);
const table = "MARKETPLACE_USER_RELATED_APPS_TABLE";
var dynamo = new Dynamo(table);
var prik = dynamo.read(dynamo.client());
console.log(prik);
var clientid = '12345'; //event['client_id']
var unique_id = parseInt(Math.random()*10000000000000000).toString();
var time_stamp = new Date();
var Authkey = clientid + ':' + unique_id + ':' + time_stamp;
console.log(Authkey+"Authkey");
console.log(time_stamp + 'time_stamp');
var cryp = new CryptoVerify(publick,privatek, Authkey);
console.log(cryp.hash+'hash key');
console.log(event['pk'])
const response = {
statusCode: 200,
body: cryp.verify(cryp.signature())
};
return response
};
function CryptoVerify(publickey, privatekey, authkey)
{
this.crypto = require ('crypto')
this.hash = this.crypto.createHash('sha256')
.update(authkey)
.digest('base64');
console.log(this.hash+'haskkk');
this.signature = function(){
try{
var signer = this.crypto.createSign('sha1');
signer.update(this.hash);
var sign = signer.sign(privatekey,'base64');
console.log(sign + "Sign")
}
catch(e)
{
var sign = "Private key Errpr: Not matching with the Public
key\nExact " +e;
}
return sign;
}
this.verify = function(sign){
var verifier = this.crypto.createVerify('sha1');
verifier.update(this.hash);
try {
this.ver = verifier.verify(publickey, sign,'base64');
console.log(this.ver);
}
catch (e){
console.log("Private key Error: Not matching with the Public
key/n Exact Error:"+e);
this.ver = e
}
return this.ver;
}
}
function Dynamo(table)
{
this.params = {Key :{"email_id":{S:"testuser#tml.com"},
"app_name":{S:"EDH"}},
TableName: table};
this.op = {};
this.client = function(){
try{
var dynamodb = new AWS.DynamoDB();
}
catch(e){
console.log("Error: Class DynamoDB is not defined");
}
return dynamodb;};
this.read = function(dy){
try{
dy.getItem(this.params, function(err, data){
if(err) console.log(err, err.stack);
else {
this.op = data;
console.log(this.op);
}
});
}
catch(e){
console.log('Please check getIten function to clear the error');
}
return this.op;
};
}
I expect result with the dynamoDB value to be fetched first then Validate class.
Use callback version of lambda handler or when using asyc await wrap all the callback into a promise or promisify the methods using Bluebird's promisify all method.
const AWS = require('aws-sdk')
const dynamodb = new AWS.DynamoDB();
var promise = require('promise');
exports.handler = async (event) => {
var privatek = '-----BEGIN RSA PRIVATE KEY-----\n'+
'MIICWwIBAAKBgQC2rWURD7fK/3B0W7d36BJnv4ITzSd+K6o+itgHkqe+0EOvoOn2\n'+
'yHK3J11j2c+BkgAcdfwYaBFhH7Gubvyt0TLAKJvxi6cIbD4DVJqoTwJPzgdCczKZ\n'+
'AdhevoYam3t/Q454pW5N7IoF5IzMgPypRbPhi7JnkqcE1/CIXC3hrysMeQIDAQAB\n'+
'AoGANFPlEIcVGdQkDWC8ZF+Y7hkglLV+q5iscq/pA/pRjMoxqVyJyIRQwABJszGQ\n'+
'TEhbOcveQ8uDtvOSPSpTvSKgy4fxmH0/RuypTYcAD/BN76T1DDODSsyn+KuNOdko\n'+
'x6bo30wexmBL/itya9VJMBM49iMMYtYBtOuoJGamMc+vUQECQQDxaWk9alNa37Yb\n'+
'SAfQRGoU7xJvuVQ8qHBY0EgCzYwaMkWuWKkk8GA058PezUxEjwZN8ZRVsYO2YHG1\n'+
'3w3vcF+ZAkEAwbdf1ZVpPEsVyXeftnnu5uPxjN6SGqojV1M1/QXQJaFVd0SFAWMY\n'+
'LE1tqFI6KEfQ1huehvwXhja6HU5z4p+f4QJAce/xRpYvHx2koj2dynLvqk+nYOmU\n'+
'U0igNZqf0grXC+ocLwwTUKbOkUmtjTNRwq3KKPFStBsi8emU4WST/CUKSQJAE2Af\n'+
'AsLl+rTb4gHIBL1fatKjx14/qNEZpdNZ1AvvzMO9Q6ej0gayVUQNUseer4a3WaL7\n'+
'kS7Hv5HbvbCqIKGsoQJAS3LAPW5Wpc2rHFJQDryIGdVnZLW6YdFXGudzMRjHsIB0\n'+
'UMKUywmjRf45ugQMBKJ+iesDwNLmXtOjcB9AdGRz5w==\n'+
'-----END RSA PRIVATE KEY-----';
var publick = '-----BEGIN PUBLIC KEY-----\n'+
'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC2rWURD7fK/3B0W7d36BJnv4IT\n'+
'zSd+K6o+itgHkqe+0EOvoOn2yHK3J11j2c+BkgAcdfwYaBFhH7Gubvyt0TLAKJvx\n'+
'i6cIbD4DVJqoTwJPzgdCczKZAdhevoYam3t/Q454pW5N7IoF5IzMgPypRbPhi7Jn\n'+
'kqcE1/CIXC3hrysMeQIDAQAB\n'+
'-----END PUBLIC KEY-----';
console.log(event["pk"]);
const table = "MARKETPLACE_USER_RELATED_APPS_TABLE";
var dynamo = new Dynamo(table);
var prik = await dynamo.read(dynamo.client());
console.log(prik);
var clientid = '12345'; //event['client_id']
var unique_id = parseInt(Math.random()*10000000000000000).toString();
var time_stamp = new Date();
var Authkey = clientid + ':' + unique_id + ':' + time_stamp;
console.log(Authkey+"Authkey");
console.log(time_stamp + 'time_stamp');
var cryp = new CryptoVerify(publick,privatek, Authkey);
console.log(cryp.hash+'hash key');
console.log(event['pk'])
const response = {
statusCode: 200,
body: cryp.verify(cryp.signature())
};
return response
};
function CryptoVerify(publickey, privatekey, authkey)
{
this.crypto = require ('crypto')
this.hash = this.crypto.createHash('sha256')
.update(authkey)
.digest('base64');
console.log(this.hash+'haskkk');
this.signature = function(){
try{
var signer = this.crypto.createSign('sha1');
signer.update(this.hash);
var sign = signer.sign(privatekey,'base64');
console.log(sign + "Sign")
}
catch(e)
{
var sign = "Private key Errpr: Not matching with the Public
key\nExact " +e;
}
return sign;
}
this.verify = function(sign){
var verifier = this.crypto.createVerify('sha1');
verifier.update(this.hash);
try {
this.ver = verifier.verify(publickey, sign,'base64');
console.log(this.ver);
}
catch (e){
console.log("Private key Error: Not matching with the Public
key/n Exact Error:"+e);
this.ver = e
}
return this.ver;
}
}
function Dynamo(table)
{
this.params = {Key :{"email_id":{S:"testuser#tml.com"},
"app_name":{S:"EDH"}},
TableName: table};
this.op = {};
this.client = function(){
try{
var dynamodb = new AWS.DynamoDB();
}
catch(e){
console.log("Error: Class DynamoDB is not defined");
}
return dynamodb;};
this.read = function(dy){
return new Promise((resolve, reject) => {
try{
dy.getItem(this.params, function(err, data){
if(err) console.log(err, err.stack);
else {
this.op = data;
console.log(this.op);
return resolve(data);
}
});
}
catch(e){
console.log('Please check getIten function to clear the error');
reject('Error')
}
});
//return this.op;
};
}
Thanks all. I have got the solution to the question by searching more on the internet. AWS itself is providing "Promise" method for every service.
I just included promise function as follows:
dynamodb.getItem(params).promise().then(function(data, err){
if(err)
//do something
else
//do something
});

Decrypting multiple env. variables in AWS Lambda

I've got a number of encrypted environmental variables I need to decrypt in an AWS Lambda function. They give an example bit of code, but I'd rather not run a huge chunk for each value I need to decrypt:
const AWS = require('aws-sdk');
const encrypted = process.env['my_password'];
let decrypted;
function processEvent(event, context, callback) {
// TODO handle the event here
}
exports.handler = (event, context, callback) => {
if (decrypted) {
processEvent(event, context, callback);
} else {
// Decrypt code should run once and variables stored outside of the function
// handler so that these are decrypted once per container
const kms = new AWS.KMS();
kms.decrypt({ CiphertextBlob: new Buffer(encrypted, 'base64') }, (err, data) => {
if (err) {
console.log('Decrypt error:', err);
return callback(err);
}
decrypted = data.Plaintext.toString('ascii');
processEvent(event, context, callback);
});
}
};
I'm wondering if the AWS SDK includes a function that lets me decrypt multiple values at once. Failing that, is there a way to elegantly chain these calls together so they don't take up ~75 lines of my otherwise simple function?
You can use promises to achieve this. See the example below for decrypting both a username and password via KMS. You can add as many additional decryption promises to the decryptPromises array as you'd like:
const AWS = require('aws-sdk');
const encrypted = {
username: process.env.username,
password: process.env.password
};
let decrypted = {};
function processEvent(event, context, callback) {
//do work
}
exports.handler = (event, context, callback) => {
if ( decrypted.username && decrypted.password ) {
processEvent(event, context, callback);
} else {
const kms = new AWS.KMS();
const decryptPromises = [
kms.decrypt( { CiphertextBlob: new Buffer(encrypted.username, 'base64') } ).promise(),
kms.decrypt( { CiphertextBlob: new Buffer(encrypted.password, 'base64') } ).promise()
];
Promise.all( decryptPromises ).then( data => {
decrypted.username = data[0].Plaintext.toString('ascii');
decrypted.password = data[1].Plaintext.toString('ascii');
processEvent(event, context, callback);
}).catch( err => {
console.log('Decrypt error:', err);
return callback(err);
});
}
};
You can find more information on how promises have been implimented for the AWS SDK in the Support for Promises in the SDK documentation.
I created a class to decrypt variables in amazon lambda. It uses async await instead of Promises.all. You do not need to import lodash library. You can modifiy the bellow class to not use it (use forEach instead).
var _ = require('lodash/core');
const AWS = require('aws-sdk');
class EnvVarsDecryptor {
constructor(encryptedVariables) {
this.encryptedVariables = encryptedVariables;
this.decrypted = {};
}
isDecrypted() {
return _.every(this.encryptedVariables, (e) => this.decrypted[e] != undefined && this.decrypted[e] != null);
}
async decryptVars() {
const kms = new AWS.KMS();
try {
for ( let index = 0; index < this.encryptedVariables.length; index++) {
const encrypted = this.encryptedVariables[index];
const data = await kms.decrypt({CiphertextBlob: new Buffer(process.env[encrypted], 'base64') }).promise();
this.decrypted[encrypted] = data.Plaintext.toString('ascii');
}
} catch( e) {
console.error(e);
}
return this.decrypted;
}
}
module.exports = EnvVarsDecryptor;
This is a sample illustrating how using the function:
exports.handler = async (event) => {
if (!decryptor.isDecrypted()) {
await decryptor.decryptVars();
}
console.log(decryptor.decrypted);
return `Successfully processed ${event.Records.length} messages.`;
};

how to upload base64 data as image to s3 using node js?

I am sending base64data of canvas to node.js script. I need the base64data to be stored as an image to the s3bucket. Is there any way to achieve it?
Store your Data URI in a variable.
Create function which decodes your data URI(64 bit encoded string) to string(Here I have created dataURItoBlob() function) and after decoding return the string.
Pass that string to in body of S3 upload function.
var myDataUri = "data:image/jpg;base64,JVBERi0xLjMKMyAwIG9iago8PC9UeXBlIC9QYW..."
var myFile=dataURItoBlob(myDataUri);
function dataURItoBlob(dataURI) {
var binary = atob(dataURI.split(',')[1]);
var array = [];
for (var i = 0; i < binary.length; i++) {
array.push(binary.charCodeAt(i));
}
return new Blob([new Uint8Array(array)], {
type: 'image/jpg'
});
}
if (myFile)) {
results.innerHTML = '';
var params = {
Key: fileName+'.jpg',
ContentType: 'image/jpg',
Body: myFile
};
bucket.upload(params, function(err, data) {
results.innerHTML = err ? 'ERROR!' : 'UPLOADED.: ' + file;
});
} else {
results.innerHTML = 'Nothing to upload.';
}
you can send base64 data with AWS putObject method as follows
var AWS = require('aws-sdk');
AWS.config.loadFromPath('./s3_config.json');
var s3Bucket = new AWS.S3( { params: {Bucket: 'myBucket'} } );
var imageBase64Data='Your base64 code '
s3Bucket.putObject(imageBase64Data, function(err, data){
if (err) {
console.log(err);
console.log('Error uploading data: ', data);
} else {
console.log('succesfully uploaded the image!');
}
});

Node.js & Amazon S3: How to iterate through all files in a bucket?

Is there any Amazon S3 client library for Node.js that allows listing of all files in S3 bucket?
The most known aws2js and knox don't seem to have this functionality.
Using the official aws-sdk:
var allKeys = [];
function listAllKeys(marker, cb)
{
s3.listObjects({Bucket: s3bucket, Marker: marker}, function(err, data){
allKeys.push(data.Contents);
if(data.IsTruncated)
listAllKeys(data.NextMarker, cb);
else
cb();
});
}
see s3.listObjects
Edit 2017:
Same basic idea, but listObjectsV2( ... ) is now recommended and uses a ContinuationToken (see s3.listObjectsV2):
var allKeys = [];
function listAllKeys(token, cb)
{
var opts = { Bucket: s3bucket };
if(token) opts.ContinuationToken = token;
s3.listObjectsV2(opts, function(err, data){
allKeys = allKeys.concat(data.Contents);
if(data.IsTruncated)
listAllKeys(data.NextContinuationToken, cb);
else
cb();
});
}
Using AWS-SDK v3 and Typescript
import {
paginateListObjectsV2,
S3Client,
S3ClientConfig,
} from '#aws-sdk/client-s3';
/* // For Deno
import {
paginateListObjectsV2,
S3Client,
S3ClientConfig,
} from "https://deno.land/x/aws_sdk#v3.32.0-1/client-s3/mod.ts"; */
const s3Config: S3ClientConfig = {
credentials: {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
region: 'us-east-1',
};
const getAllS3Files = async (client: S3Client, s3Opts) => {
const totalFiles = [];
for await (const data of paginateListObjectsV2({ client }, s3Opts)) {
totalFiles.push(...(data.Contents ?? []));
}
return totalFiles;
};
const main = async () => {
const client = new S3Client(s3Config);
const s3Opts = { Bucket: 'bucket-xyz' };
console.log(await getAllS3Files(client, s3Opts));
};
main();
For AWS-SDK v2 Using Async Generator
Import S3
const { S3 } = require('aws-sdk');
const s3 = new S3();
create a generator function to retrieve all the files list
async function* listAllKeys(opts) {
opts = { ...opts };
do {
const data = await s3.listObjectsV2(opts).promise();
opts.ContinuationToken = data.NextContinuationToken;
yield data;
} while (opts.ContinuationToken);
}
Prepare aws parameter, based on api docs
const opts = {
Bucket: 'bucket-xyz' /* required */,
// ContinuationToken: 'STRING_VALUE',
// Delimiter: 'STRING_VALUE',
// EncodingType: url,
// FetchOwner: true || false,
// MaxKeys: 'NUMBER_VALUE',
// Prefix: 'STRING_VALUE',
// RequestPayer: requester,
// StartAfter: 'STRING_VALUE'
};
Use generator
async function main() {
// using for of await loop
for await (const data of listAllKeys(opts)) {
console.log(data.Contents);
}
}
main();
thats it
Or Lazy Load
async function main() {
const keys = listAllKeys(opts);
console.log(await keys.next());
// {value: {…}, done: false}
console.log(await keys.next());
// {value: {…}, done: false}
console.log(await keys.next());
// {value: undefined, done: true}
}
main();
Or Use generator to make Observable function
const lister = (opts) => (o$) => {
let needMore = true;
const process = async () => {
for await (const data of listAllKeys(opts)) {
o$.next(data);
if (!needMore) break;
}
o$.complete();
};
process();
return () => (needMore = false);
};
use this observable function with RXJS
// Using Rxjs
const { Observable } = require('rxjs');
const { flatMap } = require('rxjs/operators');
function listAll() {
return Observable.create(lister(opts))
.pipe(flatMap((v) => v.Contents))
.subscribe(console.log);
}
listAll();
or use this observable function with Nodejs EventEmitter
const EventEmitter = require('events');
const _eve = new EventEmitter();
async function onData(data) {
// will be called for each set of data
console.log(data);
}
async function onError(error) {
// will be called if any error
console.log(error);
}
async function onComplete() {
// will be called when data completely received
}
_eve.on('next', onData);
_eve.on('error', onError);
_eve.on('complete', onComplete);
const stop = lister(opts)({
next: (v) => _eve.emit('next', v),
error: (e) => _eve.emit('error', e),
complete: (v) => _eve.emit('complete', v),
});
Here's Node code I wrote to assemble the S3 objects from truncated lists.
var params = {
Bucket: <yourbucket>,
Prefix: <yourprefix>,
};
var s3DataContents = []; // Single array of all combined S3 data.Contents
function s3Print() {
if (program.al) {
// --al: Print all objects
console.log(JSON.stringify(s3DataContents, null, " "));
} else {
// --b: Print key only, otherwise also print index
var i;
for (i = 0; i < s3DataContents.length; i++) {
var head = !program.b ? (i+1) + ': ' : '';
console.log(head + s3DataContents[i].Key);
}
}
}
function s3ListObjects(params, cb) {
s3.listObjects(params, function(err, data) {
if (err) {
console.log("listS3Objects Error:", err);
} else {
var contents = data.Contents;
s3DataContents = s3DataContents.concat(contents);
if (data.IsTruncated) {
// Set Marker to last returned key
params.Marker = contents[contents.length-1].Key;
s3ListObjects(params, cb);
} else {
cb();
}
}
});
}
s3ListObjects(params, s3Print);
Pay attention to listObject's documentation of NextMarker, which is NOT always present in the returned data object, so I don't use it at all in the above code ...
NextMarker — (String) When response is truncated (the IsTruncated
element value in the response is true), you can use the key name in
this field as marker in the subsequent request to get next set of
objects. Amazon S3 lists objects in alphabetical order Note: This
element is returned only if you have delimiter request parameter
specified. If response does not include the NextMarker and it is
truncated, you can use the value of the last Key in the response as
the marker in the subsequent request to get the next set of object
keys.
The entire program has now been pushed to https://github.com/kenklin/s3list.
In fact aws2js supports listing of objects in a bucket on a low level via s3.get() method call. To do it one has to pass prefix parameter which is documented on Amazon S3 REST API page:
var s3 = require('aws2js').load('s3', awsAccessKeyId, awsSecretAccessKey);
s3.setBucket(bucketName);
var folder = encodeURI('some/path/to/S3/folder');
var url = '?prefix=' + folder;
s3.get(url, 'xml', function (error, data) {
console.log(error);
console.log(data);
});
The data variable in the above snippet contains a list of all objects in the bucketName bucket.
Published knox-copy when I couldn't find a good existing solution. Wraps all the pagination details of the Rest API into a familiar node stream:
var knoxCopy = require('knox-copy');
var client = knoxCopy.createClient({
key: '<api-key-here>',
secret: '<secret-here>',
bucket: 'mrbucket'
});
client.streamKeys({
// omit the prefix to list the whole bucket
prefix: 'buckets/of/fun'
}).on('data', function(key) {
console.log(key);
});
If you're listing fewer than 1000 files a single page will work:
client.listPageOfKeys({
prefix: 'smaller/bucket/o/fun'
}, function(err, page) {
console.log(page.Contents); // <- Here's your list of files
});
Meekohi provided a very good answer, but the (new) documentation states that NextMarker can be undefined. When this is the case, you should use the last key as the marker.
So his codesample can be changed into:
var allKeys = [];
function listAllKeys(marker, cb) {
s3.listObjects({Bucket: s3bucket, Marker: marker}, function(err, data){
allKeys.push(data.Contents);
if(data.IsTruncated)
listAllKeys(data.NextMarker || data.Contents[data.Contents.length-1].Key, cb);
else
cb();
});
}
Couldn't comment on the original answer since I don't have the required reputation. Apologies for the bad mark-up btw.
I am using this version with async/await.
This function will return the content in an array.
I'm also using the NextContinuationToken instead of the Marker.
async function getFilesRecursivelySub(param) {
// Call the function to get list of items from S3.
let result = await s3.listObjectsV2(param).promise();
if(!result.IsTruncated) {
// Recursive terminating condition.
return result.Contents;
} else {
// Recurse it if results are truncated.
param.ContinuationToken = result.NextContinuationToken;
return result.Contents.concat(await getFilesRecursivelySub(param));
}
}
async function getFilesRecursively() {
let param = {
Bucket: 'YOUR_BUCKET_NAME'
// Can add more parameters here.
};
return await getFilesRecursivelySub(param);
}
This is an old question and I guess the AWS JS SDK has changed a lot since it was asked. Here's yet another way to do it these days:
s3.listObjects({Bucket:'mybucket', Prefix:'some-pfx'}).
on('success', function handlePage(r) {
//... handle page of contents r.data.Contents
if(r.hasNextPage()) {
// There's another page; handle it
r.nextPage().on('success', handlePage).send();
} else {
// Finished!
}
}).
on('error', function(r) {
// Error!
}).
send();
If you want to get list of keys only within specific folder inside a S3 Bucket then this will be useful.
Basically, listObjects function will start searching from the Marker we set and it will search until maxKeys: 1000 as limit. so it will search one by one folder and get you first 1000 keys it find from different folder in a bucket.
Consider i have many folders inside my bucket with prefix as prod/some date/, Ex: prod/2017/05/12/ ,prod/2017/05/13/,etc.
I want to fetch list of objects (file names) only within prod/2017/05/12/ folder then i will specify prod/2017/05/12/ as my start and prod/2017/05/13/ [your next folder name] as my end and in code i'm breaking the loop when i encounter the end.
Each Keyin data.Contents will look like this.
{ Key: 'prod/2017/05/13/4bf2c675-a417-4c1f-a0b4-22fc45f99207.jpg',
LastModified: 2017-05-13T00:59:02.000Z,
ETag: '"630b2sdfsdfs49ef392bcc16c833004f94ae850"',
Size: 134236366,
StorageClass: 'STANDARD',
Owner: { }
}
Code:
var list = [];
function listAllKeys(s3bucket, start, end) {
s3.listObjects({
Bucket: s3bucket,
Marker: start,
MaxKeys: 1000,
}, function(err, data) {
if (data.Contents) {
for (var i = 0; i < data.Contents.length; i++) {
var key = data.Contents[i].Key; //See above code for the structure of data.Contents
if (key.substring(0, 19) != end) {
list.push(key);
} else {
break; // break the loop if end arrived
}
}
console.log(list);
console.log('Total - ', list.length);
}
});
}
listAllKeys('BucketName', 'prod/2017/05/12/', 'prod/2017/05/13/');
Output:
[ 'prod/2017/05/12/05/4bf2c675-a417-4c1f-a0b4-22fc45f99207.jpg',
'prod/2017/05/12/05/a36528b9-e071-4b83-a7e6-9b32d6bce6d8.jpg',
'prod/2017/05/12/05/bc4d6d4b-4455-48b3-a548-7a714c489060.jpg',
'prod/2017/05/12/05/f4b8d599-80d0-46fa-a996-e73b8fd0cd6d.jpg',
... 689 more items ]
Total - 692
I ended up building a wrapper function around ListObjectsV2, works the same way and takes the same parameters but works recursively until IsTruncated=false and returns all the keys found as an array in the second parameter of the callback function
const AWS = require('aws-sdk')
const s3 = new AWS.S3()
function listAllKeys(params, cb)
{
var keys = []
if(params.data){
keys = keys.concat(params.data)
}
delete params['data']
s3.listObjectsV2(params, function(err, data){
if(err){
cb(err)
} else if (data.IsTruncated) {
params['ContinuationToken'] = data.NextContinuationToken
params['data'] = data.Contents
listAllKeys(params, cb)
} else {
keys = keys.concat(data.Contents)
cb(null,keys)
}
})
}
Here's what I came up with based on the other answers.
You can await listAllKeys() without having to use callbacks.
const listAllKeys = () =>
new Promise((resolve, reject) => {
let allKeys = [];
const list = marker => {
s3.listObjects({ Marker: marker }, (err, data) => {
if (err) {
reject(err);
} else if (data.IsTruncated) {
allKeys.push(data.Contents);
list(data.NextMarker || data.Contents[data.Contents.length - 1].Key);
} else {
allKeys.push(data.Contents);
resolve(allKeys);
}
});
};
list();
});
This assumes you've initialized the s3 variable like so
const s3 = new aws.S3({
apiVersion: API_VERSION,
params: { Bucket: BUCKET_NAME }
});
I made it as simple as possible. You can iterate uploading objects using for loop, it is quite simple, neat and easy to understand.
package required: fs, express-fileupload
server.js :-
router.post('/upload', function(req, res){
if(req.files){
var file = req.files.filename;
test(file);
res.render('test');
}
} );
test function () :-
function test(file){
// upload all
if(file.length){
for(var i =0; i < file.length; i++){
fileUP(file[i]);
}
}else{
fileUP(file);
}
// call fileUP() to upload 1 at once
function fileUP(fyl){
var filename = fyl.name;
var tempPath = './temp'+filename;
fyl.mv(tempPath, function(err){
fs.readFile(tempPath, function(err, data){
var params = {
Bucket: 'BUCKET_NAME',
Body: data,
Key: Date.now()+filename
};
s3.upload(params, function (err, data) {
if (data) {
fs.unlink(tempPath, (err) => {
if (err) {
console.error(err)
return
}
else{
console.log("file removed from temp loaction");
}
});
console.log("Uploaded in:", data.Location);
}
});
});
});
}
}
This should work,
var listAllKeys = async function (token) {
if(token) params.ContinuationToken = token;
return new Promise((resolve, reject) => {
s3.listObjectsV2(params, function (err, data) {
if (err){
reject(err)
}
resolve(data)
});
});
}
var collect_all_files = async function () {
var allkeys = []
conti = true
token = null
while (conti) {
data = await listAllKeys(token)
allkeys = allkeys.concat(data.Contents);
token = data.NextContinuationToken
conti = data.IsTruncated
}
return allkeys
};
Using the new API s3.listObjectsV2 the recursive solution will be:
S3Dataset.prototype.listFiles = function(params,callback) {
var self=this;
var options = {
};
for (var attrname in params) { options[attrname] = params[attrname]; }
var results=[];
var s3=self.s3Store.GetInstance();
function listAllKeys(token, callback) {
var opt={ Bucket: self._options.s3.Bucket, Prefix: self._options.s3.Key, MaxKeys: 1000 };
if(token) opt.ContinuationToken = token;
s3.listObjectsV2(opt, (error, data) => {
if (error) {
if(self.logger) this.logger.error("listFiles error:", error);
return callback(error);
} else {
for (var index in data.Contents) {
var bucket = data.Contents[index];
if(self.logger) self.logger.debug("listFiles Key: %s LastModified: %s Size: %s", bucket.Key, bucket.LastModified, bucket.Size);
if(bucket.Size>0) {
var Bucket=self._options.s3.Bucket;
var Key=bucket.Key;
var components=bucket.Key.split('/');
var name=components[components.length-1];
results.push({
name: name,
path: bucket.Key,
mtime: bucket.LastModified,
size: bucket.Size,
sizehr: formatSizeUnits(bucket.Size)
});
}
}
if( data.IsTruncated ) { // truncated page
return listAllKeys(data.NextContinuationToken, callback);
} else {
return callback(null,results);
}
}
});
}
return listAllKeys.apply(this,['',callback]);
};
where
function formatSizeUnits(bytes){
if (bytes>=1099511627776) {bytes=(bytes/1099511627776).toFixed(4)+' PB';}
else if (bytes>=1073741824) {bytes=(bytes/1073741824).toFixed(4)+' GB';}
else if (bytes>=1048576) {bytes=(bytes/1048576).toFixed(4)+' MB';}
else if (bytes>=1024) {bytes=(bytes/1024).toFixed(4)+' KB';}
else if (bytes>1) {bytes=bytes+' bytes';}
else if (bytes==1) {bytes=bytes+' byte';}
else {bytes='0 byte';}
return bytes;
}//formatSizeUnits
Although #Meekohi's answer does technically work, I've had enough heartache with the S3 portion of the AWS SDK for NodeJS. After all the previous struggling with modules such as aws-sdk, s3, knox, I decided to install s3cmd via the OS package manager and shell-out to it using child_process
Something like:
var s3cmd = new cmd_exec('s3cmd', ['ls', filepath, 's3://'+inputBucket],
function (me, data) {me.stdout += data.toString();},
function (me) {me.exit = 1;}
);
response.send(s3cmd.stdout);
(Using the cmd_exec implementation from this question)
This approach just works really well - including for other problematic things like file upload.
The cleanest way to do it for me was through execution of s3cmd from my node script like this (The example here is to delete files recursively):
var exec = require('child_process').exec;
var child;
var bucket = "myBucket";
var prefix = "myPrefix"; // this parameter is optional
var command = "s3cmd del -r s3://" + bucket + "/" + prefix;
child = exec(command, {maxBuffer: 5000 * 1024}, function (error, stdout, stderr) { // the maxBuffer is here to avoid the maxBuffer node process error
console.log('stdout: ' + stdout);
if (error !== null) {
console.log('exec error: ' + error);
}
});

Resources