I'm trying to use AWS Secrets Manager to fetch my RDS credentials,
The Secrets Manager SDK is able to get the Secret properly,
But I am unable to export it back to my calling file.
I have 2 files -
1. index.js -
var mysql = require('mysql');
var secretsManager = require('./secrets-manager');
exports.handler = (event, context, callback) => {
secretsManager.getDbCredentialFromSecretsManager(function(err,creds) {
if (err) {
console.log(err);
callback(err, null);
}
else{
console.log("Creds ", creds);
var connection = mysql.createConnection(creds);
connection.connect(function(err) {
if (err) {
console.error(err.stack);
callback(err,null);
}
else{
callback(null,connection);
}
});
}
});
}
2. secrets-manager.js -
var AWS = require('aws-sdk');
var constants = require('/opt/nodejs/utils/constants');
module.exports = {
getRDSCredsFromSM
};
function getRDSCredsFromSM (callback) {
var response = {};
let secretName = "secretId";
var client = new AWS.SecretsManager({
region: constants.aws.region
});
client.getSecretValue({SecretId: secretName}, function(err, data) {
if (err) {
console.log(err);
callback(err, null);
}
else {
if ('SecretString' in data) {
let secret = data.SecretString;
secret = JSON.parse(secret);
console.log("secret",secret);
callback(null, secret);
} else {
let buff = new Buffer(data.SecretBinary, 'base64');
let decodedBinarySecret = buff.toString('ascii');
callback(null, decodedBinarySecret);
}
}
});
}
I feel there's some mistake from me on Node side,
Which is why the callback isn't working properly,
The Lambda Timesout,
And the logs show nothing in creds variable -
console.log("Creds ", creds);
Working code -
let async = require('async');
let AWS = require('aws-sdk');
module.exports = {
getDbCredentialFromSecretsManager
};
const TAG = '[SECRETS-MANAGER-UTIL->';
function getDbCredentialFromSecretsManager (constants, callback) {
let response = {};
const METHOD_TAG = TAG + 'getDbCredentialFromSecretsManager->';
async.waterfall([
function(callback) {
let client = new AWS.SecretsManager({
region: constants.aws.region
});
client.getSecretValue({SecretId: constants.aws.sm}, function(err, data) {
if (err) {
console.log(METHOD_TAG,err);
callback(err, null);
}
else {
console.log(METHOD_TAG, 'Secrets Manager call successful');
if ('SecretString' in data) {
let secret = data.SecretString;
secret = JSON.parse(secret);
response.user = secret.username;
response.password = secret.password;
response.host = secret.host;
response.database = constants.db.database;
callback(null, response);
} else {
let buff = new Buffer(data.SecretBinary, 'base64');
let decodedBinarySecret = buff.toString('ascii');
callback(null, decodedBinarySecret);
}
}
});
}
],
function(err, response) {
if (err) {
console.log(METHOD_TAG, err);
callback(err, response);
}
else {
callback(null, response);
}
});
}
Related
I have a lambda function written in node.js. The lambda logic is to extract secrets from aws secret manager and pass the params to another module with a token to make a soap call.This lambda was working as expected without async method to get secreats when secreats hardcoded.
But after adding an async getsecrets method the lambda works intermittently while testing in aws console. The getsecreats returns params every time. But the lambda terminates without the intended output. It doesn't make any soap call.
The logic of the lambda code is
Get the secret
Pass the secret to the CallServicewithToken()
get XML data from soap and populate it into the database.
Why it works intermittently when introducing async calls? I have tried introducing async to all methods. still the same issue. Appreciate any input/help.
The code is as below
'use strict';
var soap = require('strong-soap').soap;
const aws = require("aws-sdk");
const request = require('request');
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
var rfcUrl = process.env.rfcser_url;
var testUrl = process.env.faccser_url;
var region = process.env.region;
var secretName = process.env.secretName;
var oauthurl = process.env.oauthurl;
var rfcRequestArgs;
var parsedResult;
var tokenrequest;
exports.handler = async function(event, context,callback) {
const secret = await getSecrets();
CallServicewithToken();
};
async function getSecrets() {
const config = { region : region, apiVersion: 'latest' };
let secretManager = new aws.SecretsManager(config);
const Result = await secretManager.getSecretValue({ SecretId: secretName }).promise();
parsedResult = JSON.parse(Result.SecretString);
tokenrequest = {
url: oauthurl,
form: {
client_id: parsedResult.client_id,
client_secret: parsedResult.client_secret,
grant_type:'client_credentials',
scope:parsedResult.scope
}
};
console.log('client_id: ' + parsedResult.client_id);
console.log('client_secret: ' + parsedResult.client_secret);
console.log('testservice_userid: ' + parsedResult.testservice_userid);
}
function CallServicewithToken() {
console.log('Calling CallServicewithToken ');
request.post(tokenrequest, (err, res, body) => {
if (err) {
console.log(' error2: ' + err);
return;
}
rfcRequestArgs = {
UserName: parsedResult.service_username
};
var tokenobj = JSON.parse(body);
var token = 'Bearer '+ tokenobj.access_token;
var credentials = {
Authorization:{
AuthToken: token
}
}
var options = {};
console.log('Calling Service.');
soap.createClient(rfcUrl, options, function(err, client) {
client.addSoapHeader(
`<aut:Authorization xmlns:aut="http://soap.xyznet.net">
<aut:AuthToken>${token}</aut:AuthToken>
</aut:Authorization>`
);
var method = client['GetSourceLocationData'];
method(RequestArgs, function(err, result, envelope, soapHeader) {
if (err) {
console.log('error3: ' + err);
return;
}
else
{
console.log('Received response from GetSourceLocationData().');
CallTESTService(JSON.stringify(result));
}
});
function CallTESTService(LocData)
{
var testRequestArgs = {
UserID: parsedResult.testservice_userid,
AuthorizationKey: parsedResult.testservice_authorizationkey,
LocationData: LocData
};
console.log('Calling test Service.');
options = {};
soap.createClient(testUrl, options, function(err, client) {
client.addSoapHeader(
`<Authorization xmlns="testWebService">
<AuthToken>${token}</AuthToken>
</Authorization>`
);
var test_method = client['UpdateLocationData'];
console.log('Called UpdateLocationData service method.');
test_method(testRequestArgs, function(err, result, envelope, soapHeader) {
if(err) {
console.log('test error: ' + err);
return;
}
else
{
console.log('Response: \n' + JSON.stringify(result));
console.log('Data updated through test service method.');
}
});
});
}
});
});
}
I'm trying to access user attributes in lambda function. Basically I need email of listUsersRes.Users. I'm trying through listUsersRes.Users[0].Email but it is returning undefined.
const aws = require("aws-sdk");
exports.handler = async (event, context, callback) => {
const cognitoProvider = new aws.CognitoIdentityServiceProvider({apiVersion: "2016-04-18"});
if (event.triggerSource == "PreSignUp_SignUp" ||event.triggerSource == "PreSignUp_AdminCreateUser" || event.triggerSource=="PreSignUp_ExternalProvider") {
try {
const listUserParams={UserPoolId: event.userPoolId,AttributesToGet: ["email"],Filter: `cognito:user_status= \"${"UNCONFIRMED"}\"`, Limit: 10 };
const listUsersRes = await cognitoProvider.listUsers(listUserParams).promise();
if (listUsersRes.Users.length >= 0) {
var params = {
Username: listUsersRes.Users[0].Username,
UserPoolId: event.userPoolId};
await cognitoProvider.adminGetUser(params, function(err,data){
if(err) callback(new Error(err), event);
else {
callback(new Error(data), event);
}
});
// callback(new Error(listUsersRes.Users[0].Username),event);
}else {callback(new Error("Something went wrong!"), event);}
}
catch (error) {return callback(new Error("catch error"), event);}
}
else {callback(new Error("This provider is not supported"), event);}
};
I need email of listUsers am I doing right?
I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});
I am trying to get MedGuideURL to be used in the 2nd callback but it's value is empty.It seems the second callback is always happening before the first one is done. I am thinking of using Promise/Observable but is there an easier way?
var qrImage = require('qr-image');
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({region:'us-west-2'});
exports.handler = function(event, context, callback){
var path = event.path;
var drugId = path.replace(/\//g, '');
var MedGuideURL = "";
var params = {
TableName: 'QRCodeInfo',
Key: {
"DrugId" : drugId
}
};
docClient.get(params, function(err,data) {
if (err) {
callback(err,null);
} else {
console.log("The data is: "+ data.Item.MedGuideURL); //correct value
callback(null,data);
MedGuideURL = data.Item.MedGuideURL;
}
});
callback(null, sendRes(200, MedGuideURL)); //MedGuideURL is empty!
};
const sendRes = (status, body) => {
//console.log(body);
const svg_string = qrImage.imageSync(body, { type: 'svg', size: 10 });
var response = {
statusCode: status,
headers: {
"Content-Type": "image/svg+xml"
},
body: svg_string
};
return response;
};
You need to first understand async behavior.
You have doing callback before method get complete
You can change get with promise
Example
if sendResis promise method
exports.handler = function (event, context, callback) {
var path = event.path;
var drugId = path.replace(/\//g, '');
var MedGuideURL = "";
var params = {
TableName: 'QRCodeInfo',
Key: {
"DrugId": drugId
}
};
docClient.get(params).promise().then((data) => {
console.log('success' + x);
console.log("The data is: " + data.Item.MedGuideURL);
MedGuideURL = data.Item.MedGuideURL;
return sendRes(200, MedGuideURL); //if sendResis promise method
}).then((finalResponse) => {
callback(null, finalResponse);
})
.catch((err) => {
callback(err, null);
})
};
IF sendResis is not promise :
exports.handler = function (event, context, callback) {
var path = event.path;
var drugId = path.replace(/\//g, '');
var MedGuideURL = "";
var params = {
TableName: 'QRCodeInfo',
Key: {
"DrugId": drugId
}
};
docClient.get(params).promise().then((data) => {
console.log('success' + x);
console.log("The data is: " + data.Item.MedGuideURL);
MedGuideURL = data.Item.MedGuideURL;
let finalResponse = sendRes(200, MedGuideURL); //if sendResis not promise method
return callback(null, finalResponse);
})
.catch((err) => {
callback(err, null);
})
};
This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 4 years ago.
Can someone give me a hand? I tried to figure it out but i ran out of ideas.
-------------dynamo.js....
module.exports.readUser = function (user_id) {
AWS.config = new AWS.Config();
AWS.config.update({region: "eu-west-1"});
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "user";
var user_id = user_id;
var params = {
TableName: table,
Key:{
"user_id": user_id
}
};
docClient.get(params, function(err, data) {
if (err) {
return err;
} else {
//console.log(data); <-- data is filled
return data;
}
});
var dynamo = require("./dynamo.js");
console.log(dynamo.readUser(4711998));
The data you log out inside the function is only the returned value for the anonymous function given to docClient.get as the second argument. The exported readUser function has no return statement and therefore it is undefined.
Because of the async nature of javascript you have to access the result via a callback like so:
module.exports.readUser = function (user_id, cb) {
...
docClient.get(params, function(err, data) {
if (err) {
return err;
} else {
return cb(data);
}
});
}
var dynamo = require("./dynamo.js");
dynamo.readUser(4711998, function (user) {
console.log(user);
});
docClient.get is an asynchronous function. You need promisify this function or use callback.
Use callback:
module.exports.readUser = function (user_id, cb) {
AWS.config = new AWS.Config();
AWS.config.update({ region: "eu-west-1" });
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "user";
var user_id = user_id;
var params = {
TableName: table,
Key: {
"user_id": user_id
}
};
docClient.get(params, cb);
}
var dynamo = require("./dynamo.js");
dynamo.readUser(4711998, function (err, user) {
if(err)
console.error(err);
else
console.log(user);
});
Use promise:
module.exports.readUser = function (user_id) {
AWS.config = new AWS.Config();
AWS.config.update({ region: "eu-west-1" });
var docClient = new AWS.DynamoDB.DocumentClient();
var table = "user";
var user_id = user_id;
var params = {
TableName: table,
Key: {
"user_id": user_id
}
};
return new Promise(function(resolve, reject) {
docClient.get(params, function(err, data) {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
}
var dynamo = require("./dynamo.js");
dynamo.readUser(4711998).then(function (user) {
console.log(user);
}).catch(function(err) {
console.error(err);
});
Read more:
How do I promisify the AWS JavaScript SDK?
https://aws.amazon.com/ru/blogs/developer/support-for-promises-in-the-sdk/