Retrieve AWS ssm parameter in bulk - node.js

How can I retrieve parameters from AWS Systems Manager (parameter store) in bulk (or more than one parameter) at a time? Using aws-sdk, following is the Node.js code I have written to retrieve SSM parameter from parameter store:
const ssm = new (require('aws-sdk/clients/ssm'))()
const getSSMKey = async params => {
const {Parameter: {Value: APIKey}} = await ssm.getParameter(params).promise()
return APIKey
}
const [param1, param2, param3] = await Promise.all([
getSSMKey({ Name: '/data/param/PARAM1', WithDecryption: true }),
getSSMKey({ Name: '/data/param/PARAM2', WithDecryption: true }),
getSSMKey({ Name: '/data/param/PARAM3', WithDecryption: true })
])
console.log(param1, param2, param3)
But with this code, I am sending 3 request for getting 3 parameters which is inefficient in case of large number of parameters. Is there any way to retrieve more than one parameters in one request. if ssm.getParameters() is the method to do that then please give an example (particularly parameter to that method). I tried but I receive nothing.

According to the AWS document, GetParameter gets the value for one parameter, whereas GetParameters gets the value for multiple.
Their usages are very similar too. When using GetParameters to get multiple values, pass in multiple names as a list for Names, instead of passing a single name as string for Name.
Code sample, to get parameters named "foo" and "bar", in "us-west-1" region:
const AWS = require('aws-sdk');
AWS.config.update({ region: "us-west-1" });
const SSM = require('aws-sdk/clients/ssm');
const ssm = new SSM()
const query = {
"Names": ["foo", "bar"],
"WithDecryption": true
}
let param = ssm.getParameters(query, (err, data) => {
console.log('error = %o', err);
console.log('raw data = %o', data);
})

At last it worked for me. Following is the code:
const ssmConfig = async () => {
const data = await ssm.getParameters({ Names: ['/data/param/PARAM1', '/data/param/PARAM2', '/bronto/rest//data/param/PARAM3'],
WithDecryption: true }).promise()
const config = {}
for (const i of data.Parameters) {
if (i.Name === '/data/param/PARAM1') {
config.param1 = i.Value
}
if (i.Name === '/data/param/PARAM2') {
config.rest.clientId param2 = i.Value
}
if (i.Name === '/data/param/PARAM3') {
config.param3 = i.Value
}
}
return config
}

This is what I did to retrieve all the parameters from a specific path.
**your SSM function client :**
'use strict';
const SSM = require('aws-sdk/clients/ssm');
let ssmclient;
module.exports.init = () => {
const region = process.env.REGION === undefined ? 'us-east-1' : process.env.REGION ;
ssmclient = new SSM({region: region});
}
module.exports.getParameters = async (path) => {
try {
let params = {
Path: path,
WithDecryption: true
};
let allParameters = [];
let data = await ssmclient.getParametersByPath(params).promise();
allParameters.push.apply(allParameters, data.Parameters);
while(data.NextToken) {
params.NextToken = data.NextToken;
data = await ssmclient.getParametersByPath(params).promise();
allParameters.push.apply(allParameters, data.Parameters);
}
return allParameters;
} catch (err) {
return Promise.reject(err);
}
}
calling this client:
const ssm = require("yourssmclinet");
ssm.init();
// you call only once to retrieve everything which falls under /data/param
const parameters = await getParameters("/data/param");
//from here you can fetch parameters['what ever needed'].

You essentially have two options to get parameters in bulk.
One is the method provided by #user1032613, but the other is to use the built-in function getParametersByPath().
A Lambda code example in node with all three methods can be seen below. Each method can take different params, for instance with the path you can make filters, etc. to get the exact values you need, see the documentation.
'use strict';
const AWS = require('aws-sdk');
const SSM = new AWS.SSM();
exports.handler = async (event) => {
//Example get single item
const singleParam = { Name: 'myParam' };
const getSingleParam = await SSM.getParameter(singleParam).promise();
//Example: Get Multiple values
const multiParams = {
Names: [ 'myParam1', 'myParam2', 'myParam3' ],
WithDecryption: true
};
const getMultiParams = await SSM(multiParams).promise();
//Example: Get all values in a path
const pathParams = { Path: '/myPath/', WithDecryption: true };
const getPathParams = await SSM.getParametersByPath(pathParams).promise();
return 'Success';
};
Remember that you can also use environment variables. For example, you could write singleParam like this:
const singleParam = { Name: process.env.PARAM }
That way you can have code that extracts code from DEV, PROD, etc. depending on the stage.

Related

AWS Timestream - SDK V3 Nodejs, TimestreamWriteClient.send() - TypeError: command.resolveMiddleware is not a function. How to solve this?

I have the following lambda function in NodeJs 14.x using AWS SDK V3 for a timestream insertion process:
'use strict'
// https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-timestream-write/index.html
const { TimestreamWriteClient } = require("#aws-sdk/client-timestream-write")
const client = new TimestreamWriteClient({ region: process.env.region })
module.exports.fnPostElectricityTimestream = async event => {
try {
console.log('🚀 START fnPostElectricityTimestream')
const jsonBody = event
const topic = jsonBody.topic
const arrTopic = topic.split('/')
let dbName = arrTopic[4]
dbName = 'smaj56g' //Test
const currentTime = Date.now().toString() // Unix time in milliseconds get jsonBody.e_timestamp
const e_timestamp = (jsonBody.e_timestamp)*1000
const dimensions = [{
'Name': 'n',
'Value': 'v'
}]
const e_ch_1 = {
'Dimensions':dimensions,
'MeasureName': 'e_ch_1',
'MeasureValue': '[1,2,3]',
'MeasureValueType': 'VARCHAR',
'Time': currentTime
}
const records = [e_ch_1]
const params = {
DatabaseName: dbName,
TableName:'e_ch_1_v_w',
Records: records
}
const data = await client.send(params);
console.log('data', data)
return {
message: ''
}
} catch (error) {
console.log('🚀 fnPostElectricityTimestream - error.stack:', error.stack)
return {
message: error.stack
}
}
}
When I run the lambda this is the message I am getting:
2022-08-12T14:58:39.496Z e578a391-06b4-48a9-9f9d-9440a373c19e INFO 🚀 fnPostElectricityTimestream - error.stack: TypeError: command.resolveMiddleware is not a function
at TimestreamWriteClient.send (/var/task/node_modules/#aws-sdk/smithy-client/dist-cjs/client.js:13:33)
at Runtime.module.exports.fnPostElectricityTimestream [as handler] (/var/task/src/ElectricityTimestream/fnPostElectricityTimestream.js:38:31)
at Runtime.handleOnceNonStreaming (/var/runtime/Runtime.js:73:25)
There is something with const data = await client.send(params).
I am following the asyncawait code in this documentation.
How to solve this issue?
Your current insertion code is wrong. In order to write the records in the TimeStream, you need to use the WriteRecordsCommand command. Refer to the doc for a better understanding. Sample code:
import { TimestreamWriteClient, WriteRecordsCommand } from "#aws-sdk/client-timestream-write";
const client = new TimestreamWriteClient({ region: "REGION" }); //your AWS region
const params = {
DatabaseName: dbName, //your database
TableName: tableName, //your table name
Records: records //records you want to insert
}
const command = new WriteRecordsCommand(params);
const data = await client.send(command);
you need to create a command before calling send.
For example:
import { TimestreamWriteClient, CreateDatabaseCommand } from "#aws-sdk/client-timestream-write";
const params = {
DatabaseName: dbName,
TableName:'e_ch_1_v_w',
Records: records
}
const command = new CreateDatabaseCommand(params);
const data = await client.send(command);

Unable to stub an exported function with Sinon

I need to test the following createFacebookAdVideoFromUrl() that consumes a retryAsyncCall that I'd like to stub with Sinon :
async function createFacebookAdVideoFromUrl(accountId, videoUrl, title, facebookToken = FACEBOOK_TOKEN, options = null, businessId = null) {
const method = 'POST';
const url = `${FACEBOOK_URL}${adsSdk.FacebookAdsApi.VERSION}/${accountId}/advideos`;
const formData = {
access_token: businessId ? getFacebookConfig(businessId).token : facebookToken,
title,
name: title,
file_url: videoUrl,
};
const callback = () => requestPromise({ method, url, formData });
const name = 'createFacebookAdVideoFromUrl';
const retryCallParameters = buildRetryCallParameters(name, options);
const adVideo = await retryAsyncCall(callback, retryCallParameters);
logger.info('ADVIDEO', adVideo);
return { id: JSON.parse(adVideo).id, title };
}
This retryAsyncCall function is exported as such:
module.exports.retryAsyncCall = async (callback, retryCallParameters, noRetryFor = [], customRetryCondition = null) => {
// Implementation details ...
}
Here is how I wrote my test so far:
it.only("should create the video calling business's Facebook ids", async () => {
const payload = createPayloadDataBuilder({
businessId: faker.internet.url(),
});
const retryAsyncCallStub = sinon.stub(retryAsyncCallModule, 'retryAsyncCall').resolves('random');
const createdFacebookAd = await FacebookGateway.createFacebookAdVideoFromUrl(
payload.accountId,
payload.videoUrl,
payload.title,
payload.facebookToken,
payload.options,
payload.businessId,
);
assert.strictEqual(retryAsyncCallStub.calledOnce, true);
assert.strictEqual(createdFacebookAd, { id: 'asdf', title: 'asdf' });
});
I don't expect it to work straightaway as I am working in TDD fashion, but I do expect the retryAsyncCall to be stubbed out. Yet, I am still having this TypeError: Cannot read property 'inc' of undefined error from mocha, which refers to an inner function of retryAsyncCall.
How can I make sinon stubbing work?
I fixed it by changing the way to import in my SUT :
// from
const { retryAsyncCall } = require('../../../helpers/retry-async');
// to
const retry = require('../../../helpers/retry-async');
and in my test file :
// from
import * as retryAsyncCallModule from '../../../src/common/helpers/retry-async';
// to
import retryAsyncCallModule from '../../../src/common/helpers/retry-async';
The use of destructuring seemed to make a copy instead of using the same reference, thus, the stub was not applied on the right reference.

How to get env variable and save to parameter Store?

i'm using serverless framework(aws).
From cloudFormation when creating a database, writes to the env. variable endpoint of the database.
part of my serverless.yml file
environment:
DATABASE_HOST:
"Fn::GetAtt": [ServerlessRDS, Endpoint.Address]
This variable is available from the lambda level as it is already deployed on aws. But I want to have access to this variable from locally. I came across the idea that I would write this variable to the parameter store (aws systems manager).
So I attached the script to my serverless.yml file (using serverless-scriptable-plugin).
scriptHooks, part of my serverless.yml file
:
after:aws:deploy:finalize:cleanup:
- scripts/update-dbEndopint.js
Here's the script. Nothing special, writes an environment variable process.env.DATABASE_HOST to parameter stora.
const aws = require('aws-sdk');
const ssm = new aws.SSM();
(async () => {
try {
const params = {
Name: `${process.env.AWS_SERVICE_NAME}-DATABASE_HOST-${
process.env.AWS_STAGE
}`,
Value: `${process.env.DATABASE_HOST}`,
Type: 'String',
Overwrite: true,
};
await ssm.putParameter(params).promise();
log(`[DATABASE_HOST]: ${process.env.DATABASE_HOST} `);
log('Task done.');
} catch (e) {
throw e;
}
})();
But after taking deploy the variable is undefined.
This is because the variable value is only available later.
Do you know how to get me to get the base endpoint to parameter store?
Your servreless.yml will set the environment variable for the function but not for the process.env of the scripts run by serverless-scriptable-plugin.
You'll need to save it as an output for your stack using something similar to this:
Resources:
ServerlessRDS:
....
Outputs:
ServerlessRDSEndpointAddress:
Value:
"Fn::GetAtt": [ServerlessRDS, Endpoint.Address]
Then in your script extract that value from the stack something like this:
const fs = require('fs');
const yaml = require('js-yaml');
const aws = require('aws-sdk');
const ssm = new aws.SSM();
const getStackName = (stage) => {
const content = fs.readFileSync('serverless.yml');
return `${yaml.safeLoad(content).service}-${stage}`;
};
const getStackOutputs = async (provider, stackName, stage, region) => {
const result = await provider.request(
'CloudFormation',
'describeStacks',
{ StackName: stackName },
stage,
region,
);
const outputsArray = result.Stacks[0].Outputs;
let outputs = {};
for (let i = 0; i < outputsArray.length; i++) {
outputs[outputsArray[i].OutputKey] = outputsArray[i].OutputValue;
}
return outputs;
};
(async () => {
try {
const provider = serverless.getProvider('aws');
const { stage, region } = options;
const { ServerlessRDSEndpointAddress } = await getStackOutputs(provider, getStackName(stage), stage, region)
const params = {
Name: `${process.env.AWS_SERVICE_NAME}-DATABASE_HOST-${
process.env.AWS_STAGE
}`,
Value: `${ServerlessRDSEndpointAddress}`,
Type: 'String',
Overwrite: true,
};
await ssm.putParameter(params).promise();
log(`[DATABASE_HOST]: ${ServerlessRDSEndpointAddress} `);
log('Task done.');
} catch (e) {
throw e;
}
})();
I'm not sure how saving the value in parameter store will allow you to access it locally though.
If you want to invoke the function locally you can use:
serverless invoke local -f functionName -e DATABASE_HOST=<DATABASE_HOST>
Or use dotenv for any other JavaScript code

my node.js code work unstable in aws-lambda, Sometimes it works fine, sometimes skipping the data persistence that piece of code

The data center uploads a csv file to our S3 bucket every five minutes, which triggers my lambda function to read the file and save the data to DynamoDB. But the code that performs data persistence is not stable, sometimes it will be executed, and sometimes it will be skipped completely. This makes me very confused. Here is my code.
var AWS = require('aws-sdk');
var csvtojson = require('csvtojson');
var encoding = require('text-encoding');
AWS.config.update({
accessKeyId: '********',
secretAccessKey: '**********',
region: 'us-west-2',
sslEnabled:false
});
var s3 = new AWS.S3();
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
try {
console.log(event);
console.log(event['Records'][0]['s3']['object']['key']);
//get the file name
let key = event['Records'][0]['s3']['object']['key'];
let date = `${key}`.slice(24,36);
console.log(date);
let getObject = {Bucket: 'saas-status-mockup-data', Key: `${key}`};
//get the object
let response = await s3.getObject(getObject).promise();
//transfer to csv
let csvFile= new encoding.TextDecoder("utf-8").decode(response.Body);
//transfer to json
let res = await csvtojson().fromString(csvFile);
console.log(res);
await res.map(async(item,key) => {
console.log(item);
let putParams = {};
if(item.FARM=="SMAX Internal production Functionalities"){
putParams.TableName = 'InternalProductionDb';
} else if(item.FARM=="SMAX Trial Major Functionalities"){
putParams.TableName = 'TrialMajorDb';
} else {
console.error(item);
}
putParams.Item = {
'Date' : {
S:`${date}${item.BUSINESS_PROCESS}`
},
'StatusId':{
S:`${date}${item.BUSINESS_PROCESS}`
},
'BusinessProcess':{
S:`${item.BUSINESS_PROCESS}`
},
'Status':{
S:`${item.STATUS}`
}
};
console.log(putParams);
//put data to dynamoDB, But sometimes this code sometimes does not execute.
let putRes = await ddb.putItem(putParams).promise();
console.dir(putRes);
});
}
catch(error){
console.error(error);
return error;
}
};
Array.map() returns an array not a Promise so you cannot await it (e.g. await res.map() in your code).
First, you should collect a list of promises and the use Promise.all() to wait for all of them.
exports.handler = async (event) => {
try {
console.log(event);
console.log(event['Records'][0]['s3']['object']['key']);
//get the file name
let key = event['Records'][0]['s3']['object']['key'];
let date = `${key}`.slice(24,36);
console.log(date);
let getObject = {Bucket: 'saas-status-mockup-data', Key: `${key}`};
//get the object
let response = await s3.getObject(getObject).promise();
//transfer to csv
let csvFile= new encoding.TextDecoder("utf-8").decode(response.Body);
//transfer to json
let res = await csvtojson().fromString(csvFile);
console.log(res);
// Construct the list of promises.
const promises = res.map((item, key) => {
console.log(item);
let putParams = {};
if(item.FARM=="SMAX Internal production Functionalities"){
putParams.TableName = 'InternalProductionDb';
} else if(item.FARM=="SMAX Trial Major Functionalities"){
putParams.TableName = 'TrialMajorDb';
} else {
console.error(item);
}
putParams.Item = {
'Date' : {
S:`${date}${item.BUSINESS_PROCESS}`
},
'StatusId':{
S:`${date}${item.BUSINESS_PROCESS}`
},
'BusinessProcess':{
S:`${item.BUSINESS_PROCESS}`
},
'Status':{
S:`${item.STATUS}`
}
};
console.log(putParams);
//put data to dynamoDB, But sometimes this code sometimes does not execute.
return ddb.putItem(putParams).promise();
});
// Wait for all promises to finish.
return Promise.all(promises)
}
catch(error){
console.error(error);
return error;
}
};

Parse Server edit Relations on Object very slow

I've got the following function which works as expected on Parse Server cloud code, however it's painfully slow.
The nested for loops which are internally calling queries and save functions are undoubtedly the root cause.
How can I refactor this code so that there is some async processing or even better what methods are there to remove / edit the relations on an object, the documentation around this is very poor.
ClientLabels.applyClientLabels = async (req, res) => {
const { clients, labels } = req.params;
const user = req.user;
const objectIds = clients.map((client) => client.objectId);
const clientSaveList = [];
const clientClass = Parse.Object.extend('Clients');
const query = new Parse.Query(clientClass);
query.containedIn("objectId", objectIds);
const queryResult = await query.find({ sessionToken: user.getSessionToken() })
try {
for (const client of queryResult) {
const labelRelation = client.relation('labels');
const relatedLabels = await labelRelation.query().find({ sessionToken: user.getSessionToken() });
labelRelation.remove(relatedLabels);
for (const label of labels) {
label.className = "ClientLabels";
const labelRelationObj = Parse.Object.fromJSON(label)
labelRelation.add(labelRelationObj);
};
clientSaveList.push(client);
};
const saved = await Parse.Object.saveAll(clientSaveList, { sessionToken: user.getSessionToken() })
res.success(saved);
} catch (e) {
res.error(e);
};
}
Explanation of some weirdness:
I am having to call Parse.Object.fromJSON in order to make the client side label object a ParseObjectSubClass and allow operations on it such as adding relations.
You cannot use include on a relation query as you would with a Pointer, so there needs to be a query for relations all on it's own. An array of pointers was ruled out as there is going to be an unknown amount of labels applied.
There are a few things that can be done: (1) The creation of labels in the inner loop is invariant relative to the outer loop, so that can be done one time, at the start. (2) There's no need to query the relation if you're just going to remove the related objects. Use unset() and add to replace the relations. (3) This won't save much computation, but clientSaveList is superfluous, we can just save the query result...
ClientLabels.applyClientLabels = async (req, res) => {
const { clients, labels } = req.params;
const objectIds = clients.map((client) => client.objectId);
let labelObjects = labels.map(label => {
label.className = "ClientLabels";
return Parse.Object.fromJSON(label)
});
const query = new Parse.Query('Clients');
query.containedIn("objectId", objectIds);
const sessionToken = req.user.getSessionToken;
const queryResult = await query.find({ sessionToken: sessionToken })
try {
for (const client of queryResult) {
client.unset('labels');
client.relation('labels').add(labelObjects);
};
const saved = await Parse.Object.saveAll(queryResult, { sessionToken: sessionToken })
res.success(saved);
} catch (e) {
res.error(e);
};
}

Resources