Query onto a dynamoDb table - node.js

I am trying to get all the records of a database that meet a filter that I already defined. The problem with this is that I am having to repeat it N times depending on the number of organizations. Is there a way to make it iterate for each of the organizations?
I tried putting the organization_code in an array but it fails saying that the schema is not correct. Please help
const AWS = require('aws-sdk')
AWS.config.update({ region: 'us-east-2' })
const dDBClient = new AWS.DynamoDB.DocumentClient();
const async = require('async')
let scanComplete = false
let beginDate = "2020-12-16 00:00:00.000"
let endDate = "2020-12-23 00:00:00.000"
const organization_code = "CLPIT01DB" < - HERE ARE JUST 1 ORG, NEED 6 of them!
const params = {
ExpressionAttributeValues: {
':organization_code': organization_code,
':BeginDate': beginDate,
':EndDate': endDate
},
KeyConditionExpression: 'organization_code = :organization_code',
ProjectionExpression: `booking_id,booking_payment_ids,booking_pnr,company_address,company_city,company_comuna,company_girototal_amount_clp,total_amount_exe,total_amount_vat,vat`,
FilterExpression: 'payment_createdutc >= :BeginDate and payment_createdutc < :EndDate',
TableName: 'TemporaryPayments'
}
restOfTheCode
TableConfiguration
Thank you very much!

You can try this :
const AWS = require('aws-sdk')
AWS.config.update({ region: 'us-east-2' })
const dDBClient = new AWS.DynamoDB.DocumentClient();
let beginDate = "2020-12-16 00:00:00.000"
let endDate = "2020-12-23 00:00:00.000"
let arrayData = []; // init list of query data
// Initiate the params
let params = {
ExpressionAttributeValues: {
':BeginDate': beginDate,
':EndDate': endDate
},
KeyConditionExpression: 'organization_code = :organization_code',
ProjectionExpression: `booking_id,booking_payment_ids,booking_pnr,company_address,company_city,company_comuna,company_girototal_amount_clp,total_amount_exe,total_amount_vat,vat`,
FilterExpression: 'payment_createdutc >= :BeginDate and payment_createdutc < :EndDate',
TableName: 'TemporaryPayments'
}
async function db_query(org_code) {
params.ExpressionAttributeValues = {
...params.ExpressionAttributeValues,
{
':organization_code': org_code
}
}
let res = await dDBClient.query(params).promise();
arrayData = [
...arrayData,
res.Items
]
if (res.LastEvaluatedKey) {
params.ExclusiveStartKey = res.LastEvaluatedKey;
return db_query(org_code);
}
console.log(org_code+" data length : "+arrayData.length)
arrayData = [];
}
db_query("CLPIT01DB")
You can iterate the organization queries by doing this :
const org_codes = ["CLPIT01DB", "xxxxxxxxx", etc];
let org_queries = [];
org_codes.forEach(org => {
org_queries.push(db_query(org));
})
Promise.all(org_queries)

Related

How do I check if an item with same name already exist in AWS Lambda function using DynamoDB

I have an application written in Lambda functions (with DynamoDB). It performs basic CRUD operations. While adding a new item, I want to check if an item with that same name exists already for the same user.
I implemented the logic below (the POST request). While testing on Postman I get the error "Query condition missed key schema element: user_group_id". I am actually not interested in the user_group_id cause it's only created after the group has been added.
Is there a better or special way of doing this?
What I'm I not doing right?
Thanks
const AWS = require("aws-sdk");
const { v4: uuid } = require("uuid");
const dynamo = new AWS.DynamoDB.DocumentClient();
exports.handler = async (event, context) => {
let body;
let statusCode = 200;
const headers = {
"Content-Type": "application/json",
};
try {
switch (event.routeKey) {
case "GET /user-groups/all/{id}":
const params1 = {
ExpressionAttributeValues: {
":s": parseInt(event.pathParameters.id, 10),
},
KeyConditionExpression: "user_id = :s",
TableName: "User-Groups",
};
body = await dynamo.query(params1).promise();
break;
case "PUT /user-groups/edit":
...
break;
case "PUT /user-groups": //THIS IS THE POST REQUEST
let requestJSON1 = JSON.parse(event.body);
//Check if the name already exists
const name_check = {
ExpressionAttributeValues: {
":u": requestJSON1.user_id,
":g": requestJSON1.user_group_name,
},
KeyConditionExpression: "user_id = :u and user_group_name = :g",
TableName: "User-Groups",
};
const name_exist = await dynamo.query(name_check).promise();
if(name_exist){
throw new Error(`This name already exist`);
}
const arr = requestJSON1.user_ids;
arr.push(requestJSON1.user_id);
const id = uuid();
if(requestJSON1.user_id && requestJSON1.user_group_name && requestJSON1.user_group_tags.length > 0 && requestJSON1.user_ids.length > 1){
body = await dynamo
.put({
TableName: "User-Groups",
Item: {
user_id: requestJSON1.user_id,
user_group_id: id,
user_group_name: requestJSON1.user_group_name,
user_group_tags: requestJSON1.user_group_tags,
user_ids: arr,
created_at: new Date().toGMTString(),
updated_at: new Date().toGMTString(),
},
})
.promise();
body = {
user_id: requestJSON1.user_id,
user_group_id: id,
user_group_name: requestJSON1.user_group_name,
user_group_tags: requestJSON1.user_group_tags,
user_ids: arr,
created_at: new Date().toGMTString(),
updated_at: new Date().toGMTString(),
};
break;
}
break;
case "DELETE /user-groups/{id}/{user_group_id}":
...
break;
default:
throw new Error(`Unsupported route: "${event.routeKey}"`);
}
} catch (err) {
statusCode = 400;
body = err.message;
} finally {
body = JSON.stringify(body);
}
return {
statusCode,
body,
headers,
};
};
ExpressionAttributeValues: {
":u": requestJSON1.user_id,
":g": requestJSON1.user_group_name,
},
KeyConditionExpression: "user_id = :u and user_group_name = :g",
Your issue is here. You set the KeyConditionExpression to state user_id equals :u and user_group_name equals :g However none of those are your tables partition key which is what a Query operation expects.
This means you either need to do a Scan with a FilterExpression or create a GSI with the keys user_id and user_group_name and do your query against your GSI.

AWS Timestream - SDK V3 Nodejs, TimestreamWriteClient.send() - TypeError: command.resolveMiddleware is not a function. How to solve this?

I have the following lambda function in NodeJs 14.x using AWS SDK V3 for a timestream insertion process:
'use strict'
// https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-timestream-write/index.html
const { TimestreamWriteClient } = require("#aws-sdk/client-timestream-write")
const client = new TimestreamWriteClient({ region: process.env.region })
module.exports.fnPostElectricityTimestream = async event => {
try {
console.log('🚀 START fnPostElectricityTimestream')
const jsonBody = event
const topic = jsonBody.topic
const arrTopic = topic.split('/')
let dbName = arrTopic[4]
dbName = 'smaj56g' //Test
const currentTime = Date.now().toString() // Unix time in milliseconds get jsonBody.e_timestamp
const e_timestamp = (jsonBody.e_timestamp)*1000
const dimensions = [{
'Name': 'n',
'Value': 'v'
}]
const e_ch_1 = {
'Dimensions':dimensions,
'MeasureName': 'e_ch_1',
'MeasureValue': '[1,2,3]',
'MeasureValueType': 'VARCHAR',
'Time': currentTime
}
const records = [e_ch_1]
const params = {
DatabaseName: dbName,
TableName:'e_ch_1_v_w',
Records: records
}
const data = await client.send(params);
console.log('data', data)
return {
message: ''
}
} catch (error) {
console.log('🚀 fnPostElectricityTimestream - error.stack:', error.stack)
return {
message: error.stack
}
}
}
When I run the lambda this is the message I am getting:
2022-08-12T14:58:39.496Z e578a391-06b4-48a9-9f9d-9440a373c19e INFO 🚀 fnPostElectricityTimestream - error.stack: TypeError: command.resolveMiddleware is not a function
at TimestreamWriteClient.send (/var/task/node_modules/#aws-sdk/smithy-client/dist-cjs/client.js:13:33)
at Runtime.module.exports.fnPostElectricityTimestream [as handler] (/var/task/src/ElectricityTimestream/fnPostElectricityTimestream.js:38:31)
at Runtime.handleOnceNonStreaming (/var/runtime/Runtime.js:73:25)
There is something with const data = await client.send(params).
I am following the asyncawait code in this documentation.
How to solve this issue?
Your current insertion code is wrong. In order to write the records in the TimeStream, you need to use the WriteRecordsCommand command. Refer to the doc for a better understanding. Sample code:
import { TimestreamWriteClient, WriteRecordsCommand } from "#aws-sdk/client-timestream-write";
const client = new TimestreamWriteClient({ region: "REGION" }); //your AWS region
const params = {
DatabaseName: dbName, //your database
TableName: tableName, //your table name
Records: records //records you want to insert
}
const command = new WriteRecordsCommand(params);
const data = await client.send(command);
you need to create a command before calling send.
For example:
import { TimestreamWriteClient, CreateDatabaseCommand } from "#aws-sdk/client-timestream-write";
const params = {
DatabaseName: dbName,
TableName:'e_ch_1_v_w',
Records: records
}
const command = new CreateDatabaseCommand(params);
const data = await client.send(command);

data is not showing in Dynamodb table

i am working on lex chatbot and want to store user data in dynamodb.
here is my databaseManager.js file code
'use strict';
const { v1: uuidv1 } = require('uuid');
const AWS = require('aws-sdk');
const dynamo = new AWS.DynamoDB.DocumentClient();
module.exports.saveBookingToDatabase = async function(Arrival_city, Departure_city, Flight_type, Phone_number){
console.log('saveBookingToDatabase');
const item = {};
item.bookingId = uuidv1();
item.arrivalCity = Arrival_city;
item.departureCity = Departure_city;
item.classType = Flight_type;
item.phone = Phone_number;
const params = {
TableName: 'air_stallion',
Item: item
};
try {
let result = await dynamo.put(params)
console.log(`Saving ticket ${JSON.stringify(item)}`);
return item;
} catch(e) {
throw (e)
}
}
Table has been created but data is now showing in table
The values should not be empty, give some default values to prevent null or empty values.
For Example:
const item = {};
item.bookingId = uuidv1();
item.arrivalCity = Arrival_city || "Arr";
item.departureCity = Departure_city || "Dept";
item.classType = Flight_type || "Type";
item.phone = Phone_number || "Phone";
If the values are okay, then try with
let result = await dynamo.put(params).promise()

Async Await issues with NodeJS & AWS Lambda

I'm currently developing a Lamba function call for my AWS project, but with me not being a master at asynchronous functions it appears it's falling apart, the code I've put together is:
const AWS = require("aws-sdk");
const game = require('game-api');
const uuid = require("uuid");
AWS.config.update({
region: "us-east-1"
});
exports.handler = async (event, context, callback) => {
//set db
var documentClient = new AWS.DynamoDB.DocumentClient()
//params
const params = {
Item: {
'id': uuid.v1(),
'player_1_name': null,
'player_1_network': null,
'player_1_matches': 0,
'player_1_kills': 0,
'player_1_last_updated': 0,
'player_2_name': null,
'player_2_network': null,
'player_2_matches': 0,
'player_2_kills': 0,
'player_2_last_updated': 0,
'match_id': 0,
'status': 0
},
TableName : 'matches'
};
var matchData = JSON.parse(event.body);
//player 1
const player_1_name = matchData.player_1_name ? matchData.player_1_name : null;
const player_1_network = matchData.player_1_network ? matchData.player_1_network : null;
//player 2
const player_2_name = matchData.player_2_name ? matchData.player_2_name : null;
const player_2_network = matchData.player_2_network ? matchData.player_2_network : null;
//match data
const match_id = matchData.match_id ? matchData.match_id : 0;
//game object
let gameAPI = new game(
[
"email#email.com",
"password"
]
);
//gameAPI.login() returns a Promise()
await gameAPI.login().then(() => {
//check stats for player 1, getStats returns a Promise()
gameAPI.getStats(player_1_name, player_1_network).then(stats => {
params.Item.player_1_matches = stats.lifetimeStats.matches;
params.Item.player_1_kills = stats.lifetimeStats.kills;
}).catch(err => {
//error! we must work out what to do here!
console.log(err);
});
//example insert
documentClient.put(params, function(err, data){
return callback(err, data);
});
}).catch(err => {
console.log("We failed to login!");
console.log(err);
});
};
This logic seems flawed since nothing is being thrown to my AWS logs? my idea is to send the request to the function & have it do it as quickly as possible so I can send a 200 response back to Lambda, can anyone point me in the correct direction?
When using async/await you don't need to use callback neither you need to fall into the Promise Hell.
Just await on your promises and grab the result. The great advantage here is it looks as though your code is synchronous.
Here's your refactored code:
const AWS = require("aws-sdk");
const game = require('game-api');
const uuid = require("uuid");
AWS.config.update({
region: "us-east-1"
});
exports.handler = async (event) => {
//set db
var documentClient = new AWS.DynamoDB.DocumentClient()
//params
const params = {
Item: {
'id': uuid.v1(),
'player_1_name': null,
'player_1_network': null,
'player_1_matches': 0,
'player_1_kills': 0,
'player_1_last_updated': 0,
'player_2_name': null,
'player_2_network': null,
'player_2_matches': 0,
'player_2_kills': 0,
'player_2_last_updated': 0,
'match_id': 0,
'status': 0
},
TableName : 'matches'
};
var matchData = JSON.parse(event.body);
//player 1
const player_1_name = matchData.player_1_name ? matchData.player_1_name : null;
const player_1_network = matchData.player_1_network ? matchData.player_1_network : null;
//player 2
const player_2_name = matchData.player_2_name ? matchData.player_2_name : null;
const player_2_network = matchData.player_2_network ? matchData.player_2_network : null;
//match data
const match_id = matchData.match_id ? matchData.match_id : 0;
//game object
let gameAPI = new game(
[
"email#email.com",
"password"
]
);
//gameAPI.login() returns a Promise()
await gameAPI.login()
const stats = await gameAPI.getStats(player_1_name, player_1_network)
params.Item.player_1_matches = stats.lifetimeStats.matches;
params.Item.player_1_kills = stats.lifetimeStats.kills;
//example insert
await documentClient.put(params).promise();
};
If you need to handle the exceptions (you should) just wrap your await calls in try/catch blocks, like this:
try {
console.log(await somePromise)
} catch (e) {
console.log(e)
}
The snippet above is equivalent to:
somePromise.then(console.log).catch(console.log)
with the great difference that you don't need to chain promises/asynchronous code in order to keep the execution order, so I highly recommend you pick the async/await approach and forget about .then().catch()

Retrieve AWS ssm parameter in bulk

How can I retrieve parameters from AWS Systems Manager (parameter store) in bulk (or more than one parameter) at a time? Using aws-sdk, following is the Node.js code I have written to retrieve SSM parameter from parameter store:
const ssm = new (require('aws-sdk/clients/ssm'))()
const getSSMKey = async params => {
const {Parameter: {Value: APIKey}} = await ssm.getParameter(params).promise()
return APIKey
}
const [param1, param2, param3] = await Promise.all([
getSSMKey({ Name: '/data/param/PARAM1', WithDecryption: true }),
getSSMKey({ Name: '/data/param/PARAM2', WithDecryption: true }),
getSSMKey({ Name: '/data/param/PARAM3', WithDecryption: true })
])
console.log(param1, param2, param3)
But with this code, I am sending 3 request for getting 3 parameters which is inefficient in case of large number of parameters. Is there any way to retrieve more than one parameters in one request. if ssm.getParameters() is the method to do that then please give an example (particularly parameter to that method). I tried but I receive nothing.
According to the AWS document, GetParameter gets the value for one parameter, whereas GetParameters gets the value for multiple.
Their usages are very similar too. When using GetParameters to get multiple values, pass in multiple names as a list for Names, instead of passing a single name as string for Name.
Code sample, to get parameters named "foo" and "bar", in "us-west-1" region:
const AWS = require('aws-sdk');
AWS.config.update({ region: "us-west-1" });
const SSM = require('aws-sdk/clients/ssm');
const ssm = new SSM()
const query = {
"Names": ["foo", "bar"],
"WithDecryption": true
}
let param = ssm.getParameters(query, (err, data) => {
console.log('error = %o', err);
console.log('raw data = %o', data);
})
At last it worked for me. Following is the code:
const ssmConfig = async () => {
const data = await ssm.getParameters({ Names: ['/data/param/PARAM1', '/data/param/PARAM2', '/bronto/rest//data/param/PARAM3'],
WithDecryption: true }).promise()
const config = {}
for (const i of data.Parameters) {
if (i.Name === '/data/param/PARAM1') {
config.param1 = i.Value
}
if (i.Name === '/data/param/PARAM2') {
config.rest.clientId param2 = i.Value
}
if (i.Name === '/data/param/PARAM3') {
config.param3 = i.Value
}
}
return config
}
This is what I did to retrieve all the parameters from a specific path.
**your SSM function client :**
'use strict';
const SSM = require('aws-sdk/clients/ssm');
let ssmclient;
module.exports.init = () => {
const region = process.env.REGION === undefined ? 'us-east-1' : process.env.REGION ;
ssmclient = new SSM({region: region});
}
module.exports.getParameters = async (path) => {
try {
let params = {
Path: path,
WithDecryption: true
};
let allParameters = [];
let data = await ssmclient.getParametersByPath(params).promise();
allParameters.push.apply(allParameters, data.Parameters);
while(data.NextToken) {
params.NextToken = data.NextToken;
data = await ssmclient.getParametersByPath(params).promise();
allParameters.push.apply(allParameters, data.Parameters);
}
return allParameters;
} catch (err) {
return Promise.reject(err);
}
}
calling this client:
const ssm = require("yourssmclinet");
ssm.init();
// you call only once to retrieve everything which falls under /data/param
const parameters = await getParameters("/data/param");
//from here you can fetch parameters['what ever needed'].
You essentially have two options to get parameters in bulk.
One is the method provided by #user1032613, but the other is to use the built-in function getParametersByPath().
A Lambda code example in node with all three methods can be seen below. Each method can take different params, for instance with the path you can make filters, etc. to get the exact values you need, see the documentation.
'use strict';
const AWS = require('aws-sdk');
const SSM = new AWS.SSM();
exports.handler = async (event) => {
//Example get single item
const singleParam = { Name: 'myParam' };
const getSingleParam = await SSM.getParameter(singleParam).promise();
//Example: Get Multiple values
const multiParams = {
Names: [ 'myParam1', 'myParam2', 'myParam3' ],
WithDecryption: true
};
const getMultiParams = await SSM(multiParams).promise();
//Example: Get all values in a path
const pathParams = { Path: '/myPath/', WithDecryption: true };
const getPathParams = await SSM.getParametersByPath(pathParams).promise();
return 'Success';
};
Remember that you can also use environment variables. For example, you could write singleParam like this:
const singleParam = { Name: process.env.PARAM }
That way you can have code that extracts code from DEV, PROD, etc. depending on the stage.

Resources