Lambda Node 8 function not awaiting method (AWS service call) to complete - node.js

I'm creating an invalidation to CloudFront via API Gateway call to Lambda, but the invalidation only gets processed about 1 in 5 attempts. I'm certain I am missing something that would properly wait for the createInvalidation method to be called.
'use strict';
var AWS = require('aws-sdk');
var cloudfront = new AWS.CloudFront();
var distID = process.env.DISTRIBUTION_ID;
exports.handler = async(event) => {
var path = (event.queryStringParameters.path);
console.log(event.queryStringParameters.path);
var params = {
DistributionId: distID,
InvalidationBatch: {
CallerReference: (event.requestContext.requestId),
Paths: {
Quantity: 1,
Items: [
path
]
}
}
};
await cloudfront.createInvalidation(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
return data;
});
};
The Lambda function completes every time and is receiving and parsing the API Gateway call just fine to populate the variables, but the createInvalidation only happens about 1 in 5 attempts, though no errors result from the other 4 attempts.

Try the following snippet:
// import * as AWS from 'aws-sdk';
exports.handler = async(event) =>
{
const cloudfront = new AWS.CloudFront();
const distID = process.env.DISTRIBUTION_ID;
const path = (event.queryStringParameters.path);
console.log(event.queryStringParameters.path);
const params =
{
DistributionId: distID,
InvalidationBatch:
{
CallerReference: (event.requestContext.requestId),
Paths: { Quantity: 1, Items: [ path ] }
}
};
try
{
const resp = await cloudfront.createInvalidation(params).promise();
console.log(resp);
}
catch (ex)
{
console.error(ex);
}
};

Related

Lambda NodeJS works intermittent with async method

I have a lambda function written in node.js. The lambda logic is to extract secrets from aws secret manager and pass the params to another module with a token to make a soap call.This lambda was working as expected without async method to get secreats when secreats hardcoded.
But after adding an async getsecrets method the lambda works intermittently while testing in aws console. The getsecreats returns params every time. But the lambda terminates without the intended output. It doesn't make any soap call.
The logic of the lambda code is
Get the secret
Pass the secret to the CallServicewithToken()
get XML data from soap and populate it into the database.
Why it works intermittently when introducing async calls? I have tried introducing async to all methods. still the same issue. Appreciate any input/help.
The code is as below
'use strict';
var soap = require('strong-soap').soap;
const aws = require("aws-sdk");
const request = require('request');
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
var rfcUrl = process.env.rfcser_url;
var testUrl = process.env.faccser_url;
var region = process.env.region;
var secretName = process.env.secretName;
var oauthurl = process.env.oauthurl;
var rfcRequestArgs;
var parsedResult;
var tokenrequest;
exports.handler = async function(event, context,callback) {
const secret = await getSecrets();
CallServicewithToken();
};
async function getSecrets() {
const config = { region : region, apiVersion: 'latest' };
let secretManager = new aws.SecretsManager(config);
const Result = await secretManager.getSecretValue({ SecretId: secretName }).promise();
parsedResult = JSON.parse(Result.SecretString);
tokenrequest = {
url: oauthurl,
form: {
client_id: parsedResult.client_id,
client_secret: parsedResult.client_secret,
grant_type:'client_credentials',
scope:parsedResult.scope
}
};
console.log('client_id: ' + parsedResult.client_id);
console.log('client_secret: ' + parsedResult.client_secret);
console.log('testservice_userid: ' + parsedResult.testservice_userid);
}
function CallServicewithToken() {
console.log('Calling CallServicewithToken ');
request.post(tokenrequest, (err, res, body) => {
if (err) {
console.log(' error2: ' + err);
return;
}
rfcRequestArgs = {
UserName: parsedResult.service_username
};
var tokenobj = JSON.parse(body);
var token = 'Bearer '+ tokenobj.access_token;
var credentials = {
Authorization:{
AuthToken: token
}
}
var options = {};
console.log('Calling Service.');
soap.createClient(rfcUrl, options, function(err, client) {
client.addSoapHeader(
`<aut:Authorization xmlns:aut="http://soap.xyznet.net">
<aut:AuthToken>${token}</aut:AuthToken>
</aut:Authorization>`
);
var method = client['GetSourceLocationData'];
method(RequestArgs, function(err, result, envelope, soapHeader) {
if (err) {
console.log('error3: ' + err);
return;
}
else
{
console.log('Received response from GetSourceLocationData().');
CallTESTService(JSON.stringify(result));
}
});
function CallTESTService(LocData)
{
var testRequestArgs = {
UserID: parsedResult.testservice_userid,
AuthorizationKey: parsedResult.testservice_authorizationkey,
LocationData: LocData
};
console.log('Calling test Service.');
options = {};
soap.createClient(testUrl, options, function(err, client) {
client.addSoapHeader(
`<Authorization xmlns="testWebService">
<AuthToken>${token}</AuthToken>
</Authorization>`
);
var test_method = client['UpdateLocationData'];
console.log('Called UpdateLocationData service method.');
test_method(testRequestArgs, function(err, result, envelope, soapHeader) {
if(err) {
console.log('test error: ' + err);
return;
}
else
{
console.log('Response: \n' + JSON.stringify(result));
console.log('Data updated through test service method.');
}
});
});
}
});
});
}

AWS SDK Gateway getUsagePlans not working

Im trying to get a list of all the API usage plans on my account, running the cli command returns my desired result, however I cant get the JS SDK version to work in Lambda. What's going wrong? I see in he sdk its paginated but it doesn't return data after I include that kind of info either.
CLI:
aws apigateway get-usage-plans
Output:
{
"items": [
{
"id": "3hhulv",
"name": "testplan",
"apiStages": [
{
"apiId": "dp6ounv3jd",
"stage": "default"
}
],
"throttle": {
"burstLimit": 10,
"rateLimit": 10.0
},
"quota": {
"limit": 10000,
"offset": 0,
"period": "MONTH"
}
}
]
}
In node:
const AWS = require('aws-sdk');
exports.handler = async (event) => {
var apigateway = new AWS.APIGateway();
var params = {};
var usageplans = apigateway.getUsagePlans(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
const response = {
statusCode: 200,
things : usageplans.data
};
return response;
};
output:
{
"statusCode": 200
}
I resolved the issue by removing the callback function and adding a .promise() onto it.
const AWS = require('aws-sdk');
exports.handler = async (event) => {
var apigateway = new AWS.APIGateway();
var params = {};
var usageplans = await apigateway.getUsagePlans(params).promise();
const response = {
things : usageplans.position,
statusCode: 200
};
return response;
};

Data not saved into Firestore through scheduler function with Firebase Cloud Functions

I'm trying to create APIs with Firebase Cloud Functions so as to build a system to make what's collected through the APIs and other third-party APIs routinely saved into Firestore. Here is my source code index.js at this moment, and I'd like to control all the processes in this one file.
/functions/index.js
const firebaseAdmin = require("firebase-admin");
const firebaseFunctions = require("firebase-functions");
firebaseAdmin.initializeApp();
const fireStore = firebaseAdmin.firestore();
const express = require("express");
const axios = require("axios");
const cors = require("cors");
const serviceToken = "SERVICE-TOKEN";
const serviceBaseUrl = "https://api.service.com/";
const app = express();
app.use(cors());
const getAllExamples = async () => {
var url = `${serviceBaseUrl}/examples?token=${serviceToken}`;
var config = {
method: "get",
url: url,
headers: {}
};
axios(config).then((res) => {
console.log("Data saved!");
return res.data;
}).catch((err) => {
console.log("Data not saved: ", err);
return err;
});
}
const setExample = async (documentId, dataObject) => {
fireStore.collection("examples").doc(documentId).set(dataObject).then(() => {
console.log("Document written!");
}).catch((err) => {
console.log("Document not written: ", err);
});
}
app.get("/getAllExamples", (req, res) => {
getAllExamples().then((response) => res.send(response));
});
app.put("/setExample", (req, res) => {
setExample(req.params.documentId).then((response) => res.send(response));
});
const api = firebaseFunctions.https.onRequest(app);
module.exports = { api };
module.exports.updateExamplesRoutinely = firebaseFunctions.pubsub.schedule("0 0 * * *").timeZone("America/Los_Angeles").onRun((context) => {
var examples = getAllExamples();
for(var i = 0; i < examples.length; i++) {
var example = examples[i];
var exampleId = example["id"];
if(exampleId && example) setExample(exampleId, example);
}
});
As a result, updateExamplesRoutinely is properly triggered every 00:00 in PST, but NO data is stored in Firebase and NO console logs about Firebase data updates and found in Cloud Functions Logs.
Output data collected through https://api.service.com/ is something like this below:
[
{
id: "BB345",
name: "Tim Gordon",
age: 24,
email: "tgordon#yahoo.com"
},
{
id: "CC098",
name: "John Doe",
age: 28,
email: "john.doe#gmail.com"
},
{
id: "DD777",
name: "Adam Smith",
age: 39,
email: "adamsmith#outlook.com"
},
...
]
Simply, these 3 problems are what I would like to get resolved.
How should I call APIs defined in index.js through Cloud Functions' scheduler?
How should I save data into Firestore inside Cloud Functions' scheduler?
What's the best "async" way to call the third-party APIs, wait and collect the results, and pass them to other events or functions?
How should I call APIs defined in index.js through Cloud Functions'
scheduler?
I guess you want to say "How should I call the methods defined in index.js". There is no reason to call the APIs exposed by the Cloud Funtion from this same Cloud Function.
So to call the methods, do as follows:
const getAllExamples = async () => {
var url = `${serviceBaseUrl}/examples?token=${serviceToken}`;
var config = {
method: "get",
url: url,
headers: {}
};
return axios(config).then((res) => { // !!! note the return here
console.log("Data saved!");
return res.data;
}).catch((err) => {
console.log("Data not saved: ", err);
return err;
});
}
const setExample = async (documentId, dataObject) => {
// !!! note the return on next line
return fireStore.collection("examples").doc(documentId).set(dataObject).then(() => {
console.log("Document written!");
}).catch((err) => {
console.log("Document not written: ", err);
});
}
module.exports.updateExamplesRoutinely = firebaseFunctions.pubsub.schedule("0 0 * * *").timeZone("America/Los_Angeles").onRun(async (context) => {
const examples = await getAllExamples();
const promises = [];
for(var i = 0; i < examples.length; i++) {
var example = examples[i];
var exampleId = example["id"];
if(exampleId && example) promises.push(setExample(exampleId, example));
}
return Promise.all(promises);
// See how we return the promise returned by Promise.all
// !!! This is IMPORTANT, see https://firebase.google.com/docs/functions/terminate-functions
});
How should I save data into Firestore inside Cloud Functions'
scheduler?
The above code will save the data in Firestore. You actually want to execute a variable number of call to the setExample() method, you need to use Promise.all() as shown above. This way, setExample() will write to Firestore for each example.
What's the best "async" way to call the third-party APIs, wait and
collect the results, and pass them to other events or functions?
See how we adapt the updateExamplesRoutinely Cloud Function: we use the async and await keywords.
module.exports.updateExamplesRoutinely = firebaseFunctions.pubsub.schedule("0 0 * * *").timeZone("America/Los_Angeles").onRun(async (context) => {
const examples = await getAllExamples();
// ...

Not able to test Stepfunction Execution via Lambda

I am trying to test my lambda function which invokes Step Function using AWS mock, but it is giving UnknownEndpoint: Inaccessible host: states.local-env.amazonaws.com
Here is my sample lambda function:
const AWS = require("aws-sdk");
const stepFunctions = new AWS.StepFunctions({region: 'local-env'});
exports.handler = async (event, context, config) => {
const response = {
statusCode: 200,
body: event,
};
var params = {
stateMachineArn: process.env.StateMachineARN,
input: JSON.stringify(event.body),
name: 'Testing2'
};
stepFunctions.startExecution(params, function(err, data) {
if (err) console.log(err, err.stack);
});
return response;
};
and there is my test file:
const index = require('./index')
var AWS = require('aws-sdk-mock');
const AWS_SDK = require('aws-sdk');
AWS.mock('StepFunctions', 'startExecution', function (params, callback){
callback(null, "successfully started the execution");
});
const isTest = process.env.JEST_WORKER_ID;
const config = {
convertEmptyValues: true,
...(isTest && {endpoint: 'localhost:8000', sslEnabled: false,region:'local-env'})
};
describe('Test Step Function Invocation', function () {
it('verifies successful response', async () => {
process.env.StateMachineARN = 'arn:aws:states:us-east-1:12345678:stateMachine:Testing';
var event = {
"payload": "my_payload",
"data": "some-data",
"MVId": "00156"
};
const result = await index.handler(event,{},config);
expect(result.statusCode).toEqual(200);
expect(result.body).toBe(event);
});
});
AWS.restore('StepFunctions');
I've been searching about this in the documentation and other resources but haven't found any solutions yet.

lambda with graphql and sqs send 2 messages to sqs in nodejs?

i'm working on project where i need to write a lambda function which provides AWS API to handles GraphQL query and send the payload to AWS SQS everything is working fine but when i check my AWS SQS queue it shows 2 messages every single time instead of 1 and cloud watch also shows function trigger only once. below i'm sharing my code with you any help would be very much appreciated.
index.js
const { graphql } = require("graphql");
const { schema } = require("./graphql/schema");
exports.handler = async (event) => {
// getting query from lambda event
const query = event.query;
// getting query variables from lambda event
const variables = event.variables;
return await graphql(schema, query, null, null, variables);
};
sqs.js
const AWS = require("aws-sdk");
AWS.config.update({ region: "us-east-1"});
// Create an SQS service object
const sqs = new AWS.SQS({apiVersion: '2012-11-05', "accessKeyId": process.env.ACCESS_KEY_ID, "secretAccessKey": process.env.SECRET_ACCESS_KEY});
const QueueUrl = process.env.SQS_QUEUE_URL;
const sendPayloadToSQS = message => {
const params = {
MessageBody: JSON.stringify(message),
QueueUrl
};
await sqs.sendMessage(params, function(err, data) {
if (err) {
console.log("Message sending failed : ", err);
} else {
console.log("Message queued to SQS successfully : ", data.MessageId);
}
}).promise();
};
module.exports = sendPayloadToSQS;
graphql mutation file
const { GraphQLNonNull } = require("graphql");
const { mutationWithClientMutationId } = require("../../common");
const { JobRequestEventResponse } = require("../jobRequestEventResponse");
const { JobRequestInput, JobEventMetadataInput } = require("../jobSchema");
const sendPayloadToSQS = require("../../../sqs");
const { newId } = require("../../../newId");
const JobRequestEvent = mutationWithClientMutationId({
name: "JobRequestEvent",
inputFields: {
eventMetadataInput: {
type: new GraphQLNonNull(JobEventMetadataInput),
},
eventInput: {
type: new GraphQLNonNull(JobRequestInput),
},
},
outputFields: {
JobRequestEventResponse: {
type: JobRequestEventResponse,
},
},
mutateAndGetPayload: async (params) => {
const new_id = newId();
if(params.eventInput.jobId === null || params.eventInput.jobId === undefined) {
params.eventInput.jobId = new_id;
}
const payload = {
_id: new_id,
transactionId: new_id,
name: params.eventMetadataInput.name,
userRole: params.eventMetadataInput.userRole,
date: params.eventMetadataInput.date,
languageCode: params.eventMetadataInput.languageCode,
eventInput: params.eventInput,
};
//send payload to sqs
await sendPayloadToSQS(payload);
return {
JobRequestEventResponse: {
id: payload._id,
transactionId: payload.transactionId,
status: "Success",
},
};
},
});
module.exports = {
JobRequestEvent,
};
I read the documentation again and found the callback is the root cause of my problem: if I provide a callback to sendMessage it triggers my function and after when I write promise() again it triggers my function so I remove the callback only as you can see below.
Refer : AWS Official documentation
Instead of this:
await sqs.sendMessage(params, function(err, data) {
if (err) {
console.log("Message sending failed : ", err);
} else {
console.log("Message queued to SQS successfully : ", data.MessageId);
}
}).promise();
I write this:
const request = sqs.sendMessage(params);
const result = await request.promise();
if(result) {
console.log("Message queued to SQS successfully : ", result.MessageId);
} else {
console.log("Message queued failed");
}

Resources