My example is quite simple. I am using AWS Lambda in proxy mode where the index.js looks like this.
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
exports.handler = (event, context) => {
console.log(`EVENT: ${JSON.stringify(event)}`);
return awsServerlessExpress.proxy(server, event, context, 'PROMISE').promise;
};
I have a separate app.js file which has a POST endpoint. Inside this endpoint I need to make two queries on a DynamoDB table.
I need to first query the table to determine if something exists.
My code looks like this.
// DynamoDB Document Client
const docClient = new AWS.DynamoDB.DocumentClient();
app.post('/profile/:userSub', (req, res) => {
const userSub = req.params.userSub;
const accountName = req.body.accountName;
// First query
const params = {
TableName: profileTableName,
IndexName: profileIndexAccountName,
KeyConditionExpression: 'accountName = :accountName',
ExpressionAttributeValues: {
':accountName' : accountName
}
};
docClient.query(params, (err, data) => {
// Process the results
});
}
The problem is that I want to do a follow up query docClient.put based on the results from the first query.
I don't understand how to chain the queries together so that the first one is completed before the second one is executed.
Can someone please point me to an example? Or alternatively, if there's a better using async/await then I'm happy to follow it.
TL;DR prefer the SDK's async-await patterns.
Here is an example of consecutive async-await calls using the AWS SDK for JavaScript v3.*. Note the async keyword in the function signature and the await keyword before the promise-returning method calls. The docs have the complete client initialization code.
// myLambda.js
import { DynamoDBDocument } from '#aws-sdk/lib-dynamodb';
// ... configure client
const ddbDocClient = DynamoDBDocument.from(client);
export async function handler(event) {
// ... other stuff
const getResult = await ddbDocClient.get({ TableName, Key });
const putResult = await ddbDocClient.put({
TableName,
Item: { id: '2', content: getResult.Item.Name },
});
}
* The #aws-sdk/lib-dynamodb library's DynamoDBDocument is the v3 equivalent of the v2 client in your code. It exposes convenience methods like .get and .put. It also helps convert between native js types and DynamoDB attribute values.
Related
I'm running a NodeJS lambda function which is triggered by API Gateway.
My goal is to push the data and then send a status response. I think the lambda stops running before the insertData function finishes its execution, because sometimes it works but in most requests it doesn't.
Could someone lend a hand on this?
Here is my code:
// Set a table name that we can use later on
const tableName = "InterestRates"
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create the DynamoDB service object
var ddb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
exports.handler = async (event) => {
// TODO implement
console.log(">>> Running LoanInterestRates")
if(event['resource'] == '/rates'){
if(event.httpMethod === 'POST'){
return newRate(event);
}
}
};
function insertData(params){
let status;
// Call DynamoDB to add the item to the table
ddb.putItem(params, function(err, data) {
if (err) {
status = "Error";
} else {
status = "Success";
}
});
return status
}
function newRate (event){
const newRate = JSON.parse(event.body);
var params = {
TableName: 'InterestRates',
Item: {
'termlength' : {N: newRate["length"]},
'InterestRate': {S: newRate["rate"]}
}
};
let addNewRate = insertData(params);
return {
statusCode: 200,
body: JSON.stringify({
response: addNewRate
})
}
}
I also tried using Async/Await but it didn't work.
You lambda function is Async but your code is not. You need to await the completion of your function newRate which in turn should also await the function inserData which should also await your DDB request.
I would advise you to do one of two things:
Learn how JS Async nature works and ensure you understand when you need to await.
Use a synchronous programming language like Python/Boto3 where you will not run into such issues.
This is probably pretty easy but I've been unable to piece it together properly.
I'm trying to use the Sequelize NPM to call a stored procedure that I built and then I want to trigger it with a GET request from from an express api and return the output of the procedure to the api.
Here is what my code looks like for the Sequelize portion....
// Testing stored procedure //
const Retrieve = (testName) => connection.testdata_connection.query("EXEC [SPROC] [INPUTS]")
module.exports = {
tests: Tests(),
retrieve: Retrieve()
};
This part "connection.testdata_connection" is just establishing my connection to the database and I have tested this and I know this part is set.
I would like to be able to hit that with something like...
const query = require('./database/queries'); ///Imports sequelize queries
const app = express();
app.get('/decrypt', function(req,res){
query.retrieve()
})
})
This doesn't work at all.
Now if I do something like this in the queries file...
const Retrieve = async function() {
const decrypt = await connection.testdata_connection.query("EXEC [SPROC] [INPUT]")
console.log(decrypt)
}
module.exports = {
tests: Tests(),
retrieve: Retrieve()
};
This will log to my console with correct data when I start the server. I want it to do that when I hit it with my endpoint.
First, your function should be exported but not executed:
// creating an async function (all i/o operations should be async).
const Retrieve = async(testName) => connection.testdata_connection.query("EXEC [SPROC] [INPUTS]")
module.exports = {
retrieve: Retrieve,
// retrieve: Retrieve() if you call the function with (), the function will be executed and we don't want that yet
};
Now we can call it in the route:
const query = require('./database/queries'); ///Imports sequelize queries
const app = express();
// the route is async because it is executing async code
app.get('/decrypt', async (req,res) => {
// waiting for the answer with await
const response = await query.retrieve();
// Doing something with the response
})
You still need to check for errors, but that is the basics.
Here is what I want to achieve : I want to get a JSON on a daily basis from a URL and convert it to a cloud firestore collection in order to be able to use it in my Flutter app. Ideally, the script would only add new data to the collection.
I saw that I can use scheduler from Firebase cloud functions to run tasks daily. That's not the problem for now.
However, I don't know how to use Firebase cloud functions properly to get data from URL and convert it to collection. Maybe that's not the point of cloud functions and I misunderstood something. So first question : Can I run classic nodeJS stuff inside cloud functions? I suppose I can
Next, I initialized a cloud function project locally, connected it to my Google account and started to write code into index.js.
const functions = require("firebase-functions");
const admin = require('firebase-admin');
const fetch = require('node-fetch');
const db = admin.firestore();
const collectionToiletRef = db.collection('mycollection');
let settings = { method: "Get" };
let url = "my-url.com"
fetch(url, settings)
.then(res => res.json())
.then((json) => {
print(json);
// TODO for each json object, add new document
});
Second question : How can I run this code to see if it works ? I saw that emulator can be used but how can I check visually my cloud firestore collection ? On this simple example, I only want to print my json to see if I can get the data correctly. Where would the printing be done ?
Maybe cloud functions is not what I need for this task. Maybe my code is bad. I don't know. Thanks for your help.
EDIT
I tried this but the call never ends. I think it's waiting for a promise that never returns or something like that.
const functions = require("firebase-functions");
const admin = require('firebase-admin');
const fetch = require('node-fetch');
admin.initializeApp();
const db = admin.firestore();
exports.tempoCF = functions
.firestore.document('/tempo/{docId}')
.onCreate(async (snap, context) => {
console.log("onCreate");
let settings = { method: "Get" };
let url = "https://opendata.paris.fr/api/records/1.0/search/?dataset=sanisettesparis&q=&rows=-1"
try {
let response = await fetch(url, settings);
let json = await response.json();
// TODO for each json object, add new document
await Promise.all(json["records"].map(toiletJsonObject => {
return db.collection('toilets').doc(toiletJsonObject["recordid"]).set({}); // Only to create documents, I will deal with the content later
}));
}
catch(error) {
console.log(error);
return null;
}
}
);
This code works and create all the documents I want but never return. However, the async (snap, context) => {} passed to onCreate is a Promise. And this promise ends when Promise.all ends. I'm missing something but I don't know why. I'm struggling a lot with async programming with Dart or JS. Not very clear in my mind.
Can I run classic nodeJS stuff inside cloud functions?
Sure! Since the fetch method returns a Promise you can very well use it in a background triggered or a scheduled Cloud Function.
How can I run this code to see if it works?
Your code will work perfectly in the emulator suite, but you will need to trigger the Cloud Function with one of the Firebase services that can run in the emulator. For example you can trigger the Cloud Function by creating a document in the Firestore emulator console.
The following Cloud Function will do the trick: just create a doc in a dummy tempo collection and the CF will add a new doc in a newDocscollection. It's up to you to adapt the fields values for this doc, I've just used the entire JSON object.
exports.tempoCF = functions
.firestore.document('/tempo/{docId}')
.onCreate((snap, context) => {
let settings = { method: "Get" };
let url = "https://..."
return fetch(url, settings)
.then(res => res.json())
.then((json) => {
console.log(json);
// TODO for each json object, add new document
return admin.firestore().collection('newDocs').add(json);
})
.catch(error => {
console.log(error);
return null;
});
});
You could also deploy your Cloud Function to the Firebase backend, and if you want to schedule it, just change the code as follows (change the trigger):
exports.scheduledFunction = functions.pubsub.schedule('every 5 minutes').onRun((context) => {
let settings = { method: "Get" };
let url = "https://..."
return fetch(url, settings)
.then(res => res.json())
.then((json) => {
console.log(json);
// TODO for each json object, add new document
return admin.firestore().collection('newDocs').add(json);
})
.catch(error => {
console.log(error);
return null;
});
});
Edit following your edit:
The following code does work correctly in the emulator, creating docs in the toilets collection.
exports.tempoCF = functions.firestore
.document('/tempo/{docId}')
.onCreate(async (snap, context) => {
console.log('onCreate');
let settings = { method: 'Get' };
let url =
'https://opendata.paris.fr/api/records/1.0/search/?dataset=sanisettesparis&q=&rows=-1';
try {
let response = await fetch(url, settings);
let json = await response.json();
return Promise.all( // Here we return the promise returned by Promise.all(), so the life cycle of the CF is correctly managed
json['records'].map((toiletJsonObject) => {
admin
.firestore()
.collection('toilets')
.doc(toiletJsonObject['recordid'])
.set({ adresse: toiletJsonObject.fields.adresse });
})
);
} catch (error) {
console.log(error);
return null;
}
});
I want to get user details from AWS IAM so I have created a lambda function but there is an error with response code 502. My code is as below.
var AWS = require('aws-sdk');
var iam = new AWS.IAM();
AWS.config.loadFromPath('./config.json');
let getUsers = async (event, callback) => {
var params = {
UserName: "5dc6f49d50498e2907f8ee69"
};
iam.getUser(params, (err, data) => {
if (err) {
callback(err)
} else {
console.log(data)
callback(data)
}
})
};
Since your function already is async you don't need to use the old, outdated callback approach.
The AWS SDK methods provide a .promise() method which you can append to every AWS asynchronous call and, with async, you can simply await on a Promise.
var AWS = require('aws-sdk');
var iam = new AWS.IAM();
AWS.config.loadFromPath('./config.json');
let getUsers = async event => {
var params = {
UserName: "5dc6f49d50498e2907f8ee69"
};
const user = await iam.getUser(params).promise()
console.log(user)
};
Hopefully you have a handler that invokes this code, otherwise this is not going to work.
If you want to export getUsers as your handler function, make sure export it through module.exports first. I.e: module.exports.getUsers = async event.... Double check that your Lambda handler is properly configured on the function itself, where index.getUsers would stand for index being the filename (index.js) and getUsers your exported function.
I'm trying to write a unit test using aws-sdk-mock's promise support. I'm using DocumentClient.
My code looks like this:
const docClient = new AWS.DynamoDB.DocumentClient();
const getItemPromise = docClient.get(params).promise();
return getItemPromise.then((data) => {
console.log('Success');
return data;
}).catch((err) => {
console.log(err);
});
My mock and unit test looks like this:
const AWS = require('aws-sdk-mock');
AWS.Promise = Promise.Promise;
AWS.mock('DynamoDB.DocumentClient', 'get', function (params, callback)
{
callback(null, { Item: { Key: 'test value } });
});
dynamoStore.getItems('tableName', 'idName', 'id').then((actualResponse) => {
// assertions
done();
});
Runnning my unit test, does not return my test value, it actually bypasses my mock, and calls calls dynamoDb directly. What am I doing wrong? How can I get my mock set up properly?
It's unclear from your code but aws-sdk-mock has this note
NB: The AWS Service needs to be initialised inside the function being tested in order for the SDK method to be mocked
so the following will not mock correctly
var AWS = require('aws-sdk');
var sns = AWS.SNS();
var dynamoDb = AWS.DynamoDB();
exports.handler = function(event, context) {
// do something with the services e.g. sns.publish
}
but this will
var AWS = require('aws-sdk');
exports.handler = function(event, context) {
var sns = AWS.SNS();
var dynamoDb = AWS.DynamoDB();
// do something with the services e.g. sns.publish
}
see more here https://github.com/dwyl/aws-sdk-mock#how-usage
It might be too late for an answer, but I had the same problem and I stumbled upon this question. After a few tries I found a solution that doesn't involve aws-sdk-mock but only plain Sinon, and I hope that sharing it would help someone else. Note that the DynamoDB client is create outside the lambda.
The lambda itself looks like this:
const dynamoDB = new DynamoDB.DocumentClient();
exports.get = async event => {
const params = {
TableName: 'Tasks',
Key: {
id: event.pathParameters.id
}
};
const result = await dynamoDB.get(params).promise();
if (result.Item) {
return success(result.Item);
} else {
return failure({ error: 'Task not found.' });
}
};
And the test for this lambda is:
const sandbox = sinon.createSandbox();
describe('Task', () => {
beforeAll(() => {
const result = { Item: { id: '1', name: 'Go to gym'}};
sandbox.stub(DynamoDB.DocumentClient.prototype, 'get').returns({promise: () => result});
});
afterAll(() => {
sandbox.restore();
});
it('gets a task from the DB', async () => {
// Act
const response = await task.get(getStub);
// Assert
expect(response.statusCode).toEqual(200);
expect(response.body).toMatchSnapshot();
});
});
I like to use Sinon's sandbox to be able to stub a whole lot of different DynamoDB methods and clean up everything in a single restore().
sinon and proxyquire can be used to mock the dynamodb client.
It supports both callback based and async/await based calls.
Refer this link for full details
https://yottabrain.org/nodejs/nodejs-unit-test-dynamodb/
Somewhat related to the question, expanding wyu's solution - i too faced similar issue - for me, below didn't work with aws-sdk-mock
const AWS = require('aws-sdk');
AWS.config.update({region: 'us-east-1'});
let call = function (action, params) {
const dynamoDb = new AWS.DynamoDB.DocumentClient();
return dynamoDb[action](params).promise();
};
where as this worked
let call = function (action, params) {
const AWS = require('aws-sdk');
AWS.config.update({region: 'us-east-1'});
const dynamoDb = new AWS.DynamoDB.DocumentClient();
return dynamoDb[action](params).promise();
};
I had exactly the same problem of mock failing but resolved the issue after following the suggestion by a user who above by moving the following line within the function rather than defining outside:
let sns = new AWS.SNS(.....)