I have a SQS queue with thousands of messages. I cannot consume them on lambda due to size of function (using puppeteer) so I've moved my consumer to EC2 instance(s). As this is POC, I did a very basic implementation of consumer using this (https://github.com/bbc/sqs-consumer) library and launched the same process using pm2 multiple times.
import { SQSClient } from '#aws-sdk/client-sqs';
import { Consumer } from 'sqs-consumer';
const sqs = new SQSClient({
apiVersion: '2012-11-05',
region: 'us-east-1',
credentials: {
accessKeyId: #####
secretAccessKey: 232131321312
},
});
const app = Consumer.create({
queueUrl: EnvsEnum.QUEUE_URL,
terminateVisibilityTimeout: true,
waitTimeSeconds: 20,
handleMessage: async (message) => {
... my handler here doing some work on the message
},
shouldDeleteMessages: true,
region: 'us-east-1',
sqs,
});
app.on('error', (err, item) => {
...
});
app.on('processing_error', (err) => {
...
});
console.log('starting app...');
app.start();
I expected to see multiple messages in flight, but on SQS dashboard I only see 1 constantly. What could be the reason? Maybe some good guide to implementing a proper consumer if I went about it completely wrong?
Update
I solved the problem by realizing that I was sending the same MessageGroupId to the queue - fixing that makes the consumers work in parallel as expected.
Related
I have 3 messages at my aws message inbox. But i am only getting one message when I'm trying
to retrieve all. I am getting only the last message, but the other two messages are not showing.
Here is my code:
const AWS = require("aws-sdk");
AWS.config.update({ region: "REGION" });
const sqs = new AWS.SQS({ apiVersion: "2012-11-05" });
const readMessage = function () {
const params = {
MaxNumberOfMessages: 10,
QueueUrl: "https://sqs.us-east-1.amazonaws.com/56622448/my-queue",
};
sqs.receiveMessage(params, function (err, data) {
if (err) {
console.log("Error", err);
} else if (data.Messages) {
console.log(data);
}
});
};
readMessage();
This is the output:
{
ResponseMetadata: { RequestId: 'a9e53cee-afc8-57c-664408f1e602' },
Messages: [
{
MessageId: 'bb3e23f9-07fd-4205-9b53-a48f826',
ReceiptHandle: 'AQEBD6rCOZlaBfCNn3nU4AEE7OlYwFJFeblTaiDoxIy8HiwsjdxZX+3SICY/YW5PI+RuFscMMh6VyExoo1i8Zo2JlbYj3t32b9CXnToYugzBqgZuxuYOTzXRAnrGwlavSL7hcLQvW6y8me1gnj65N3tPYEmcfXX5GIiQTn1yNEou3rUNff9DfkSije/0zvp33yfWfcW+RDzB2y6ND6eKHxfsP/cqmHjRaT0bE9rlXorjgh36YwVJ57e5bjUa/1dVqOf3ybXfEX/5C2eZM+T1V2JBxlguvuL1B3aHKAC+R9Pdgpdg2kmK3+bVmOxbQJKfU0s3sD9fElZJmLuLLMPb835z5hbVv44fKJVuEc7ad2uL3d1AUCbq3MKRCb38t77L4Ifa/ob3QQ==',
MD5OfBody: '7b84813a4b4bf10f0edb9e8da7',
Body: "Handsome Person Basic Information."
}
]
}
My Expected Output:
{
ResponseMetadata: { RequestId: 'a9e53cee-afc8-57c-664408f1e602' },
Messages: [
{
MessageId: 'bb3e23f9-07fd-4205-9b53-a48f826',
ReceiptHandle: 'AQEBD6rCOZlaBfCNn3nU4AEE7OlYwFJFeblTaiDoxIy8HiwsjdxZX+3SICY/YW5PI+RuFscMMh6VyExoo1i8Zo2JlbYj3t32b9CXnToYugzBqgZuxuYOTzXRAnrGwlavSL7hcLQvW6y8me1gnj65N3tPYEmcfXX5GIiQTn1yNEou3rUNff9DfkSije/0zvp33yfWfcW+RDzB2y6ND6eKHxfsP/cqmHjRaT0bE9rlXorjgh36YwVJ57e5bjUa/1dVqOf3ybXfEX/5C2eZM+T1V2JBxlguvuL1B3aHKAC+R9Pdgpdg2kmK3+bVmOxbQJKfU0s3sD9fElZJmLuLLMPb835z5hbVv44fKJVuEc7ad2uL3d1AUCbq3MKRCb38t77L4Ifa/ob3QQ==',
MD5OfBody: '7b84813a4b4bf10f0edb9e8da7',
Body: "Handsome Person Basic Information."
},
{
Body: "Handsome Person Basic Information II"
},
{
Body: "Handsome Person Basic Information III"
}
]
}
Under messages, i should be getting three. but retrieving is only one message.
Take note: I have just short cut the value of my expected value just to prove my point.
How would I able to get all of the three messages? Thanks !
This is the default behavior of AWS SQS
"If the number of messages in the queue is small (fewer than 1,000), you most likely get fewer messages than you requested per ReceiveMessage call. If the number of messages in the queue is extremely small, you might not receive any messages in a particular ReceiveMessage response. If this happens, repeat the request. "
Class ReceiveMessageCommand
Long polling might help, but not guarantee you will receive all message at once
Long polling is the solution. You can achieve by 2 ways :
Recall your "readMessage()" recursively(I won't recommend to use setInterval)
Use npm SQS-consumer
I'm trying to build an application with a basic client-server infrastructure. The server infrastructure is hosted on AWS, and when a client logs on, it sends a message to the server to set up various infrastructure considerations. One of the pieces of infrastructure is an SQS Queue that the client can poll from to get updates from the server (eventually I'd like to build a push service but I don't know how for right now).
I'm building this application in NodeJS using the Node AWS SDK. The problem I'm having is I need the queue ARN to do various things like subscribe the SQS queue to an SNS topic that the application uses, but the create queue API returns the queue URL, not ARN. So I can get the ARN from the URL using the getQueueAttributes API, but it doesn't seem to be working. Whenever I call it, I get undefined as the response. Here's my code, please tell me what I'm doing wrong:
exports.handler = (event, context, callback) => {
new aws.SQS({apiVersion: '2012-11-05'}).createQueue({
QueueName: event.userId
}).promise()
)
.then(data => { /* This has the Queue URL */
new aws.SQS({apiVersion: '2012-11-05'}).getQueueAttributes({
QueueUrl: data.QueueUrl,
AttributeNames: ['QueueArn']
}).promise()
})
.then(data => {
console.log(JSON.stringify(data)); /* prints "undefined" */
})
/* Some more code down here that's irrelevant */
}
Thanks!
const AWS = require('aws-sdk');
const sqs = new AWS.SQS();
exports.handler = async(event, context, callback) => {
var params = {
QueueUrl: 'my-queue-url',
AttributeNames: ['QueueArn']
};
let fo = await sqs.getQueueAttributes(params).promise();
console.log(fo);
};
and it printed
{
ResponseMetadata: { RequestId: '123456-1234-1234-1234-12345' },
Attributes: {
QueueArn: 'arn:aws:sqs:eu-west-1:12345:my-queue-name'
}
}
With the help of Ersoy, I realized that I was using block-formatting (with {}) to write my Promises, but I was never returning anything from those blocks. I had thought that the last value in the Promise block was the return value by default, but it seems that was not the case. When I added return before the SQS API command, then it worked (without using async/await).
Created an issue on nexmo/nexmo-node repo, but posting here nevertheless for a greater reach
I've deployed a phone number verifier lambda function through serverless. The nexmo.message.sendSms is working without an issue on my local dev environment (tested it with serverless offline). But I don't think it's working after deployed to AWS.
Here's what I'm doing:
const Nexmo = require('nexmo');
const privateKey = require('./privateKey');
const getNexmoInstance = (environment) => {
const nexmo = new Nexmo({
apiKey: environment.nexmoAPIKey,
apiSecret: environment.nexmoAPISecret,
applicationId: environment.nexmoAPPId,
privateKey: Buffer.from(privateKey.key),
}, {
debug: true,
});
return nexmo;
};
const sendSMS = async (from, to, text, environment) => {
const nexmo = getNexmoInstance(environment);
nexmo.message.sendSms(from, to, text, { type: 'unicode' }, (err, responseData) => {
console.log('nexmo err', err);
console.log('nexmo responseData', responseData);
});
return null;
};
See that log messages inside the callback? (console.log('nexmo err', err);, etc). They're not showing up after deployed (As I said, working fine on local).
Since I set the debug: true, it's logging the following:
- info: sending message from +44750....
- info: Request: { host: 'rest.nexmo.com', port: 443....
But the callback is not getting called, no message log is registered on the Nexmo dashboard too. Is there any additional serverless related config that I'm unaware of?
Nexmo package version: 2.6.0
Runtime: nodejs12.x
Well from what I can see the issue is with the return null statement which returns before the callback is being fired and that's why you don't see any results.
also I don't see any reason to use async as well.
const sendSMS = (from, to, text, environment) => {
const nexmo = getNexmoInstance(environment);
return nexmo.message.sendSms(from, to, text, { type: 'unicode' }, (err, responseData) => {
if(err) console.error(err);
console.info(responseData);
return;
});
};
should do the trick.
if you could share your lambda handler/config it might be even easier to help you perfect this endpoint/function.
I'm running a "Node.JS" lambda on AWS that sends a message to SQS.
For some reason the SQS callback function get execute only once every couple of calls. It's looks like that the thread that running the lambda finish the run (because it's not a synchronous call to SQS and also can't return a Future) and therefore the lambda doesn't "stay alive" for the callback to get executed.
How can I solve this issue and have the lambda wait for the SQS callback to get execute?
Here is my lambda code:
exports.handler = async (event, context) => {
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create an SQS service object
var sqs = new AWS.SQS({apiVersion: '2012-11-05'});
const SQS_QUEUE_URL = process.env.SQS_QUEUE_URL;
var params = {
MessageGroupId: "cv",
MessageDeduplicationId: key,
MessageBody: "My Message",
QueueUrl: SQS_QUEUE_URL
};
console.log(`Sending notification via SQS: ${SQS_QUEUE_URL}.`);
sqs.sendMessage(params, function(err, data) { //<-- This function get called about one time every 4 lambda calls
if (err) {
console.log("Error", err);
context.done('error', "ERROR Put SQS");
} else {
console.log("Success", data.MessageId);
context.done(null,'');
}
});
};
You should either stick to callback based approach, or to promise based one. I recommend you to use the latter:
exports.handler = async (event, context) => {
// Set the region
AWS.config.update({region: 'us-east-1'});
// Create an SQS service object
var sqs = new AWS.SQS({apiVersion: '2012-11-05'});
const SQS_QUEUE_URL = process.env.SQS_QUEUE_URL;
var params = {
MessageGroupId: "cv",
MessageDeduplicationId: key,
MessageBody: "My Message",
QueueUrl: SQS_QUEUE_URL
};
console.log(`Sending notification via SQS: ${SQS_QUEUE_URL}.`);
try {
await sqs.sendMessage(params).promise(); // since your handler returns a promise, lambda will only resolve after sqs responded with either failure or success
} catch (err) {
// do something here
}
};
P.S. Instantiating aws classes in the handler is not a good idea in lambda environment, since it increases the cold start time. It's better to move new AWS.SQS(...) action out of handler and AWS.config.update() too, since these actions will be executed on each call of the handler, but you really need them to be executed only once.
According to this aws doc http://docs.aws.amazon.com/general/latest/gr/api-retries.html automatic retry feature is build in the aws sdk in my case node.js aws sdk. I configured the DocumentClient object like this:
var dynamodb = new AWS.DynamoDB.DocumentClient({
region: 'us-west-2',
retryDelayOptions: {base: 50},
maxRetries: 20
});
but I still cannot make it auto-retry for me. I want to auto-retry with all UnprocessedItems as well.
Can you point me to where is my mistake?
Thanks
The retryDelayOptions and maxRetries are the options present on AWS.DynamoDB. The DocumentClient has to be configured by setting the DynamoDB service.
var dynamodb = new AWS.DynamoDB({maxRetries: 5, retryDelayOptions: {base: 300} });
var docClient = new AWS.DynamoDB.DocumentClient({service : dynamodb});
The AWS Client SDKs all have built-in mechanisms for retry indeed, however those retries are at the request level. That means that any request that gets rejected by the server with a 500-level error, or in some cases, a 400-level throttling error will get automatically retried based on the configured settings.
What you are asking for is business-layer retry behavior which is NOT built into the SDK. The UnprocessedItems collection contains items that were rejected by the service for various reasons and you have to write your own logic to handle those.
After sending Response we can handle unprocessed Item's background Process until all unprocessed Items should be complete. below code is useful for you
var AWS= require('aws-sdk');
var docClient = new AWS.DynamoDB.DocumentClient();
router.post('/someBatchWrites',(req,res)=>{
docClient.batchWrite(params, function (error, data) {
res.send(error, data);
handler(error, data);//handling unprocessed items /back ground
})
});
//handle Method
function handler(err, data) {
if (err) {
console.log("Error", err);
} else {
console.log("Success", data);
if (Object.keys(data.UnprocessedItems).length) {
setTimeout(() => { docClient.batchWrite({ RequestItems: data.UnprocessedItems }, handler);
}, 100000);
}
}
}