I want my application (lets say a simple node file for now) to work as it is even if redis is not available. I'm not able to do it the correct way. This is what I've tried.
var redis = require('redis');
var redisClient = null;
var getRedisClient = function(){
if(redisClient){
return redisClient;
}
try {
redisClient = redis.createClient({connect_timeout : 5000, max_attempts : 1});
redisClient.on("error", function(err) {
console.error("Error connecting to redis", err);
redisClient = null;
});
return redisClient;
} catch(ex){
console.log("error initialising redis client " + ex);
return null;
}
};
try {
var client = getRedisClient();
console.log("done!");
} catch (ex){
console.log("Exception");
}
However, with this code my application exits if redis is not available (it shouldn't because i've not given a process.exit() command).
How can I solve this?
Checking for Successful Connection on Start
Using a promise, you could guarantee that at least initially, you were able to connect to redis without error within a specified time period:
const redis = require('redis');
const Promise = require('bluebird');
function getRedisClient(timeoutMs){
return new Promise((resolve, reject) => {
const redisClient = redis.createClient();
const timer = setTimeout(() => reject('timeout'), timeoutMs);
redisClient.on("ready", () => {
clearTimeout(timer);
resolve(redisClient);
});
redisClient.on("error", (err) => {
clearTimeout(timer);
reject(err);
});
});
};
const redisReadyTimeoutMs = 10000;
getRedisClient(redisReadyTimeoutMs)
.then(redisClient => {
// the client has connected to redis sucessfully
return doSomethingUseful();
}, error => {
console.log("Unable to connect to redis", error);
});
You Need Proper Error Handling
The redis client being non-null does NOT guarantee using it won't throw an error.
you could experience infrastructure misfortune e.g. crashed redis process, out of memory or network being down.
a bug in your code could cause an error e.g. invalid or missing arguments to a redis command.
You should be handling redis client errors as a matter of course.
DON'T null the Redis Client on Error
It won't give you much but it will force you to check for null every time you try and use it.
The redis client also has inbuilt reconnect and retry mechanisms that you'll miss out on if you null it after the first error. See the redis package docs, look for retry_strategy.
DO Wrap your redis client code with try .. catch ... or use .catch in your promise chain.
DO Make use of a retry_strategy.
Related
I have a lambda function using Node 12.
I need to add a new connection to a Redis database hosted in AWS ElastiCache.
Both are in one private VPC and the security groups/subnets are configured properly.
Solution:
globals.js:
const redis = require('redis');
const redisClient = redis.createClient(
`redis://${process.env.REDIS_HOST}:${process.env.REDIS_PORT}/${process.env.REDIS_DB}`,
);
redisClient.on('error', (err) => {
console.log('REDIS CLIENT ERROR:' + err);
});
module.exports.globals = {
REDIS: require('../helpers/redis')(redisClient),
};
index.js (outside handler):
const { globals } = require('./config/globals');
global.app = globals;
const lambda_handler = (event, context, callback) => { ... }
exports.handler = lambda_handler;
helpers/redis/index.js:
const get = require('./get');
module.exports = (redisClient) => {
return {
get: get(redisClient)
};
};
helpers/redis/get.js:
module.exports = (redisClient) => {
return (key, cb) => {
redisClient.get(key, (err, reply) => {
if (err) {
cb(err);
} else {
cb(null, reply);
}
});
};
};
Function call:
app.REDIS.get(redisKey, (err, reply) => {
console.log(`REDIS GET: ${err} ${reply}`);
});
Problem:
When increasing lambda timeout to a value greater than Redis timeout, I get this error:
REDIS CLIENT ERROR:Error: Redis connection to ... failed - connect ETIMEDOUT ...
Addition:
I tried quiting/closing the connection after each transaction:
module.exports = (redisClient) => {
return (cb) => {
redisClient.quit((err, reply) => {
if (err) {
cb(err);
} else {
cb(null, reply);
}
});
};
};
app.REDIS.get(redisKey, (err, reply) => {
console.log(`REDIS GET: ${err} ${reply}`);
if (err) {
cb(err);
} else {
if (reply) {
app.REDIS.quit(() => {
cb()
});
}
}
})
Error:
REDIS GET: AbortError: GET can't be processed. The connection is already closed.
Extra Notes:
I have to use callbacks, this is why I pass ones in the above examples
I'm using "redis": "^3.0.2"
It's not a configuration issue as the cache was accessed hundred of times in a small period of time but it then started giving the timeout errors.
Everything works normally locally
It's not a configuration issue as the cache was accessed hundred of times in a small period of time but it then started giving the timeout errors.
i think it is origin of issue, probably redis database size hit the size limit, and it cannot process new data?
Can you delete old data in it?
Also it is possible Elastic Cache has limits on new TCP clients' connections, and if its depleted, new connections are refused with similar error message you mentioned.
If redis client in aws lambda function cannot establish connection, aws lambda function fails - and new one is started. New lambda function makes one more connection to redis, redis cannot process it, and one more lambda function is started...
So, at one moment, we hit the limit on active redis connections, and system is in deadlock.
I think you can temporary stop all lambda functions, and scale up Elastic Cache redis database.
I'm trying to do a get() from my AWS Lambda (NodeJS) on ElastiCache Redis using node_redis client. I believe that I'm able to connect to redis but I'm getting Time out (Lambda 60 sec time out) when I'm trying to perform a get() operation.
I have also granted my AWS lambda Administrator access just to be certain that it's not a permissions issue. I'm hitting lambda by going to AWS console and clicking the Test button.
Here is my redisClient.js:
const util = require('util');
const redis = require('redis');
console.info('Start to connect to Redis Server');
const client = redis.createClient({
host: process.env.ElastiCacheEndpoint,
port: process.env.ElastiCachePort
});
client.get = util.promisify(client.get);
client.set = util.promisify(client.set);
client.on('ready',function() {
console.log(" subs Redis is ready"); //Can see this output in logs
});
client.on('connect',function(){
console.log('subs connected to redis'); //Can see this output in logs
})
exports.set = async function(key, value) {
console.log("called set!");
return await client.set(key, value);
}
exports.get = async function(key) {
console.log("called get!"); //Can see this output in logs
return await client.get(key);
}
Here's my index.js which calls the redisClient.js:
const redisclient = require("./redisClient");
exports.handler = async (event) => {
const params = event.params
const operation = event.operation;
try {
console.log("Checking RedisCache by calling client get") // Can see this output in logs
const cachedVal = await redisclient.get('mykey');
console.log("Checked RedisCache by calling client get") // This doesn't show up in logs.
console.log(cachedVal);
if (cachedVal) {
return {
statusCode: 200,
body: JSON.stringify(cachedVal)
}
} else {
const setCache = await redisclient.set('myKey','myVal');
console.log(setCache);
console.log("*******")
let response = await makeCERequest(operation, params, event.account);
console.log("CE Request returned");
return response;
}
}
catch (err) {
return {
statusCode: 500,
body: err,
};
}
}
This is the output (time out error message) that I get:
{
"errorMessage": "2020-07-05T19:04:28.695Z 9951942c-f54a-4b18-9cc2-119eed65e9f1 Task timed out after 60.06 seconds"
}
I have tried using Bluebird (changing get to getAsync()) per this: https://github.com/UtkarshYeolekar/promisify-redis-client/blob/master/redis.js but still got the same behavior.
I also changed the port to use a random value (like 8088) that I'm using to create client (to see the behavior of connect event for a failed connection) - in this case I still see a Timed Out error response but I don't see the subs Redis is ready and subs connected to redis in my logs.
Can anyone please point me in the right direction? I don't seem to understand why I'm able to connect to redis but the get() request times out.
I figured out the issue and posting here in case it helps anyone in future as the behavior wasn't very intuitive for me.
I had enabled AuthToken param while setting up my redis. I was passing the param to lambda with the environment variables but wasn't using it while sending the get()/set() requests. When I disabled the AuthToken requirement from redis configuration - Lambda was able to hit redis with get/set requests. More details on AuthToken can be found here: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticache-replicationgroup.html#cfn-elasticache-replicationgroup-authtoken
I'd like my Node JS app to exit immediately if it can't connect to Mongo. I'm using the mongodb node library.
I've reduced the code down to
const {MongoClient} = require('mongodb');
MongoClient.connect('mongodb://localhost:27017');
If Mongo is not running, I get an UnhandledPromiseRejectionWarning with ECONNREFUSED, which I fully expect, but then the program hangs and never exits. This is with Node version 10.0.0.
Since the connection never succeeded I don't have a connection handle to close. I've tried various ways to catch the rejected promise, but I have been unsuccessful in getting the program to exit.
What do I need to do to shut down the MongoClient and make the program exit in this case?
Your application is remaining alive because it is trying to reconnect. You can try disabling the recconect:
MongoClient.connect('mongodb://localhost:27017', {
autoReconnect: false
}, (err, client) => {
if (client) client.close();
});
Or, you can terminate the process using process.exit(1) to kill the program.
const {
MongoClient
} = require('mongodb');
// Callback syntax
MongoClient.connect('mongodb://localhost:27017', (err, db) => {
if (err) process.exit(1);
});
// Promise syntax
MongoClient
.connect('mongodb://localhost:27017')
.catch(err => {
process.exit(1);
});
// Async/await syntax
(async function() {
let db;
try {
db = await MongoClient.connect('mongodb://localhost:27017');
} catch (err) {
process.exit(1);
}
}());
I have simple example:
var redis = require('redis'),
client = redis.createClient();
var test = function() {
client.brpop('log', 0, function(err, reply) {
if (err != null ) {
console.log(err);
} else {
.... parse log string ....
}
test();
});
}
test();
How to reconnect redis connection after restart redis server ?
The Redis client automatically reconnects. Just make sure you handle the "error" event from the client. As per the example:
var redis = require('redis');
client = redis.createClient();
client.on('error', function(err){
console.error('Redis error:', err);
});
Otherwise, this code is where the process begins.
this.emit("error", new Error(message));
// "error" events get turned into exceptions if they aren't listened for. If the user handled this error
// then we should try to reconnect.
this.connection_gone("error");
Next, the .connection_gone() method runs on the client.
Notice that you can also listen to a "reconnecting" event to be notified when this happens.
Currently I'm using https://github.com/mranney/node_redis as my node redis client.
client.retry_delay is set to 250ms default.
I tried connecting to redis and once connection was successful, I manually stopped the redis server to see whether client.retry_delay works. But I didn't see it working.
The following log messages are logged on ready & end events on redisClients created using createClient:
[2012-03-30 15:13:05.498] [INFO] Development - Node Application is running on port 8090
[2012-03-30 15:13:08.507] [INFO] Development - Connection Successfully Established to '127.0.0.1' '6379'
[2012-03-30 15:16:33.886] [FATAL] Development - Connection Terminated to '127.0.0.1' '6379'
I didn't see Success message again [ready event was not fired] when the server came back live.
Am I missing something? When will be the retry constant used? Is there a work around to find whether a redis server has come up after a failure from node?
I can't reproduce this. Could you try this code, stop your redis server, and check the log output?
var client = require('redis').createClient();
client.on('connect' , log('connect'));
client.on('ready' , log('ready'));
client.on('reconnecting', log('reconnecting'));
client.on('error' , log('error'));
client.on('end' , log('end'));
function log(type) {
return function() {
console.log(type, arguments);
}
}
Answer # Feb-2020
const redis = require('redis');
const log = (type, fn) => fn ? () => {
console.log(`connection ${type}`);
} : console.log(`connection ${type}`);
// Option 1: One connection is enough per application
const client = redis.createClient('6379', "localhost", {
retry_strategy: (options) => {
const {error, total_retry_time, attempt} = options;
if (error && error.code === "ECONNREFUSED") {
log(error.code); // take actions or throw exception
}
if (total_retry_time > 1000 * 15) { //in ms i.e. 15 sec
log('Retry time exhausted'); // take actions or throw exception
}
if (options.attempt > 10) {
log('10 attempts done'); // take actions or throw exception
}
console.log("Attempting connection");
// reconnect after
return Math.min(options.attempt * 100, 3000); //in ms
},
});
client.on('connect', log('connect', true));
client.on('ready', log('ready', true));
client.on('reconnecting', log('reconnecting', true));
client.on('error', log('error', true));
client.on('end', log('end', true));
For complete running example clone node-cheat and run node connect-retry.js.
Adding to the answer above. Small change. The callback provided should be a method name and not execute the method itself. Something like below:
function redisCallbackHandler(message){
console.log("Redis:"+ message);
}
var redis = require("redis");
var redisclient = redis.createClient();
redisclient.on('connect', redisCallbackHandler);
redisclient.on('ready', redisCallbackHandler);
redisclient.on('reconnecting', redisCallbackHandler);
redisclient.on('error', redisCallbackHandler);
redisclient.on('end', redisCallbackHandler);