I am building a NodeJS app that connects to a Redis cluster using the ioredis module.
Overview of the issue:
A lookup query for a key that doesn't exist in the cache returns an empty object as expected
A lookup query for a key that DOES exist cache is throwing an error but the error is undefined.
I created a controller to create and manage the connection:
const redis = require('ioredis');
const consoleLogger = require('../logger/logger.js').console;
const redisCtrl = {};
redisCtrl.defaultPrefix = 'lookup:';
// Local variable to store the full time connection to Redis for lookups
let _redisClient;
// Redis connection config
redisCtrl.redisConnect = {
port: process.env.REDIS_PORT || 6379,
host: process.env.REDIS_HOST,
password: process.env.REDIS_PASSWORD
};
// Redis config
redisCtrl.redisConfig = {
dnsLookup: (address, callback) => callback(null, address),
redisOptions: {
tls: process.env.REDIS_SSL === 'false' ? false : true,
password: process.env.REDIS_PASSWORD,
maxRetriesPerRequest: 1
}
};
// Retrieve the redis connection
redisCtrl.redisClient = async () => {
if (!_redisClient) {
_redisClient = await redisCtrl.getNewConnect();
}
return _redisClient;
}
redisCtrl.getNewConnect = async () => {
let makeConnection;
if (process.env.REDIS_CLUSTER === 'true') {
makeConnection = new redis.Cluster([redisCtrl.redisConnect], redisCtrl.redisConfig);
} else {
makeConnection = new redis(redisCtrl.redisConnect);
}
makeConnection.on("connect", function (err) {
if (!err) {
consoleLogger.info("REDIS connected");
} else {
consoleLogger.info("REDIS connection error");
consoleLogger.error(JSON.stringify(err));
}
});
makeConnection.on("error", function (error) {
consoleLogger.info("REDIS error");
consoleLogger.error(JSON.stringify(error));
throw new Error(error);
});
return makeConnection;
}
redisCtrl.closeInstance = (cb) => {
if (_redisClient) {
_redisClient.quit(cb);
}
}
module.exports = redisCtrl;
This works to establish the connection.
However, when attempting to get a result, an empty error is thrown from the hgetall method.
/**
* Lookup asset by assetId in Redis cache
* Return asset data object
* #param {str} assetId
*/
assetsCtrl.lookupByAssetId = async (assetId) => {
// Prepend default cache prefix to lookup value
const lookupKey = `${redisPrefix || `lookup:`}${assetId}`;
let cachedAsset;
try {
cachedAsset = await assetsCtrl.redisClient.hgetall(lookupKey);
} catch (e) {
consoleLogger.error(`Lookup by assetId failed. Lookup key: ${lookupKey}`);
consoleLogger.error(e);
throw new Error(e);
}
return cachedAsset;
}
The error is thrown but the error is undefined. The "catch" block of the redisClient.hgetall(lookupKey) line is getting called but the error is not defined.
error: Lookup by assetId failed. Lookup key: lookup:test123456789
**error: undefined {"command":{"name":"hget","args":["lookup:test123456789"]}}**
Questions: How can I troubleshoot this issue? How can I see the details of the error that is being thrown?
As stated in the comments above, the hgetall() is not working because the datatype corresponding to the lookup value was not hash.
Changing it to get() has fixed the issue.
Related
I'm trying to use the code from this answer to extend one OSS application.
However app.js is sync and no matter what I do, I cant force it to wait for the promise to resolve.
app.js
var cosmos = require('./cosmos.js');
const key = cosmos.key(var1, var2, var3);
console.log(key); // << shows Promise { <pending> }
mongoose.connect(`redacted`, {
auth: {
username: config.database.name,
password: key
}
});
cosmos.js
async function retriveKey(subId, resGrp, server) {
const { EnvironmentCredential } = require("#azure/identity");
const { CosmosDBManagementClient } = require("#azure/arm-cosmosdb");
const armClient = new CosmosDBManagementClient(
new EnvironmentCredential(), subId
);
const { primaryMasterKey } = await armClient.databaseAccounts.listKeys(
resGrp, server
);
return new Promise(resolve => {
setTimeout(() => resolve(primaryMasterKey), 1000);
});
}
exports.key = retriveKey
If i console.log() inside the async function it actually shows the key, however mongoose db connection doesn't wait for the promise to get resolved, it starts connecting straight away and fails with something like: password must be a string.
If i hardcode actual key instead of this promise - everything works fine.
EDIT:
halfway there:
// pull cosmos keys
async function retriveKey(subId, resGrp, server) {
const { EnvironmentCredential } = require("#azure/identity");
const { CosmosDBManagementClient } = require("#azure/arm-cosmosdb");
const armClient = new CosmosDBManagementClient(
new EnvironmentCredential(), subId
);
const { primaryMasterKey } = await armClient.databaseAccounts.listKeys(
resGrp, server
);
return primaryMasterKey // don't even need a promise here
}
exports.key = retriveKey
var mongooseConnected; // global variable
app.use(function (req, res, next) {
if (!moongooseConnected) {
moongooseConnected = cosmos.key(var1, var2, var3).then(function (key) {
mongoose.connect(`xxx`,
{
auth: {
username: config.database.name,
password: key
}
}
);
console.log(key); // works as expected
require('./models/user');
require('./models/audit');
require('./routes/user')(app);
require('./routes/audit')(app, io);
});
}
moongooseConnected.then(function () {
next();
});
});
the database connection gets established, console.log(key) shows proper key in the log, however no routes are present in the app.
if i move routes or models outside of this app.use(xyz) - i'm starting to see failures due to:
Connection 0 was disconnected when calling createCollection
or
MongooseError [MissingSchemaError]: Schema hasn't been registered for model "User".
which (i assume) means they require mongoose to be instantiated, but they are not waiting.
If you switch from CommonJS modules to ES modules, you can use await to wait for a promise to resolve:
import cosmos from './cosmos.js';
const key = await cosmos.key(var1, var2, var3);
console.log(key);
await mongoose.connect(`redacted`, {
auth: {
username: config.database.name,
password: key
}
});
Alternatively, you can wait with the initialization of mongoose until the first request comes in, because express middleware is asynchronous:
var mongooseConnected; // global variable
function connectMongoose() {
if (!mongooseConnected)
mongooseConnected = cosmos.key(var1, var2, var3)
.then(key => mongoose.connect(`redacted`, {
auth: {
username: config.database.name,
password: key
}
}));
return mongooseConnected;
}
module.exports = connectMongoose;
If the code above is needed elsewhere, it can be put in a separate module and imported wherever needed:
const connectMongoose = require("./connectMongoose");
app.use(function(req, res, next) {
connectMongoose().then(function() {
next();
});
});
require('./routes/user')(app);
require('./routes/audit')(app, io);
Note that if several parallel requests come in, only the first of these will let the global variable mongooseConnected equal a promise, and all these requests will wait for it to resolve before calling next().
Also note that additional routes of app must be registered after this app.use command, not inside it.
unless somebody comes up with a way to do this with less changes to the original code base, this is what I'm using:
cosmos.js
// pull cosmos keys
async function retriveKey(subId, resGrp, server) {
const { DefaultAzureCredential } = require("#azure/identity");
const { CosmosDBManagementClient } = require("#azure/arm-cosmosdb");
const armClient = new CosmosDBManagementClient(
new DefaultAzureCredential(), subId
);
const { primaryMasterKey } = await armClient.databaseAccounts.listKeys(
resGrp, server
);
return primaryMasterKey
}
exports.key = retriveKey
app.js
// pull cosmos keys
var cosmos = require('./cosmos');
let key = cosmos.key(var1, var2, var3)
mongoose.connect(
`xxx`,
{
auth: {
username: config.database.name,
password: key
}
}
).catch(
err => {
console.log("dOrty h4ck");
key.then(k => mongoose.connect(
`xxx`,
{
auth: {
username: config.database.name,
password: k
}
}
)
);
}
)
basically, like Heiko mentioned, mongoose.connect() is actually async, but somehow blocking (??). so while first mongoose.connect() always fails - it gives enough time for the code to retrieve the key, then I catch the error and connect again. no other changes to the original code base are needed.
I am using a Node JS application that has Oracle Autonomous Database as a backend.
It works perfectly on my local machine and I am able to connect it well without any issue.
I tried deploying Node JS project to Azure WebAppService on Linux server.
Initially after deployment I my project was not able to find the Oracle client and so after searching a lot I was able to fix that problem by below
steps
with this I was able to solve client issue.
I have wallet files which I received from oracle which I have placed in admin folder
but now the problem is when I make any request I am getting this error
data:{"message":"db.doConnect is not a function","stack":"TypeError:
db.doConnect is not a `function\n`
createPool() callback: ORA-28759: failure to open file
my code:
// Include all external dependencies
const oracledb = require('oracledb');
// Intialize variables
const numRows = 100;
let respArr = [];
let connectionObject;
async function initialize(envName) {
await oracledb.createPool({
user: process.env.DATABASEUSERNAME,
password: process.env.DATABASEPASSWORD,
connectString: process.env.DATABASECONNECTIONSTRING,
});
}
async function close(poolAlias) {
await oracledb.getPool(poolAlias).close();
}
// Function to iterate through all the rows fetched in the result set and resturn the same
async function fetchRowsFromRS(connection, resultSet, numRows) {
// Get the rows
try {
const rows = await resultSet.getRows(numRows);
// no rows, or no more rows, then close the result set
if (rows.length === 0) {
console.log('No rows returned');
// doClose(connection, resultSet);
} else if (rows.length > 0) {
console.log(`Got ${rows.length} rows`);
respArr = respArr.concat(rows);
// Call the function recursively to get more rows
await fetchRowsFromRS(connection, resultSet, numRows);
}
// Return the rows
return respArr;
} catch (err) {
console.log(err);
}
}
async function simpleExecute(statement, binds = [], numberOutCur, poolAlias, opts = {}) {
try {
await initialize();
opts.outFormat = oracledb.OBJECT;
opts.autoCommit = true;
connectionObject = await oracledb.getConnection(poolAlias);
const finalResult = {};
const result = await connectionObject.execute(statement, binds, opts);
let promises = [];
for (let idx = 0; idx < numberOutCur; idx++) {
const refCurName = `ref_cur_${idx}`;
promises.push(fetchRowsFromRS(connectionObject, result.outBinds[refCurName], numRows));
const resultRows = await Promise.all(promises);
respArr = [];
finalResult[refCurName] = resultRows;
promises = [];
}
return finalResult;
// const values = await Promise.all(promises);
// return values;
} catch (error) {
return error;
} finally {
if (connectionObject) {
try {
await connectionObject.close();
} catch (err) {
console.log(err);
}
}
}
}
// Function to release the connection
function doRelease(connection) {
connection.close(
(err) => {
if (err) {
console.log(err.message);
}
},
);
}
// Function to close the result set connection
function doClose(connection, resultSet) {
resultSet.close(
(err) => {
if (err) { console.log(err.message); }
doRelease(connection);
},
);
}
// Export functions
module.exports.simpleExecute = simpleExecute;
module.exports.initialize = initialize;
module.exports.close = close;
I call my procs using this
const allProducts = await dbSvc.simpleExecute(query, cart_data_binds, 1,
'default');
what I understood with this message is I am not able to connect to my cloud database and I am not sure how to solve this can anyone help me with it its been 2 weeks now with this problem.
In Node JS project I am using simpleoracle library to connect my oracle cloud anonymous database
Thank you, Christopher Jones and saiharika213. Posting your suggestions as an answer to help other community members.
This ORA-28759: failure to open file error seems you need to update the WALLET_LOCATION directory in sqlnet.ora. You can refer to Connecting to Oracle Cloud Autonomous Databases
You can resolve this error by changing the connection string to point to the wallet location.
Move the wallets to respective project folder and modify the connection string as below in dbConfig.js.
Provide the wallet path from root till wallet folder.
For example:
module.exports = {
user: username,
password:password,
connectString:"(DESCRIPTION = (ADDRESS = (PROTOCOL = TCPS)(HOST = Hostname )(PORT = XXXX))(CONNECT_DATA = (SERVER = DEDICATED)(SERVICE_NAME = Servicename))(SECURITY=(MY_WALLET_DIRECTORY=/Users/yharika/Node_Projects/invoice_app/config/wallet)))"
}
You can refer to Unable to connect to oracle database from node js using sqlnet.ora with oracle wallet and Connecting with TCPS - ora-28759: failure to open file
The official documentation of the Node.js Driver version 3.6 contains the following example for the .find() method:
const { MongoClient } = require("mongodb");
// Replace the uri string with your MongoDB deployment's connection string.
const uri = "mongodb+srv://<user>:<password>#<cluster-url>?w=majority";
const client = new MongoClient(uri);
async function run() {
try {
await client.connect();
const database = client.db("sample_mflix");
const collection = database.collection("movies");
// query for movies that have a runtime less than 15 minutes
const query = { runtime: { $lt: 15 } };
const options = {
// sort returned documents in ascending order by title (A->Z)
sort: { title: 1 },
// Include only the `title` and `imdb` fields in each returned document
projection: { _id: 0, title: 1, imdb: 1 },
};
const cursor = collection.find(query, options);
// print a message if no documents were found
if ((await cursor.count()) === 0) {
console.log("No documents found!");
}
await cursor.forEach(console.dir);
} finally {
await client.close();
}
}
To me this somewhat implies that I would have to create a new connection for each DB request I make.
Is this correct? If not, then what is the best practise to keep the connection alive for various routes?
You can use mongoose to set a connection with your database.
mongoose.connect('mongodb://localhost:27017/myapp', {useNewUrlParser: true});
then you need to define your models which you will use to communicate with your DB in your routes.
const MyModel = mongoose.model('Test', new Schema({ name: String }));
MyModel.findOne(function(error, result) { /* ... */ });
https://mongoosejs.com/docs/connections.html
It's 2022 and I stumbled upon your post because I've been running into the same issue. All the tutorials and guides I've found so far have setups that require reconnecting in order to do anything with the Database.
I found one solution from someone on github, that creates a class to create, save and check if a client connection exist. So, it only recreates a client connection if it doesn't already exist.
const MongoClient = require('mongodb').MongoClient
class MDB {
static async getClient() {
if (this.client) {
return this.client
}
this.client = await MongoClient.connect(this.url);
return this.client
}
}
MDB.url='<your_connection_url>'
app.get('/yourroute', async (req, res) => {
try {
const client = await MDB.getClient()
const db = client.db('your_db')
const collection = db.collection('your_collection');
const results = await collection.find({}).toArray();
res.json(results)
} catch (error) {
console.log('error:', error);
}
})
I am maintaining the bot state in a local MongoDB storage. When I am trying to hand-off the conversation to an agent using directline-js, it shows an error of BotFrameworkAdapter.sendActivity(): Missing Conversation ID. The conversation ID is being saved in MongoDB
The issue is arising when I change the middle layer from Array to MongoDB. I have already successfully implemented the same bot-human hand-off using directline-js with an Array and the default Memory Storage.
MemoryStorage in BotFramework
const { BotFrameworkAdapter, MemoryStorage, ConversationState, UserState } = require('botbuilder')
const memoryStorage = new MemoryStorage();
conversationState = new ConversationState(memoryStorage);
userState = new UserState(memoryStorage);
Middle Layer for Hand-Off to Agent
case '#connect':
const user = await this.provider.connectToAgent(conversationReference);
if (user) {
await turnContext.sendActivity(`You are connected to
${ user.userReference.user.name }\n ${ JSON.stringify(user.messages) }`);
await this.adapter.continueConversation(user.userReference, async
(userContext) => {
await userContext.sendActivity('You are now connected to an agent!');
});
}
else {
await turnContext.sendActivity('There are no users in the Queue right now.');
}
The this.adapter.continueConversation throws the error when using MongoDB.
While using Array it works fine. The MongoDB and Array object are both similar in structure.
Since this works with MemoryStorage and not your MongoDB implementation, I'm guessing that there's something wrong with your MongoDB implementation. This answer will focus on that. If this isn't the case, please provide your MongoDb implementation and/or a link to your repo and I can work off that.
Mongoose is only necessary if you want to use custom models/types/interfaces. For storage that implements BotState, you just need to write a custom Storage adapter.
The basics of this are documented here. Although written for C#, you can still apply the concepts to Node.
1. Install mongodb
npm i -S mongodb
2. Create a MongoDbStorage class file
MongoDbStorage.js
var MongoClient = require('mongodb').MongoClient;
module.exports = class MongoDbStorage {
constructor(connectionUrl, db, collection) {
this.url = connectionUrl;
this.db = db;
this.collection = collection;
this.mongoOptions = {
useNewUrlParser: true,
useUnifiedTopology: true
};
}
async read(keys) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
const data = {};
await Promise.all(keys.map(async (key) => {
const doc = await col.findOne({ _id: key });
data[key] = doc ? doc.document : null;
}));
return data;
} finally {
client.close();
}
}
async write(changes) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
await Promise.all(Object.keys(changes).map((key) => {
const changesCopy = { ...changes[key] };
const documentChange = {
_id: key,
document: changesCopy
};
const eTag = changes[key].eTag;
if (!eTag || eTag === '*') {
col.updateOne({ _id: key }, { $set: { ...documentChange } }, { upsert: true });
} else if (eTag.length > 0) {
col.replaceOne({ _id: eTag }, documentChange);
} else {
throw new Error('eTag empty');
}
}));
} finally {
client.close();
}
}
async delete(keys) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
await Promise.all(Object.keys(keys).map((key) => {
col.deleteOne({ _id: key });
}));
} finally {
client.close();
}
}
async getClient() {
const client = await MongoClient.connect(this.url, this.mongoOptions)
.catch(err => { throw err; });
if (!client) throw new Error('Unable to create MongoDB client');
return client;
}
async getCollection(client) {
return client.db(this.db).collection(this.collection);
}
};
Note: I've only done a little testing on this--enough to get it to work great with the Multi-Turn-Prompt Sample. Use at your own risk and modify as necessary.
I based this off of a combination of these three storage implementations:
memoryStorage
blobStorage
cosmosDbStorage
3. Use it in your bot
index.js
const MongoDbStorage = require('./MongoDbStorage');
const mongoDbStorage = new MongoDbStorage('mongodb://localhost:27017/', 'testDatabase', 'testCollection');
const conversationState = new ConversationState(mongoDbStorage);
const userState = new UserState(mongoDbStorage);
I am getting this huge error every 10-15 minutes at my Node server when it try to communicate with a Java server. Each time i'm getting this error, I can't continue working, and need to restart the node server to solve the issue.
Here is the error:
AssertionError [ERR_ASSERTION] The expression evaluated to a falsy value: assert(this.pending.length === 0) AssertionError [ERR_ASSERTION]: The expression evaluated to a falsy value: assert(this.pending.length === 0) at Channel.C.sendOrEnqueue (/opt/osr/client_dev/node_modules/amqplib/lib/channel.js:73:5) at Channel.C._rpc (/opt/osr/client_dev/node_modules/amqplib/lib/channel.js:136:8) at /opt/osr/client_dev/node_modules/amqplib/lib/channel_model.js:59:17 at tryCatcher (/opt/osr/client_dev/node_modules/bluebird/js/release/util.js:16:23) at Function.Promise.fromNode.Promise.fromCallback (/opt/osr/client_dev/node_modules/bluebird/js/release/promise.js:185:30) at Channel.C.rpc (/opt/osr/client_dev/node_modules/amqplib/lib/channel_model.js:58:18) at Channel.C.assertQueue (/opt/osr/client_dev/node_modules/amqplib/lib/channel_model.js:86:15) at Promise (/opt/osr/client_dev/src/services/rabbitmq.service.js:143:52) at new Promise () at RabbitMQService.produceMessage (/opt/osr/client_dev/src/services/rabbitmq.service.js:99:16) at StreamService.requestStreamData (/opt/osr/client_dev/src/services/stream.service.js:1373:53) at StreamRoute.router.post (/opt/osr/client_dev/src/routes/stream.route.js:113:66) at Layer.handle [as handle_request] (/opt/osr/client_dev/node_modules/express/lib/router/layer.js:95:5) at next (/opt/osr/client_dev/node_modules/express/lib/router/route.js:137:13) at module.exports (/opt/osr/client_dev/src/middlewares/validate-authentication.js:30:13) at process._tickCallback (internal/process/next_tick.js:68:7)
And here is my code:
// This method produce a message to the RabbitMQ.
produceMessage(message, fieldNameToExcludeForLog = null) {
return new Promise(async (resolve, reject) => {
const {
rabbitChannel,
routingkey,
exchangeName,
queueName
} = this;
// Verify that the message data is not empty.
if (!message) {
reject('No message object was provided (1000168)');
}
// Log the send message data.
logUtils.logObjectData({
logLevel: LogLevel.INFO,
applicationType: ApplicationType.SERVER,
upperTitle: 'RabbitMQ Producer - Send message',
lowerTitle: 'message',
objectData: textUtils.getObjectKeyValueRecursive({
object: message
})
});
// Convert the message in JSON file.
const requestJSON = JSON.stringify(message);
// Generate a unique correlationId.
const correlationId = textUtils.getCreateUniqueGuid(configurations.RABBITMQ_CORRELATION_ID_GUID_LENGTH);
if (!correlationId) {
reject('No correlationId parameter was provided (1000169)');
}
try {
// Create the assert exchange and bind to the queue.
rabbitChannel.assertExchange(exchangeName, 'direct', {
durable: true
});
rabbitChannel.bindQueue(queueName, exchangeName, routingkey);
// Create the assert queue.
const q = await this.rabbitChannel.assertQueue('', {
exclusive: true
});
// Publish the message to the exchange.
rabbitChannel.publish(exchangeName, routingkey, Buffer.from(requestJSON), {
correlationId: correlationId,
replyTo: q.queue
});
// Register for the response event.
rabbitChannel.consume(q.queue, (response) => {
// Validate the data existence.
if (!response) {
reject('No response object was provided (1000170)');
}
if (!response.content) {
reject('No response object was provided (1000171)');
}
// Check if the correlationId is identical.
if (response.properties.correlationId !== correlationId) {
reject('Invalid correlationId message (1000172)');
}
const parsedMessage = (x => {
try {
return JSON.parse(x);
}
catch (ex) {
reject({ message: x.toString() });
}
})(response.content);
// This object holds the data to log.
let objectLog = null;
// Determine if to log a shallow copy of the parsedMessage, or the full parsedMessage data.
// Create a shallow copy (without the main data) to log.
if (fieldNameToExcludeForLog) {
objectLog = coreUtils.deleteObjectPath({
object: parsedMessage,
path: fieldNameToExcludeForLog
});
}
else {
objectLog = parsedMessage;
}
// Log the received message data.
logUtils.logObjectData({
logLevel: LogLevel.INFO,
applicationType: ApplicationType.SERVER,
upperTitle: 'RabbitMQ Producer - Receive message',
lowerTitle: 'objectLog',
objectData: textUtils.getObjectKeyValueRecursive({
object: objectLog
})
});
resolve(parsedMessage);
}, { noAck: false });
} catch (error) {
if (error) {
(async () => {
await this.close();
this.rabbitConnection = null;
this.rabbitChannel = null;
this.rabbitConnection = await this.connect();
this.rabbitChannel = await this.createChannel(this.rabbitConnection);
});
reject(error);
}
}
});
}
According to the stack error log, the line of the exception is the following:
// Create the assert queue.
const q = await this.rabbitChannel.assertQueue('', {
exclusive: true
});
Any idea what can I do to solve this?