Our original code to create redis clients
import Redis, { Redis as RedisClient, Cluster, ClusterOptions } from 'ioredis';
import config from '../../../config';
const {
port, host, cluster, transitEncryption,
} = config.redis;
const retryStrategy = (times: number): number => Math.min(times * 50, 10000);
function GetRedisClient(): RedisClient {
return new Redis({
host,
port: port as number,
retryStrategy,
});
}
function GetRedisClusterClient(): Cluster {
const clusterOptions: ClusterOptions = {
clusterRetryStrategy: retryStrategy,
redisOptions: { db: 0 },
};
if (transitEncryption) {
clusterOptions.dnsLookup = (address, callback) => callback(null, address);
clusterOptions.redisOptions!.tls = {};
}
return new Redis.Cluster([
{
host,
port: port as number,
},
], clusterOptions);
}
function getClient() {
return cluster ? GetRedisClusterClient() : GetRedisClient();
}
export default getClient();
has become this as we upgrade to version 5.0.1 of ioredis:
import Redis, { Cluster, ClusterOptions } from 'ioredis';
import config from '../../../config';
const {
port, host, cluster, transitEncryption,
} = config.redis;
const retryStrategy = (times: number): number => Math.min(times * 50, 10000);
function GetRedisClient(): Redis {
return new Redis({
host,
port: port as number,
retryStrategy,
});
}
function GetRedisClusterClient(): Cluster {
const clusterOptions: ClusterOptions = {
clusterRetryStrategy: retryStrategy,
redisOptions: { db: 0 },
};
if (transitEncryption) {
clusterOptions.dnsLookup = (address, callback) => callback(undefined, address);
clusterOptions.redisOptions!.tls = {};
}
return new Cluster([
{
host,
port: port as number,
},
], clusterOptions);
}
function getClient() {
return cluster ? GetRedisClusterClient() : GetRedisClient();
}
export default getClient();
The dev dependency #types/ioredis has been removed from our package.json.
I'm getting a semantic error when I compile the project typescript.
xxx/node_modules/#types/connect-redis/index.d.ts(22,51): error TS2694: Namespace '"xxx/node_modules/ioredis/built/index"' has no exported member 'Redis'.
And the jest tests no longer run, flagging up the line return new Cluster, failing with the following
TypeError: _ioredis.Cluster is not a constructor
The tests:
import Redis, { Cluster, ClusterOptions } from 'ioredis';
import config from '../../../config';
jest.mock('ioredis');
jest.mock('../../../config', () => ({
redis: {
host: 'redis',
port: 1234,
cluster: false,
transitEncryption: false,
},
service_name: 'app name',
version: 'test',
}));
describe('redis client', () => {
const clusterTest = (clusterOptions: ClusterOptions) => {
// eslint-disable-next-line global-require
require('./client');
expect(Cluster)
.toHaveBeenCalled();
};
const clusterOptions: ClusterOptions = {
clusterRetryStrategy: expect.any(Function),
redisOptions: { db: 0 },
};
test('if the cluster variable is set, then the module should return a clustered client', () => {
config.redis.cluster = true;
clusterTest(clusterOptions);
});
test('if the cluster variable is set and transit encryption is set, then the module should return a clustered client with encryption', () => {
config.redis.cluster = true;
config.redis.transitEncryption = true;
clusterOptions.dnsLookup = expect.any(Function);
clusterOptions.redisOptions = { db: 0, tls: {} };
clusterTest(clusterOptions);
});
test('if the cluster variable is not set, then the module should return a standalone client', () => {
config.redis.cluster = false;
// eslint-disable-next-line global-require
require('./client');
expect(Redis).toHaveBeenCalled();
});
});
My colleague seems to get the tests running individually, but using code branch, I get the above problem.
Any suggestions about where I've gone wrong?
Related
Anybody know how to enable the listen hook in Nuxt in production ?
It works with npm run dev (dev) but not with npm run build & npm run preview (production)
I want to have access to the node server to link it with a WebSocketServer, this is my nuxt.config.ts:
// https://nuxt.com/docs/api/configuration/nuxt-config
export default defineNuxtConfig({
hooks: {
listen(server) {
console.log(server)
const yogaApp = createYoga({
graphqlEndpoint: '/api/graphql',
schema: createSchema({
typeDefs: /* GraphQL */ `
type Query {
hello: String
}
type Subscription {
countdown(from: Int!): Int!
}
`,
resolvers: {
Query: {
hello: () => 'world'
},
Subscription: {
countdown: {
// This will return the value on every 1 sec until it reaches 0
subscribe: async function* (_, { from }) {
for (let i = from; i >= 0; i--) {
await new Promise((resolve) => setTimeout(resolve, 1000))
yield { countdown: i }
}
}
}
}
}
}),
graphiql:{
subscriptionsProtocol: "WS"
}
})
console.log(yogaApp.graphqlEndpoint)
// Get NodeJS Server from Yoga
// Create WebSocket server instance from our Node server
const wsServer = new WebSocketServer({
server,
path: yogaApp.graphqlEndpoint
})
// Integrate Yoga's Envelop instance and NodeJS server with graphql-ws
useServer(
{
execute: (args: any) => args.rootValue.execute(args),
subscribe: (args: any) => args.rootValue.subscribe(args),
onSubscribe: async (ctx, msg) => {
const { schema, execute, subscribe, contextFactory, parse, validate } =
yogaApp.getEnveloped({
...ctx,
req: ctx.extra.request,
socket: ctx.extra.socket,
params: msg.payload
})
const args = {
schema,
operationName: msg.payload.operationName,
document: parse(msg.payload.query),
variableValues: msg.payload.variables,
contextValue: await contextFactory(),
rootValue: {
execute,
subscribe
}
}
const errors = validate(args.schema, args.document)
if (errors.length) return errors
return args
}
},
wsServer
)
}
},
})
I was trying to connect Redis (v4.0.1) to my express server with typescript but having a bit issue. Am learning typescript. It's showing redlines on host inside redis.createClient() Can anyone help me out?
const host = process.env.REDIS_HOST;
const port = process.env.REDIS_PORT;
const redisClient = redis.createClient({
host,
port,
});
Argument of type '{ host: string | undefined; port: string | undefined; }' is not assignable to parameter of type 'Omit<RedisClientOptions<never, RedisScripts>, "modules">'.
Object literal may only specify known properties, and 'host' does not exist in type 'Omit<RedisClientOptions<never, RedisScripts>, "modules">'.ts(2345)
Options have changed when redis updated to 4.0.1. This should help you.
This works as expected (redis v4.1.0)
const url = process.env.REDIS_URL || 'redis://localhost:6379';
const redisClient = redis.createClient({
url
});
what I did in my project was this
file: services/internal/cache.ts
/* eslint-disable no-inline-comments */
import type { RedisClientType } from 'redis'
import { createClient } from 'redis'
import { config } from '#app/config'
import { logger } from '#app/utils/logger'
let redisClient: RedisClientType
let isReady: boolean
const cacheOptions = {
url: config.redis.tlsFlag ? config.redis.urlTls : config.redis.url,
}
if (config.redis.tlsFlag) {
Object.assign(cacheOptions, {
socket: {
// keepAlive: 300, // 5 minutes DEFAULT
tls: false,
},
})
}
async function getCache(): Promise<RedisClientType> {
if (!isReady) {
redisClient = createClient({
...cacheOptions,
})
redisClient.on('error', err => logger.error(`Redis Error: ${err}`))
redisClient.on('connect', () => logger.info('Redis connected'))
redisClient.on('reconnecting', () => logger.info('Redis reconnecting'))
redisClient.on('ready', () => {
isReady = true
logger.info('Redis ready!')
})
await redisClient.connect()
}
return redisClient
}
getCache().then(connection => {
redisClient = connection
}).catch(err => {
// eslint-disable-next-line #typescript-eslint/no-unsafe-assignment
logger.error({ err }, 'Failed to connect to Redis')
})
export {
getCache,
}
then you just import where you need:
import { getCache } from '#services/internal/cache'
const cache = await getCache()
cache.setEx(accountId, 60, JSON.stringify(account))
The option to add a host, port in redis.createClient is no longer supported by redis. So it is not inside type createClient. use URL instead.
import { createClient } from 'redis';
const client = createClient({
socket: {
host: process.env.REDIS_HOST,
port: parseInt(process.env.REDIS_PORT)
},
password: process.env.REDIS_PW
});
client.on('error', (err) => console.error(err));
client.connect();
export { client };
There are 3 bases (front, node, remote). Front <=> node, node <=> remote. When the front base is updated, the data goes to the remote base, but the node is not updated. In theory, the node should be updated first, and then the remote base.
Render db
addPouchPlugin(PouchdbAdapterIdb)
addPouchPlugin(PouchHttpPlugin)
addRxPlugin(RxDBReplicationCouchDBPlugin)
addRxPlugin(RxDBMigrationPlugin)
addRxPlugin(RxDBLeaderElectionPlugin)
addRxPlugin(RxDBQueryBuilderPlugin)
addRxPlugin(RxDBAjvValidatePlugin)
addRxPlugin(RxDBUpdatePlugin)
export const createDb = async () => {
console.log('[src/renderer/database/createDb] createDb')
const productsName = collectionName.getCollectionProductsName()
const documentsName = collectionName.getCollectionDocumentsName()
const settingsName = collectionName.getCollectionSettingsName()
const db = await createRxDatabase<Collections>({
name: 'renderer',
// use pouchdb with the indexeddb-adapter as storage engine.
storage: getRxStoragePouch('idb'),
})
await initCommonCollections({ db, documentsName, productsName, settingsName })
syncDbCollections(db, [productsName, documentsName, settingsName])
db.$.subscribe(({ operation, documentId, documentData }) => {
if (documentData.type === SettingsTypes.DEVICE_SETTING) {
console.log(`Change database RENDER event:\n ${operation}, \n documentData:`, documentData)
}
})
return db
}
Render sync
const remoteDbUrl = `http://localhost:3030/db/`
const logPath = '[src/renderer/database/syncDbCollections]'
export const syncDbCollections = (db: RxDatabase<Collections>, collectionNames: (keyof Collections)[]) => {
console.log('syncDbCollections', collectionNames)
collectionNames.forEach(name => {
const rxReplicationState = db.collections[name].syncCouchDB({
remote: `${remoteDbUrl}${name}`,
options: {
live: true,
retry: true,
},
})
rxReplicationState.error$.subscribe(error => {
console.error(logPath, name, 'error', JSON.stringify(error))
})
})
}
Node base
addPouchPlugin(PouchdbAdapterHttp)
addPouchPlugin(LevelDbAdapter)
addRxPlugin(RxDBAjvValidatePlugin)
addRxPlugin(RxDBMigrationPlugin)
addRxPlugin(RxDBServerPlugin)
addRxPlugin(RxDBLeaderElectionPlugin)
addRxPlugin(RxDBQueryBuilderPlugin)
addRxPlugin(RxDBUpdatePlugin)
addRxPlugin(RxDBReplicationCouchDBPlugin)
let db: RxDatabase<Collections>
export const getMainDb = () => {
if (!db) {
throw new Error('No available database.')
}
return db
}
export const getDocumentCollection = (): DocumentsRxCol => {
return db[collectionNames.getCollectionDocumentsName()]
}
export const getSettingsCollection = (): SettingsRxCol => {
return db[collectionNames.getCollectionSettingsName()]
}
export const getProductsCollection = (): ProductsRxCol => {
return db[collectionNames.getCollectionProductsName()]
}
export const initDatabase = async () => {
console.log(logPathAlias, 'initDatabase')
if (db) {
console.warn(logPathAlias, 'db instance already created!')
return db
}
db = await createRxDatabase<Collections>({
name: `${electronApp.getPath('userData')}/db`,
storage: getRxStoragePouch(LevelDown),
})
const productsName = collectionNames.getCollectionProductsName()
const documentsName = collectionNames.getCollectionDocumentsName()
const settingsName = collectionNames.getCollectionSettingsName()
await initCommonCollections({ db, productsName, documentsName, settingsName })
await syncCollections([productsName, documentsName, settingsName])
db.$.subscribe(({ operation, documentId, documentData }) => {
// if (documentData.type === SettingsTypes.DEVICE_SETTING) {
console.log(`Change database NODE event:\n ${operation}, \n documentData:`, documentData)
// }
})
const { app } = await db.server({
startServer: false, // (optional), start express server
// options of the pouchdb express server
cors: false,
pouchdbExpressOptions: {
inMemoryConfig: true, // do not write a config.json
logPath: `${electronApp.getPath('temp')}/rxdb-server.log`, // save logs in tmp folder
},
})
return app
}
const lastRetryTime = {}
const syncCollections = async (collections: CollectionNames[]) => {
collections.map(collectionName => {
const rxReplicationState = db.collections[collectionName].syncCouchDB({
remote: `${CouchDbServerUrl}/${collectionName}`,
options: {
live: true,
retry: true,
// #ts-ignore
// headers: {
// Authorization: `Bearer ${getAccessToken()}`,
// },
},
})
rxReplicationState.error$.subscribe(async error => {
console.error(logPathAlias, collectionName, String(error))
if (error.status === 401 && dayjs().diff(lastRetryTime[collectionName], 'seconds') > 10 && getIsRefreshFresh()) {
lastRetryTime[collectionName] = dayjs()
await rxReplicationState.cancel()
await refreshTokens()
await syncCollections([collectionName])
}
})
})
}
No errors
Moreover, if you save data in a remote database, then they are synchronized with the node
Help me :(
typescript connection to mongo database throws error , it can read conn of undefined after conn has been declared globally
UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'conn' of undefined
at Object.connectToDatabase [as default]
import { MongoClient, Db } from "mongodb";
import config from "../config/config";
const { dbName, mongoDBUri } = config;
type MongoConnection = {
client: MongoClient;
db: Db;
};
declare global {
namespace NodeJS {
interface Global {
mongodb: {
conn: MongoConnection | null;
promise: Promise<MongoConnection> | null;
};
}
}
}
let cached = global.mongodb;
async function connectToDatabase() {
if (cached.conn) {
return cached.conn;
}
if (!cached.promise) {
const opts = {
useNewUrlParser: true,
useUnifiedTopology: true,
};
cached.promise = MongoClient.connect(mongoDBUri as string, opts).then(
(client) => {
return {
client,
db: client.db(dbName),
};
}
);
}
cached.conn = await cached.promise;
return cached.conn;
}
export default connectToDatabase;
You can use the below setup
//interfaces/db.interface
export interface dbConfig {
host: string;
port: number;
database: string;
username: string;
password: string;
}
//database.ts
import { dbConfig } from "#interfaces/db.interface";
const { host, port, database, username, password }: dbConfig = config.get("dbConfig");
export const dbConnection = {
url: `mongodb://${username}:${password}#${host}:${port}/${database}?authSource=admin`,
options: {
useNewUrlParser: true,
useUnifiedTopology: true,
useFindAndModify: false,
useCreateIndex: true
},
};
//app.ts (express app)
import { dbConnection } from "#databases";
constructor(routes: Routes[]) {
this.app = express();
this.port = process.env.PORT || 5000;
this.env = process.env.NODE_ENV || "development";
this.connectToDatabase();
}
private connectToDatabase() {
if (this.env !== "production") {
set("debug", true);
}
connect(dbConnection.url, dbConnection.options)
.catch((error) =>
console.log(`${error}`)
);
}
Here I am assuming you have the setup of paths in the tsconfig.json file so that # will work in imports.
After several times of trying, I had to use the NextJs MongoDB connection pattern and convert it to typescript and it worked perfectly fine
import config from "./../config/config";
import { MongoClient, Db } from "mongodb";
const { dbName, mongoDBUri } = config;
if (!mongoDBUri) {
throw new Error(
"Define the mongoDBUri environment variable inside .env"
);
}
if (!dbName) {
throw new Error(
"Define the dbName environment variable inside .env"
);
}
type MongoConnection = {
client: MongoClient;
db: Db;
};
declare global {
namespace NodeJS {
interface Global {
mongodb: {
conn: MongoConnection | null;
promise: Promise<MongoConnection> | null;
};
}
}
}
let cached = global.mongodb;
if (!cached) {
cached = global.mongodb = { conn: null, promise: null };
}
export default async function connectToDatabase() {
if (cached.conn) {
return cached.conn;
}
if (!cached.promise) {
console.log("Establishing new database connection");
const opts = {
useNewUrlParser: true,
useUnifiedTopology: true,
};
cached.promise = MongoClient.connect(mongoDBUri as string, opts).then(
(client) => {
return {
client,
db: client.db(dbName),
};
}
);
}
cached.conn = await cached.promise;
return cached.conn;
}
UPDATED
I am getting the following error when trying to invoke my Lambda function
{
"errorType": "TypeError",
"errorMessage": "e is not a function",
"trace": [
"TypeError: e is not a function",
" at Runtime.handler (/var/task/serverless_sdk/index.js:9:88355)",
" at Runtime.handleOnce (/var/runtime/Runtime.js:66:25)"
]
}
I have tracked this down to the reference to DB (see last few lines of schema.js DB should be imported at the top of schema.js
const { DB } = require('./db.js')
Indeed, when I try the same code on my local computer, there is no issue.
Does this have to do with some subtle ways how Lambda Functions (LF) are frozen for re-use in AWS? Where should I be initializing the DB connection in a LF?
I tried merging db.js into schema.js (no import) and I still get the same error.
I have checked the zip file that serverless loaded and it looks fine (node_modules and mine).
This is very hard to debug. So any tips in that direction would help.
server.js
const { ApolloServer } = require('apollo-server')
const { ApolloServer: ApolloServerLambda } = require('apollo-server-lambda')
const { typeDefs, resolvers, connect } = require('./schema.js')
// The ApolloServer constructor requires two parameters: your schema
// definition and your set of resolvers.
async function setup(where) {
if (where == 'local') {
const server = new ApolloServer({ typeDefs, resolvers })
let { url } = await server.listen()
console.log(`Server ready at ${url}`)
} else {
const server = new ApolloServerLambda({
typeDefs,
resolvers,
playground: true,
introspection: true,
cors: {
origin: '*',
credentials: true,
},
context: ({ event, context }) => (
{
headers: event.headers,
functionName: context.functionName,
event,
context
})
})
exports.graphqlHandler = server.createHandler()
}
}
let location = (process.env.USERNAME == 'ysg4206') ? 'local' : 'aws'
connect(location, setup)
schema.js
const { gql } = require('apollo-server')
const { GraphQLDateTime } = require('graphql-iso-date')
const { DB } = require('./db.js')
exports.typeDefs = gql`
scalar DateTime
type User {
id: Int
"English First Name"
firstName: String
lastName: String
addressNumber: Int
streetName: String
city: String
email: String
createdAt: DateTime
updatedAt: DateTime
}
type Query {
users: [User]
findUser(firstName: String): User
hello(reply: String): String
}
type Mutation {
addUser(user: UserType): User!
}
type Subscription {
newUser: User!
}
`
exports.resolvers = {
Query: {
users: () => DB.findAll(),
findUser: async (_, { firstName }) => {
let who = await DB.findFirst(firstName)
return who
},
hello: (_, { reply }, context, info) => {
console.log(`hello with reply ${reply}`)
console.log(`context : ${JSON.stringify(context)}`)
console.log(`info : ${JSON.stringify(info)}`)
return reply
}
},
Mutation: {
addUser: async (_, args) => {
let who = await DB.addUser(args.user)
return who
}
}
}
exports.connect = async (where, setup) => {
console.log(`DB: ${DB}') // BUG DB is returning null
await DB.dbSetup(where) //BUG these lines cause Lambda to fail
await DB.populate() //BUG these lines cause Lambda to fail
let users = await DB.findAll() //BUG these lines cause Lambda to fail
console.log(users) //BUG these lines cause Lambda to fail
await setup(where)
}
db.js
const { Sequelize } = require('sequelize')
const { userData } = require('./userData')
const localHost = {
db: 'm3_db',
host: 'localhost',
pass: 'xxxx'
}
const awsHost = {
db: 'mapollodb3_db',
host: 'apollodb.cxeokcheapqj.us-east-2.rds.amazonaws.com',
pass: 'xxxx'
}
class DB {
async dbSetup(where) {
let host = (where == "local") ? localHost : awsHost
this.db = new Sequelize(host.db, 'postgres', host.pass, {
host: host.host,
dialect: 'postgres',
logging: false,
pool: {
max: 5,
min: 0,
idle: 20000,
handleDisconnects: true
},
dialectOptions: {
requestTimeout: 100000
},
define: {
freezeTableName: true
}
})
this.User = this.db.define('users', {
firstName: Sequelize.STRING,
lastName: Sequelize.STRING,
addressNumber: Sequelize.INTEGER,
streetName: Sequelize.STRING,
city: Sequelize.STRING,
email: Sequelize.STRING,
})
try {
await this.db.authenticate()
console.log('Connected to DB')
} catch (err) {
console.error('Unable to connect to DB', err)
}
}
async select(id) {
let who = await this.User.findAll({ where: { id: id } })
return who.get({ plain: true })
}
async findFirst(name) {
let me = await this.User.findAll({ where: { firstName: name } })
return me[0].get({ plain: true })
}
async addUser(user) {
let me = await this.User.create(user)
return me.get({ plain: true })
}
async populate() {
await this.db.sync({ force: true })
try {
await this.User.bulkCreate(userData, { validate: true })
console.log('users created');
} catch (err) {
console.error('failed to create users')
console.error(err)
} finally {
}
}
async findAll() {
let users = await this.User.findAll({ raw: true })
return users
}
async close() {
this.db.close()
}
}
exports.DB = new DB()
serverless.yml
service: apollo-lambda
provider:
name: aws
stage: dev
region: us-east-2
runtime: nodejs10.x
# cfnRole: arn:aws:iam::237632220688:role/lambda-role
functions:
graphql:
# this is formatted as <FILENAME>.<HANDLER>
handler: server.graphqlHandler
vpc:
securityGroupIds:
- sg-a1e6f4c3
subnetIds:
- subnet-4a2a7830
- subnet-1469d358
- subnet-53b45038
events:
- http:
path: graphql
method: post
cors: true
- http:
path: graphql
method: get
cors: true
folder structure of zip
When AWS Lambda imports your file, the export isn't available yet. That's why it complains that your handler is not a function (because it is actually undefined at that time it is being imported).
Here are a couple of suggested solutions:
1. Use only apollo-server-lambda and use serverless-offline for local development. This way your handler code is exactly the same as what you have in Lambda.
const { ApolloServer: ApolloServerLambda } = require("apollo-server-lambda");
const { typeDefs, resolvers, connect } = require("./schema.js");
const server = new ApolloServerLambda({
typeDefs,
resolvers,
playground: true,
introspection: true,
cors: {
origin: "*",
credentials: true
},
context: ({ event, context }) => ({
headers: event.headers,
functionName: context.functionName,
event,
context
})
});
exports.graphqlHandler = server.createHandler();
2. Use apollo-server-lambda in your Lambda but use apollo-server in another file (e.g. local.js).. Then, you just use node local.js for local development. No need for that process.env.USERNAME check that you do at the end.
Found the problem. It is a bit embarrassing. But I post it in case others need this.
I was trying to connect to the DB as part of the initialization of the lambda app. Hoping that when the cold start or warm start happened, the variable with DB would be already holding the connection.
That is anti-pattern.
With apollo one has to reconnect to the DB on each request. that is in the resolver for the GraphQL one has to reconnect to the DB and then close it so that AWS can see there are no open connections and then close the Lambda function.
What threw me was this worked fine when running as ApolloServer and connecting to a local DB.