Lambda timeout after 10s both in serverless offline and online - node.js

I am new to lambda, mongodb and nodejs and I'm trying to access a mongodb database follwoing this tutorial: [pretty nice tutorial][1] . This is my second attempt at connecting to mongodb and after spending hours setting things up I run into an unsurmountable wall and none of all the answers on stack overflow have helped. Any help will be greatly appreciated.
I have inserted console.log instructions and they both are executed so the timout happens after connecting to the database, but after that I don't know what is executed. I have also set up VPC properly in AWS Lambda so I have no idea why this is timing out. ( See edit below) It might be worth mentioning that I have never been able to get past this stage: of all the things I have tried there is always a timeout error.
I have a handler.js file which, among other functions, has this one:
'use strict';
require('dotenv').config({ path: './variables.env' });
const connectToDatabase = require('./db');
const Note = require('./models/Note');
module.exports.create = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
console.log("here1");
connectToDatabase()
.then(() => {
Note.create(JSON.parse(event.body))
.then(note => callback(null, {
statusCode: 200,
body: JSON.stringify(note)
}))
.catch(err => callback(null, {
statusCode: err.statusCode || 500,
headers: { 'Content-Type': 'text/plain' },
body: 'Could not create the note.'
}));
});
console.log("here2");
};
I also have a db.js file and a Note.js file as instructed in the tutorial ( here posted respectively )
//file db.js
const mongoose = require('mongoose');
mongoose.Promise = global.Promise;
let isConnected;
module.exports = connectToDatabase = () => {
if (isConnected) {
console.log('=> using existing database connection');
return Promise.resolve();
}
console.log('=> using new database connection');
return mongoose.connect(process.env.DB)
.then(db => {
isConnected = db.connections[0].readyState;
});
};
//file Note.js
const mongoose = require('mongoose');
const NoteSchema = new mongoose.Schema({
title: String,
description: String
});
module.exports = mongoose.model('Note', NoteSchema);
Both serverless offline and serverless online give me a timeout error:
Serverless offline:
here1
=> using new database connection
here2
✖ Lambda timeout.
Serverless online:
START RequestId: f75c6e23-84e2-485c-8fb4-065a95bbbb9b Version: $LATEST
2022-10-25T11:53:44.171Z f75c6e23-84e2-485c-8fb4-065a95bbbb9b INFO here1
2022-10-25T11:53:44.171Z f75c6e23-84e2-485c-8fb4-065a95bbbb9b INFO => using new database connection
2022-10-25T11:53:44.311Z f75c6e23-84e2-485c-8fb4-065a95bbbb9b INFO here2
END RequestId: f75c6e23-84e2-485c-8fb4-065a95bbbb9b
REPORT RequestId: f75c6e23-84e2-485c-8fb4-065a95bbbb9b Duration: 10013.48 ms Billed Duration: 10000 ms Memory Size: 128 MB Max Memory Used: 100 MB Init Duration: 864.37 ms
2022-10-25T11:53:54.174Z f75c6e23-84e2-485c-8fb4-065a95bbbb9b Task timed out after 10.01 seconds
I am sending this JSON POST request through Insomnia:
{
"title":"some title",
"description": "some description"
}
After changing the timeout time to 5 mins, I am getting an error related to the connection to mongo saying I should whitelist my IP. It is whitelisted but I will investigate.
[1]: https://medium.com/hackernoon/building-a-serverless-rest-api-with-node-js-and-mongodb-2e0ed0638f47

Related

AWS Redis Cluster MOVED Error using redis node library

I have created a Redis MemoryDB cluster with 2 nodes in AWS:
I connect to it using redis node library v4.0.0 like this:
import { createCluster } from 'redis';
(async () => {
const REDIS_USERNAME = 'test-username';
const REDIS_PASSWORD = 'test-pass';
const cluster = createCluster({
rootNodes: [
{
url: `rediss://node1.amazonaws.com:6379`,
},
{
url: `rediss://node2.amazonaws.com:6379`,
},
],
defaults: {
url: `rediss://cluster.amazonaws.com:6379`,
username: REDIS_USERNAME,
password: REDIS_PASSWORD,
}
});
cluster.on('error', (err) => console.log('Redis Cluster Error', err));
await cluster.connect();
console.log('connected to cluster...');
await cluster.set('key', 'value');
const value = await cluster.get('key');
console.log('Value', value);
await cluster.disconnect();
})();
But sometimes I get the error ReplyError: MOVED 12539 rediss://node2.amazonaws.com:6379 and I cannot get the value from the key.
Do you have any idea if there is something wrong with the configuration of the cluster or with the code using redis node library?
Edit:
I tried it with ioredis library and it works, so it's something wrong with the redis library.
Node.js Version: 16
Redis Server Version: 6
I had created an issue to redis library, so it's going to be solved soon with this PR.

MongooseError: Operation users.insertOne() buffering timed out after 10000ms” in Mongo Db atlas

I am currently working with node and mongoDB
here is my code
import dotenv from "dotenv";
import mongoose from "mongoose";
dotenv.config();
mongoose
.connect(
`mongodb+srv://OmniBotBuilder:${process.env.DBPASS}${process.env.DBUSER}.kx2vg.mongodb.net/${process.env.DBNAME}?retryWrites=true&w=majority`,
{
useUnifiedTopology: true,
useNewUrlParser: true,
}
)
.catch(() => console.error("Unable to connect to DB"));
mongoose.connection.on("connected", () => {});
const Schema = mongoose.Schema;
const omniGamesSchema = new Schema({
discordId: Number,
steamId: Number,
});
const omniGamesModel = mongoose.model("omniGamesSchema", omniGamesSchema);
const createNewUser = (discordId, steamId) => {
const newUserMap = new omniGamesModel({
discordId: discordId,
steamId: steamId,
});
newUserMap.save((err) => {
if (err) {
console.error(err);
}
});
};
export { createNewUser };
and the error i am getting is this one
MongooseError: Operation omnigamesschemas.insertOne() buffering timed out after 10000ms
at Timeout. (C:\Users\dahiy\OneDrive\Desktop\bots\omni-games\node_modules\mongoose\lib\drivers\node-mongodb-native\collection.js:198:23)
at listOnTimeout (node:internal/timers:557:17)
at processTimers (node:internal/timers:500:7)
This issue normally is caused because:
wrong auth, by the meaning the mongo path string is wrong so maybe double check your pass and username
check the allowed IP to access the database from the mongo website, if you want it to be accessed from everywhere just use IP: 0.0.0.0/0
Your internet connection might be slow to the point it cannot connect to the DB
Hope you found this helpful! :)
You can open https://cloud.mongodb.com/
Click connect Goto the Connect your application
Select Driver Node.js version 4.1 leter
Copy the link which you have to show than paste this link to your
project .env file now try to run application connect Database and
check.

Mongoose Unexpected token u in JSON at position 0

I'm creating an AWS Lambda API with Node JS. There's a template for the db connection itself, but when I run a test in Lambda I get an error and I believe the problem is the connection itself.
Here's the error message I get with Lambda test:
START RequestId: 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5 Version: $LATEST
2020-05-23T01:46:23.685Z 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5 INFO => using new database connection
2020-05-23T01:46:23.689Z 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5 ERROR (node:8) DeprecationWarning: current URL string parser is deprecated, and will be removed in a future version. To use the new parser, pass option { useNewUrlParser: true } to MongoClient.connect.
2020-05-23T01:46:23.726Z 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5 ERROR (node:8) DeprecationWarning: current Server Discovery and Monitoring engine is deprecated, and will be removed in a future version. To use the new Server Discover and Monitoring engine, pass option { useUnifiedTopology: true } to the MongoClient constructor.
2020-05-23T01:46:23.828Z 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5 ERROR Unhandled Promise Rejection {"errorType":"Runtime.UnhandledPromiseRejection","errorMessage":"SyntaxError: Unexpected token u in JSON at position 0","reason":{"errorType":"SyntaxError","errorMessage":"Unexpected token u in JSON at position 0","stack":["SyntaxError: Unexpected token u in JSON at position 0"," at JSON.parse ()"," at /var/task/handler.js:17:22"," at processTicksAndRejections (internal/process/task_queues.js:97:5)"]},"promise":{},"stack":["Runtime.UnhandledPromiseRejection: SyntaxError: Unexpected token u in JSON at position 0"," at process. (/var/runtime/index.js:35:15)"," at process.emit (events.js:310:20)"," at processPromiseRejections (internal/process/promises.js:209:33)"," at processTicksAndRejections (internal/process/task_queues.js:98:32)"]}
END RequestId: 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5
REPORT RequestId: 6b9eef97-9c54-44bc-8ecc-dfbec7e200e5 Duration: 186.61 ms Billed Duration: 200 ms Memory Size: 1024 MB Max Memory Used: 34 MB
Unknown application error occurred
DB.js
const mongoose = require("mongoose");
mongoose.Promise = global.Promise;
let isConnected;
module.exports = connectToDatabase = () => {
if (isConnected) {
console.log("=> using existing database connection");
return Promise.resolve();
}
console.log("=> using new database connection");
return mongoose.connect(process.env.DB).then((db) => {
isConnected = db.connections[0].readyState;
});
};
Handler.js
"use strict";
const connectToDatabase = require("./db");
const Lead = require("./leads.model.js");
require("dotenv").config({ path: "./variables.env" });
module.exports.hello = (event, context, callback) => {
console.log("Hello World");
callback(null, "Hello World");
};
module.exports.create = (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
connectToDatabase().then(() => {
Lead.create(JSON.parse(event.body))
.then((lead) =>
callback(null, {
statusCode: 200,
body: JSON.stringify(lead),
})
)
.catch((err) =>
callback(null, {
statusCode: err.statusCode || 500,
headers: { "Content-Type": "text/plain" },
body: "Could not create the lead.",
})
);
});
};
variables.env
DB=mongodb+srv://dbUser:password#api-jodqc.mongodb.net/test?retryWrites=true&w=majority
Whenever I see Unexpected token u in JSON at position 0, it usually means JSON.parse is trying to parse undefined. Looking at the code, I only see you executing parse in one spot
Lead.create(JSON.parse(event.body))
I'm guessing when you execute the Lambda, you're passing in an object that looks like a lead directly, so maybe something like below.
{
isALead: true
}
Instead, you need to test with an object that looks like
{
body: "{ \"isALead\": true }"
}

Integration tests sporadically timeout on Heroku hobby-dev

Since we do not get any helpful support from help.heroku.com we make the last try here.
We are developping a classic web app consisting of:
----- ----- ----
|WEB| <----> |API| <----> |DB|
----- ----- ----
We currently are working with the following Heroku Dynos/Datastores
Heroku Postgres: Hobby Basic
Heroku API Dyno: Hobby
Heroku WEB Dyno: Hobby
The tech stack is:
runtime: nodejs (4.4.0)
db: postgres (9.6.1)
testframework: jasminejs (2.5.1)
query builder: knexjs (0.10.0)
We recently moved to from self hosted docker environment to Heroku and configured the Herokus CI pipeline which works fine for unit testing - but not integration testing.
The tests sporadically fails with timeouts (in average every 3rd test of the same commit). This is not stable enough to build up CI/CD.
Here the error messages we get:
**************************************************
* Failures *
**************************************************
1) integration test collections repository create() should return AUTHENTICATION_REQUIRED if user is anonymous
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
2) integration test collections repository create() should return AUTHORIZATION_REQUIRED if user is not space MEMBER
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
3) integration test collections repository create() should return collection if user is space MEMBER
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
- Error: Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.
For testing purposes we configured knex connection pooling to use one connection only:
export default {
client: 'pg',
connection: DB_URL,
pool: {
max: 1,
min: 1
}
};
The typical integration test setup is:
describe('integration test', () => {
describe('collections repository create()', () => {
//////////////////////////////////////////////////////////////////////
//Before the test block is execute all fixtures are loaded into the DB
//////////////////////////////////////////////////////////////////////
beforeAll((callback) => {
seed(path.join(__dirname, 'fixtures'))
.then(callback)
.catch((error) => {
fail(error);
callback();
});
});
////////////////////////////////////////////////////////////////////////
//Truncate all data from the DB before the next test block gets executed
////////////////////////////////////////////////////////////////////////
afterAll(resetDB);
it('should return AUTHENTICATION_REQUIRED if user is anonymous', (callback) => {
testUtils.testAsync({
callback,
catchTest: (error) => {
expect(error).toEqual(jasmine.any(repositoryErrors.AUTHENTICATION_REQUIRED));
},
runAsync: create({ props: { space: { id: 1 } } })
});
});
it('should return AUTHORIZATION_REQUIRED if user is not space MEMBER', (callback) => {
testUtils.testAsync({
callback,
catchTest: (error) => {
expect(error).toEqual(jasmine.any(repositoryErrors.AUTHORIZATION_REQUIRED));
},
runAsync: create({ props: { space: { id: 1 } }, userId: 1 })
});
});
it('should return collection if user is space MEMBER', (callback) => {
testUtils.testAsync({
callback,
runAsync: create({ props: { space: { id: 1 } }, userId: 2 }),
thenTest: (outcome) => {
expect(outcome).toEqual({ id: '1' });
}
});
});
...
seed:
const tableOrder = [
'users',
'guidelines',
'collections',
'spaces',
'details',
'files',
'guidelinesAuthors',
'collectionsUsers',
'spacesUsers',
'guidelinesCollections',
'spacesCollections',
'guidelinesDetails',
'guidelinesFiles',
'comments',
'commentsReplies',
'notifications'
];
export default function seed(path) {
return db.raw('BEGIN;')
.then(() => {
return tableOrder.reduce((promise, table) => {
return promise
.then(() => slurp({ path, table }))
.then(() => {
updateIdSequence(table)
.catch((error) => {
// eslint-disable-next-line no-console
console.log(
`Updating id sequence for table '${table}' failed! Error: `,
error.message
);
});
});
}, Promise.resolve()).then(() => db.raw('COMMIT;'));
})
.catch((error) => {
// eslint-disable-next-line no-console
console.log('SEED DATA FAILED', error);
return db.raw('ROLLBACK;');
});
}
...
resetDB:
export default function resetDB(callback) {
const sql = 'BEGIN; '
+ 'SELECT truncateAllData(); '
+ 'SELECT restartSequences(); '
+ 'COMMIT;';
return db.raw(sql)
.then(callback)
.catch((error) => {
// eslint-disable-next-line no-console
console.log('TRUNCATE TABLES FAILED', error);
return db.raw('ROLLBACK;');
});
}
Until now, these test have been running on local machines (Linux/Mac) and Codeship without any problem.
After almost two weeks of trying to get this work we made zero progress on this issue. I can't see anything wrong with this configuration and I start to belive Heroku has a serious issue with the datastores...
Has anybody experienced similar issues on Heroku?
Any idea what we can try else to get this work?

AWS Lambda and Redis

I'm trying to write an AWS Lambda function which uses redis(on amazon elasticcache). The problem – I can't connect to redis. I use code like this
'use strict'
function handler (data, context, cb) {
const redis = require("redis")
console.log('before client initialization')
const client = redis.createClient({
url: 'redis://propper-url-cache.some.0001.euw1.cache.amazonaws.com:6379',
retry_strategy: function(options) {
console.log(options)
if (options.total_retry_time > 1000) {
throw new Error('can`t connect to redis')
}
}
})
console.log('after client initialization')
client.on("error", function (err) {
console.log('in error')
cb({error: err})
});
client.get("counter", function (err, counter) {
console.log('counter', counter)
if(_.isNull(counter)) {
counter = 0
}
client.set('counter', counter + 1, function(err) {
console.log(err)
cb(null, {counter: counter})
})
});
}
exports.handler = handler
as a result I see something like this in logs:

15:33:41
START RequestId: d8024ec2-7f36-11e6-996c-1bfcb60572c6 Version: $LATEST

15:33:42
2016-09-20T13:33:42.632Z d8024ec2-7f36-11e6-996c-1bfcb60572c6 before client initialization

15:33:42
2016-09-20T13:33:42.813Z d8024ec2-7f36-11e6-996c-1bfcb60572c6 after client initialization

15:33:44
END RequestId: d8024ec2-7f36-11e6-996c-1bfcb60572c6

15:33:44
REPORT RequestId: d8024ec2-7f36-11e6-996c-1bfcb60572c6 Duration: 3002.67 ms Billed Duration: 3000 ms Memory Size: 128 MB Max Memory Used: 19 MB

15:33:44
2016-09-20T13:33:44.620Z d8024ec2-7f36-11e6-996c-1bfcb60572c6 Task timed out after 3.00 seconds
when I change redis url for something which definitely makes no sense I have an additional row:
2016-09-20T13:29:42.953Z 48fcb071-7f36-11e6-bc52-c5ac58c12843 { attempt: 1, error: { [Error: Redis connection to some-url.euw1.cache.amazonaws.com:6379 failed - getaddrinfo ENOTFOUND some-url.euw1.cache.amazonaws.com some-url.euw1.cache.amazonaws.com:6379] code: 'ENOTFOUND', errno: 'ENOTFOUND', syscall: 'getaddrinfo', hostna
Any ideas?
You need to have Redis in same VPC as Lambda. Check your security group settings. And then if you have EC2 access, install redis-cli and try to connect Redis. If this get connected your Lambda Redis will also get connected. As said earlier, you need to have your lambda in same VPC.
The following is boilerplate code demonstrating connecting to Lambda:
console.log('before client initialization')
const redisOptions = {
host: 'xxxx.xxx.xxx.xxx.xxx.amazonaws.com',
port: 6379,
}
var client = redis.createClient(redisOptions);
console.log('after client initialization');
client.on('connect', function(result) {
console.log('connected');
}

Resources