I am just learning NodeJS and I come from a Java/Scala background.
I am writing a service that communicates with Amazon SNS and handles endpoints/tokens.
Basically there is a list of SNS applications that I have in my environment, and this list is rarely modified, so I would like to pre-load its values into a variable or constant on server startup.
The SNS-SDK provided by Amazon has this function for listing the applications:
listPlatformApplications(params, callback)
So what I naively tried to do was this:
var applications = [];
var loadApplications = function() {
sns.listPlatformApplications({}, function(err, data){
if (err) {
console.log(err);
} else {
return data['PlatformApplications'].map(function (app) {
return app['PlatformApplicationArn']
});
}
});
}
loadApplications();
And basically what happens is that some calls come in before this callback finishes, when the list is still empty.
How would I go about pre-loading this data, or any other data, before the server starts responding to requests?
Or maybe my reasoning for this is wrong, and there would be another approach to handle this on NodeJS that is more idiomatic
If you absolutely must use a callback instead of a promise (spoiler alert: you don't), you have to call the server startup command within the SNS callback. So you would do something like this:
var startServer = require('my-server.js');
var applications = [];
var loadApplications = function() {
sns.listPlatformApplications({}, function(err, data){
if (err) {
console.log(err);
} else {
var snsData = data['PlatformApplications'].map(function (app) {
return app['PlatformApplicationArn']
});
startServer(snsData)
}
});
}
loadApplications();
== RECOMMENDED SOLUTION ==
If instead, you can use promises (which you absolutely should!), you could start the SNS request at server start and await the result whenever needed. Your code would look something like this:
const startServer = require('my-server.js');
const loadApplications = async () => {
const data = sns.listPlatformApplications({}).promise();
const snsData = data['PlatformApplications'].map(function (app) {
return app['PlatformApplicationArn']
});
return snsData;
};
const applications = loadApplications();
startServer(applications);
// inside my-server.js
const startServer = async (applications) => {
const doSomethingWithApplications = await applications;
...
}
Related
Ultimately, I'd like to have an extra feature in my app if the app is running on AWS EC2.
How do I check and set a variable to indicate if it is on AWS or not? I found this thread to do the check, but upon startup how do I set a variable across the app like a boolean? Something like:
let checkAWS;
metadata.isEC2().then(function (onEC2) {
checkAWS = true;
console.log("EC2: " + onEC2);
});
let app = express();
app.locals.isAWS = checkAWS;
console.log(checkAWS);
Every time, I always get the same output:
undefined
EC2: true
I am using the isAWS variable in my .ejs file to decide on that functionality.
metadata.isEC2() is asynchronous so its value is available some time later when the .then() handler runs. Meanwhile, you're trying to use the value of checkAWS BEFORE that .then() handler has run.
Since you want access to that value before you start your Express sever, you could only start it inside the .then() handler like this:
let isAWS;
metadata.isEC2().then(function (onEC2) {
console.log("EC2: " + onEC2);
isAWS = true;
}).catch(err => {
isAWS = false;
}).finally(err => {
const app = express();
app.locals.isAWS = isAWS;
// do the rest of your app initialization here
app.listen(...);
});
// do not use app here (it won't be defined here)
If you're trying to export app for other modules to use, you will have to take a slightly different approach. We'd have to see your broader code context to know what to recommend for that. But, basically the idea here is that you shouldn't start your server until you have the asynchronously retrieved isAWS result.
Also, you should know that with that metadata.isEC2() call you're using, that looks for a known endpoint in an EC2 instance to detect EC2. If that endpoint does not exist, then this will take 500ms to timeout and hit the .catch() branch above. If it is an EC2 instance, it will be quick.
Note, it might be simpler to just check for the presence of some environment variables that are automatically set by the AWS environment such as AWS_REGION or AWS_EXECUTION_ENV. Those can be checked for synchronously.
let app = express();
app.locals.isAWS = !!process.env.AWS_REGION;;
For future reference in case anyone else is looking, this is what worked for me in order to be able to tell if you are running on AWS.
let isAwsSet = false; //has the app checked whether it is running on AWS by setting app.locals.isAWS
app.locals.isAWS = false;
app.get('/home', function(request, response) {
if (!isAwsSet){
urlExists().then(function (onAWS){
app.locals.isAWS = true;
isAwsSet = true;
response.render('home');
}).catch(function(error){
app.locals.isAWS = false;
isAwsSet = true;
response.render('home');
})
} else {
response.render('home');
}
});
function urlExists() {
return new Promise((resolve, reject) => {
//https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
const options = {
method: 'HEAD',
host: '169.254.169.254',
path: '/latest/meta-data/',
port: 80,
timeout: 1500
};
const req = http.request(options, (res) => {
// reject on bad status
if (res.statusCode !== 200) {
return reject(new Error('statusCode=' + res.statusCode));
}
resolve(res.statusCode = 200);
});
req.on('timeout', () => {
req.destroy();
});
req.on('error', function(err) {
reject(err);
});
req.end();
});
}
I think the rendering takes place before the searching of the string on the files, i have tried different methods but don't seems to get this working. any help will be appreciated. im a noob on to the nodejs. im trying to get the id of the user and query and get all the data and there after see if he is in any of the lists given and finally render the page.
const j = [];
let name = '';
const filename = [];
var ext = '';
module.exports = function(app, express) {
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/cusdetails', isLoggedIn, function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
connection.query(insertQuerys,[cusid],
function(err, rows){
rows.forEach( (row) => {
name=row.fncus;
});
fs.readdir('./views/iplist', function(err, files) {
if (err)
throw err;
for (var index in files) {
j.push(files[index])
}
j.forEach(function(value) {
var k = require('path').resolve(__dirname, '../views/iplist/',value);
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) throw err;
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
}
});
}
else {
console.log(" FileNotExist ");
}
});
});
});
console.log(filename);
res.render('cusdetails.ejs', {rows: rows, user:req.user , aml: filename });
});
})
You can create simple Promise wrapper and then use it inside async/await function to pause execution until resolved.
// use mysql2 package as it provides promise, less work to write promise wrappers
const mysql = require('mysql2/promise');
// create the connection to database
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'test'
});
// sample wrapper
function some(k) {
// more advisable to have local variables, why do you need this to be array?
var filename = [];
return new Promise((resolve, reject) => {
// doing this is also not recommended check nodejs documentation **fs.exists** for more info
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) reject(err);
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
resolve(filename)
}
});
}
else {
// reject(new Error("FileNotExist"))
console.log(" FileNotExist ");
}
});
})
}
// note the use of async
app.post('/cusdetails', isLoggedIn, async function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
// using await to pause excution, waits till query is finished
const [rows] = await connection.query(insertQuerys,[cusid])
rows.forEach( (row) => {
name=row.fncus;
});
// then you can
var result = await some(k)
...
Note however this way you loose the advantage of concurrent execution, as it's kindoff blocking. If the result of one call is not used in another, you can execute in parallel and await for result to achieve sequencing like
const [rows] = connection.query(insertQuerys,[cusid])
var result = some(k)
console.log(await rows) // do something
console.log(await result) // do something
JavaScript is asynchronous. This means that if you have a function with a callback (i.e. your query), the callback will be called asynchronously, at an unknown time, while the other code executes.
You need to look up some tutorials how to deal with callbacks, to get a proper understanding of it. Another method is using async/await and/or promises.
Basically, if you take the following code:
console.log("this will print first");
setTimeout(function () {
console.log("this will print last");
}, 1000);
console.log("this will print second");
If you run the code above, the top level is executed synchronously, so, it first calls console.log, then it executes setTimeout, which is synchronous. It sets a timeout, then says "I'm ready", and the code continues to the other console.log. After 1 second (1000 milliseconds), the callback in the setTimeout function is executed, and only then that console.log is called. You can not make the rest of the code wait this way, you need to restructure your code or read into promises.
I am making use of "socket.io-client" and "socket.io stream" to make a request and then stream some data. I have the following code that handles this logic
Client Server Logic
router.get('/writeData', function(req, res) {
var io = req.app.get('socketio');
var nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
var nameNodeData = {};
async.waterfall([
checkForDataNodes,
readFileFromS3
], function(err, result) {
if (err !== null) {
res.json(err);
}else{
res.json("Finished Writing to DN's");
}
});
function checkForDataNodes(cb) {
nameNodeSocket.on('nameNodeData', function(data) {
nameNodeData = data;
console.log(nameNodeData);
cb(null, nameNodeData);
});
if (nameNodeData.numDataNodes === 0) {
cb("No datanodes found");
}
}
function readFileFromS3(nameNodeData, cb) {
for (var i in nameNodeData['blockToDataNodes']) {
var IP = nameNodeData['blockToDataNodes'][i]['ipValue'];
var dataNodeSocket = io.connect('http://'+ IP +":5000");
var ss = require("socket.io-stream");
var stream = ss.createStream();
var byteStartRange = nameNodeData['blockToDataNodes'][i]['byteStart'];
var byteStopRange = nameNodeData['blockToDataNodes'][i]['byteStop'];
paramsWithRange['Range'] = "bytes=" + byteStartRange.toString() + "-" + byteStopRange.toString();
//var file = require('fs').createWriteStream('testFile' + i + '.txt');
var getFileName = nameNodeData['blockToDataNodes'][i]['key'].split('/');
var fileData = {
'mainFile': paramsWithRange['Key'].split('/')[1],
'blockName': getFileName[1]
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
s3.getObject(paramsWithRange).createReadStream().pipe(stream);
//dataNodeSocket.disconnect();
}
cb(null);
}
});
Server Logic (that gets the data)
var dataNodeIO = require('socket.io')(server);
var ss = require("socket.io-stream");
dataNodeIO.on('connection', function(socket) {
console.log("Succesfully connected!");
ss(socket).on('sendData', function(stream, data) {
var IP = data['ipValue'];
var blockName = data['blockName'];
var mainFile = data['mainFile'];
dataNode.makeDir(mainFile);
dataNode.addToReport(mainFile, blockName);
stream.pipe(fs.createWriteStream(mainFile + '/' + blockName));
});
});
How can I properly disconnect the connections in function readFileFromS3. I have noticed using dataNodeSocket.disconnect() at the end does not work as I cannot verify the data was received on the 2nd server. But if I comment it out, I can see the data being streamed to the second server.
My objective is to close the connections in Client Server side
It appears that the main problem with closing the socket is that you weren't waiting for the stream to be done writing before trying to close the socket. So, because the writing is all asynchronous and finishes sometime later, you were trying to close the socket before the data had been written.
Also because you were putting asynchronous operations inside a for loop, you were also running all your operations in parallel which may not be exactly what you want as it makes error handling more difficult and server load more difficult.
Here's the code I would suggest that does the following:
Create a function streamFileFromS3() that streams a single file and returns a promise that will notify when it's done.
Use await in a for loop with that streamFileFromS3() to serialize the operations. You don't have to serialize them, but then you would have to change your error handling to figure out what to do if one errors while the others are already running and you'd have to be more careful about concurrency issues.
Use try/catch to catch any errors from streamFileFromS3().
Add error handling on the stream.
Change all occurrences of data['propertyName'] to data.propertyName. The only time you need to use brackets is if the property name contains a character that is not allowed in a Javascript identifier or if the property name is in a variable. Otherwise, the dot notation is preferred.
Add socket.io connection error handling logic for both socket.io connections.
Set returned status to 500 when there's an error processing the request
So, here's the code for that:
const ss = require("socket.io-stream");
router.get('/writeData', function(req, res) {
const io = req.app.get('socketio');
function streamFileFromS3(ip, data) {
return new Promise((resolve, reject) => {
const dataNodeSocket = io.connect(`http://${ip}:5000`);
dataNodeSocket.on('connect_error', reject);
dataNodeSocket.on('connect_timeout', () {
reject(new Error(`timeout connecting to http://${ip}:5000`));
});
dataNodeSocket.on('connection', () => {
// dataNodeSocket connected now
const stream = ss.createStream().on('error', reject);
paramsWithRange.Range = `bytes=${data.byteStart}-${data.byteStop}`;
const filename = data.key.split('/')[1];
const fileData = {
'mainFile': paramsWithRange.Key.split('/')[1],
'blockName': filename
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
// get S3 data and pipe it to the socket.io stream
s3.getObject(paramsWithRange).createReadStream().on('error', reject).pipe(stream);
stream.on('close', () => {
dataNodeSocket.disconnect();
resolve();
});
});
});
}
function connectError(msg) {
res.status(500).send(`Error connecting to ${NAMENODE_ADDRESS}`);
}
const nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
nameNodeSocket.on('connect_error', connectError).on('connect_timeout', connectError);
nameNodeSocket.on('nameNodeData', async (nameNodeData) => {
try {
for (let item of nameNodeData.blockToDataNodes) {
await streamFileFromS3(item.ipValue, item);
}
res.json("Finished Writing to DN's");
} catch(e) {
res.status(500).json(e);
}
});
});
Other notes:
I don't know what paramsWithRange is as it is not declared here and when you were doing everything in parallel, it was getting shared among all the connections which is asking for a concurrency issue. In my serialized implementation, it's probably safe to share it, but the way it is now bothers me as it's a concurrency issue waiting to happen.
I want to use gridfs-stream in a nodejs application.
A simple example is given in the documentation:
var mongoose = require('mongoose');
var Grid = require('gridfs-stream');
Grid.mongo = mongoose.mongo;
mongoose.connect('mongodb://localhost:27017/test');
// make sure the db instance is open before passing into `Grid`
mongoose.connection.once('open', function () {
var gfs = Grid(mongoose.connection);
// all set!
})
My problem is described by the comment:
make sure the db instance is open before passing into Grid
I try to use gfs in a post request. Now when the code gets initialized, the gfs variable is not defined yet.
api.post('/upload', function(req, res) {
req.pipe(gfs.createWriteStream({
filename: 'test'
}).on('close', function(savedFile){
console.log('file saved', savedFile);
return res.json({file: savedFile});
}));
})
Initializing my route from a callback seems kind of odd.
I read in this post (Asynchronous initialization of Node.js module) that require('') is performed synchronous, and since I rely on the connection being established, I'm kind of forced to wait
Basically I'm not sure if I should use a async pattern on startup now, or if I just miss a more elegant way to solve this.
I have a very similar problem with my server. In my case I am reading https certs asynchronously, the software version from git asynchronously and I want to make sure I have it all together by the time the user comes to log in so I can pass the software version back as a reply to login.
The solution is to use promises. Create the promises on user start up for each activity. Then in the code where you want to be sure its all ready, just call then on either the promise itself or Promise.all(array of promises).then()
Here is an example of what I am doing to read the ssl certs to start the server
class Web {
constructor(manager,logger) {
var self = this;
this.server = false;
this.logger = logger;
var key = new Promise((resolve,reject) => {
fs.readFile(path.resolve(__dirname, 'key.pem'),(err,data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
var cert = new Promise((resolve,reject) => {
fs.readFile(path.resolve(__dirname, 'certificate.pem'), (err,data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
});
});
Promise.all([key,cert]).then(values => {
var certs = {
key: values[0],
cert: values[1],
};
return certs;
}).then(certs => {
self.server = require('http2').createServer(certs,(req,res) => {
// NOW Started and can do the rest of the stuff
});
self.server.listen(...);
});
NEEDS SOME MORE CLOSING BRACKETS
I'm looking for a package (or pattern) to handle events from mongodb so I can avoid nested callbacks and keep mongodb logic out of my request handlers.
Right now I've got code that looks like this:
start-express.js (server)
var express = require('express');
var Resource = require('express-resource');
var app = express.createServer();
// create express-resource handler which essentially does app.get('things', ...)
var things = app.resource('things', require('./things.js'));
app.listen(port);
things.js (express-resource request handler)
require('./things-provider');
// handle request 'http://example.com/things'
exports.index = function(request, response) {
sendThings(db, response);
};
things-provider.js (handles mongodb queries)
var mongodb = require('mongodb')
// create database connection
var server = new mongodb.Server(host, port, {auto_reconnect: true});
var db = new mongodb.Db(dbName, server);
db.open(function (err, db) {
if (err) { }
// auto_reconnect will reopen connection when needed
});
function sendThings(db, response) {
db.collection('things', function(err, collection) {
collection.find(function(err, cursor) {
cursor.toArray(function(err, things) {
response.send(things);
});
});
});
}
module.exports.sendThings = sendThings;
I'd like to avoid passing my http response object to my database handler or (worse) handling my db request in my http response handler.
I recently realized that what I want to do is create an event handler that registers an http request/response and waits for a response (event) from database before processing and sending the http response.
That sounds like a lot of duplication of what node.js already does though. Is there an existing framework that handles this use case?
Here's the solution I've come up with.
I used mongojs which greatly simplifies the mongodb interface --at the cost of flexibility in configuration-- but it hides the nested callbacks the mongodb driver requires. It also makes the syntax much more like the mongo client.
I then wrap the HTTP Response object in a closure and pass this closure to the mongodb query method in a callback.
var MongoProvider = require('./MongoProvider');
MongoProvider.setCollection('things');
exports.index = function(request, response){
function sendResponse(err, data) {
if (err) {
response.send(500, err);
}
response.send(data);
};
MongoProvider.fetchAll(things, sendResponse);
};
It is still essentially just passing the response object to the database provider, but by wrapping it in a closure that knows how to handle the response, it keeps that logic out of my database module.
A slight improvement is to use a function to create a response handler closure outside my request handler:
function makeSendResponse(response){
return function sendResponse(err, data) {
if (err) {
console.warn(err);
response.send(500, {error: err});
return;
}
response.send(data);
};
}
So now my request handler just looks like this:
exports.index = function(request, response) {
response.send(makeSendResponse(response));
}
And my MongoProvider looks like this:
var mongojs = require('mongojs');
MongoProvider = function(config) {
this.configure(config);
this.db = mongojs.connect(this.url, this.collections);
}
MongoProvider.prototype.configure = function(config) {
this.url = config.host + "/" + config.name;
this.collections = config.collections;
}
MongoProvider.prototype.connect = function(url, collections) {
return mongojs.connect(this.url, this.collections);
}
MongoProvider.prototype.fetchAll = function fetchAll(collection, callback) {
this.db(collection).find(callback);
}
MongoProvider.prototype.fetchById = function fetchById(id, collection, callback) {
var objectId = collection.db.bson_serializer.ObjectID.createFromHexString(id.toString());
this.db(collection).findOne({ "_id": objectId }, callback);
}
MongoProvider.prototype.fetchMatches = function fetchMatches(json, collection, callback) {
this.db(collection).find(Json.parse(json), callback);
}
module.exports = MongoProvider;
I can also extend MongoProvider for specific collections to simplify the API and do additional validation:
ThingsProvider = function(config) {
this.collection = 'things';
this.mongoProvider = new MongoProvider(config);
things = mongoProvider.db.collection('things');
}
ThingsProvider.prototype.fetchAll = function(callback) {
things.fetchAll(callback);
}
//etc...
module.exports = ThingsProvider;
Well, first off I find Mongoose somewhat easier to use in a well-structured app than straight mongo. So that might help you.
Second, I think what you're trying to do could easily be accomplished through middleware (app level or route level), since you're using express already. Alternatively, parameter-filtering, if your query will vary based on params. A pattern I've seen on the last looks like this:
var User = mongoose.model("user'); // assumes your schema is previously defined
app.param('user_id', function(req,res,next, id){
User.find(id, function(err,user){
if(err) next(err);
else {
req.user = user;
next();
}
});
});
It still has some nesting, but not nearly so bad as your example, much more manageable. Then, let's say you have a '/profile' endpoint, you can just do:
app.get('/profile/:user_id', function(req,res){ res.render('profile', req.user); }