Being new to Node, I am still having some troubles with callbacks.
In the mapBpiIfindex function I am trying to loop through all of the VLANs found by the vlans function. Once it has looped through all VLANs, creating the map, I want to output the map to the browser. But, the only output I am getting is {}. How can I send the mapping to the browser? I am not even sure if I am using my callbacks correctly.
var express = require('express');
var router = express.Router();
var snmp = require('snmp-native');
// Create a Session with explicit default host, port, and community.
let session = new snmp.Session({ host: 'AASW0120', port: 161, community: 'community' })
let Mibs = {
hostname: [1,3,6,1,2,1,1,5,0],
vlans: [1,3,6,1,4,1,9,9,46,1,3,1,1,2],
dot1dBasePortIfIndex: [1,3,6,1,2,1,17,1,4,1,2]
}
/* Get all VLANs on switch */
function vlans(snmpSession, cb) {
let vlans = []
session.getSubtree({ oid: Mibs.vlans }, function (error, varbinds) {
if (error) {
console.log('Fail :(');
} else {
varbinds.forEach(function (varbind) {
vlans.push(varbind.oid[varbind.oid.length -1])
})
}
cb(vlans)
})
}
/* Map BPIs to Ifindices */
function mapBpiIfindex(session, cb) {
let map = {}
vlans(session, function (vlans) {
vlans.forEach(function (vlan) {
session.getSubtree({oid: Mibs.dot1dBasePortIfIndex, community: 'community#' + vlan}, function (error, varbinds) {
if (error) {
console.log('Fail :(')
} else {
varbinds.forEach(function (varbind) {
map[varbind.oid[varbind.oid.length -1]] = {ifindex: varbind.value, vlan: vlan}
})
}
})
})
cb(map)
})
}
router.get('/vlans', function (req, res, next) {
vlans(session, function (vlans) {
res.send(vlans)
})
})
router.get('/bpi-ifindex', function (req, res, next) {
mapBpiIfindex(session, function (mapping) {
res.send(mapping)
})
})
The answer is no, youre not using it correctly ;)
A few things here:
You should be clear that only the code within the callback is executed after the operation has finished, so cb(map)does not wait until all youre looped callbacks have finished. Thats why nothing is returned (because when cb is called, the async functions have not finished yet and map values are undefined. Have a look at this How do I return the response from an asynchronous call?, its the same principle.
Have a look at async module. Specifically, the do* or whilst methods. It'll help you process loops with async function calls.
Apart from that, you should not use forEach if you mind about performance.
Related
I am trying to append some data to my request object using a middleware, but I want to do it only once the server is up.
So I tried doing it with a middleware, while trying to use a function's context, but it's a bit problematic to perform such an action on a middleware, because I cannot pass a promise as a middleware.
This is what I'm trying to do:
const setupData = async () => {
const data = await getSomeData();
return (req, res, next) => {
req.data = data;
next();
}
}
app.use(setupData());
I tried using the solution suggested here, but it won't work as this will happen on every request.
Any idea how can I go around this? I can always put the info on a global var, but I would like to avoid it.
I also saw some in-memory packages to help with it (such as node-cache), but I would like to do it with a middleware.
Thanks in advance
Just cache the result using a normal variable:
let data = null;
function setupData (req, res, next) {
if (data !== null) {
req.data = data;
next();
}
else {
getSomeData().then(result => {
data = result
req.data = data;
next();
});
}
}
app.use(setupData);
This is the minimal, least complicated implementation. You can of course refactor it to be much DRYer and less prone to mistakes by taking out the caching logic:
Cleaner Implementation
let cache = null;
async function getCachedData() {
if (cache === null) {
cache = await getSomeData();
}
return cache;
}
Which makes setupData much cleaner:
function setupData (req, res, next) {
getCachedData().then(data => {
req.data = data;
next();
});
}
Either way, the cache is triggered on the first request. This means that there is a possibility that a second request may arrive before the data is possibly cached. So at startup the getSomeData() function may run more than once.
Really call getSomeData() ONLY ONCE
If you really want to call getSomeData only once you must call it before setting up Express:
async function main () {
const data = await getSomeData();
const app = express();
//
// set up express middlewares...
//
app.use((req,res,next) => {
req.data = data;
next();
});
//
// set up routes...
//
app.listen(config.port);
}
main(); // start everything
The key here is to realize that we have been trying to do everything backwards: to set up a constant value asynchronously AFTER starting to set up Express. The natural flow of the program wants the constant value to exist BEFORE we begin setting up Express so we only perform everything else inside the async function (here called main). Not try to run the async function while setting up Express.
You can do it without async:-
const setupData = (req, res, next) => {
// You can put a condition here so that it runs only once
getSomeData().then((data) => {
req.app.locals.data = data //The data can be accessed in the next middleware using req.app.locals
next();
}).catch((error) => {
console.log("Error Occured");
res.status(400).end("Error Occurred");
})
}
app.use(setupData);
You should see the documentation of getSomeData and see how it works
I'm struggling right now with "async".
I'm starting with NestJs Framework and i need run a bunch of task on parallel.
Before, using Nodejs + Express only, i have been useing something like this
...
...
.get('/some/path', function(){
async.parallel({
todayBalance: async.apply(function (cb) {
return SomeFunction(params, cb);
}),
currentWeek: async.apply(function (cb) {
return SomeFunction2(params, cb);
}),
...
...
// more tasks
},
function (err, results) {
# some code for logic
# here i get my result and gather data and RETURN
});
});
Nowadays, using NestJs framework, i have got something like this
myservice.ts
This is a service created for doing this.
// Service Definitions
async someFunction(userBusinessId: string): Promise<any> {
// i tried to use same strategy from Async
// but nothing accuring
async.parallel({
todayBalance: async.apply(function (cb) {
return SomeFunction(params, cb);
}),
currentWeek: async.apply(function (cb) {
return SomeFunction2(params, cb);
}),
...
...
// more tasks
},
function (err, results) {
# some code for logic
# here i get my result and gather data and RETURN
# DOESNT RETURN, NEVER EVER GETS HERE
});
}
Any idea what's wrong?
Thanks for your support!
First off I made two basics mistake
1. I Forgot use cb Function
using Nodejs + Express we just made something like this
In this case I use mysql
const SomeFunction= (userBusinessId, cb) => {
cnx.connection.query(`some pretty query`,
cb // <===== use Callback here
);
};
now using Nestjs i tried(badly results) to made something like this. Ignoring cb
const SomeFunction= (userBusinessId, cb) => {
const data = await getManager().query(`some pretty query`);
return data; // <===== WRONG INSTEAD USE cb Function, using like this parallel function will never triggered
};
Instead return single data, we must trigger the cb function overloading with result
Documentation
const SomeFunction= (userBusinessId, cb) => {
...
cb(null, data); // EXECUTE LIKE THIS, this will resume the pipe. null param suppose no error
};
2. Try to return the function service value inside the Async callback
Even if you try to do something like
async list():Promise<any>{
async.parallel({
...
// more tasks
},
function (err, results) {
return data;
});
}
OR
async list():Promise<any>{
const data = await async.parallel({
...
// more tasks
},
function (err, results) {
return data;
});
return data;
}
this funtion with always return undefined. even if you remove the Funtion Type Promise<any>
For avoid this you must return a Promise like
async list():Promise<any>{
return new Promise( (resolver, reject) => {
async.parallel({
...
// more tasks
},
function (err, results) {
if(err)
reject(err);
...
resolver(results);
});
} )
}
Hope this help you too!
I have two arrays. I would like to iterate the arrays using async.foreach.
But when I do so, only the second array is getting executed.How to execute both.
Below is my code:
var _sizesArray1 = [_2000px, _1400px]
var _sizesArray2 = [_800px, _400px, _200px, _100px]
async.forEachOf(_sizesArray1, function(value, key, callback) {
async.waterfall([
function download(next){
//code
},
function convert(response, next) {
//code
},
function process(response, next) {
gm(response).command('convert')
.resize(_sizesArray[key].width,_sizesArray[key].width)
.gravity('Center')
.extent(_sizesArray[key].width,_sizesArray[key].width)
.quality('20')
.toBuffer(
'JPG', function(err,
buffer) {
if (err) {
next(err);
} else {
console.timeEnd(
"processImage array1"
);
next(null,
buffer,
key);
}
});
}
});
async.forEachOf(_sizesArray2, function(value, key, callback) {
async.waterfall([
function download1(next){
//code
},
function convert2(response, next) {
//code
},
function process3(response, next) {
//code
}
});
In my code, only array2 is getting invoked.Why don't first one get executed.
Is there any mistake in my code. Can somebody help resolve this.
How about this simple technique:
var allSizes = _sizesArray1.concat(_sizesArray2);
async.foreach(allSizes, function(value, key, next) {
// do whatever you like with each item in the concatenated arrays
// once you are done move to the next item
next()
})
Updated based on comments
Version 1, based on callbacks (welcome callback hell):
function asyncIterate(array, onEach, onDone) {
async.forEach(array, (val, index, onNext) => {
onEach(val, key, function onDecoratedNext() {
// tell the async.forEach it's ready for the next item
onNext();
// but if it got to the end,
// then mark the completion of the whole iteration
if (index === array.length) {
onDone();
}
});
});
}
and implement it like:
function doAsyncStuffWithEachItem(val, index, cb) {
// do async stuff with each item
// make sure to mark the completion of the async operation
cb();
}
asyncIterate(
_sizesArray1,
doAsyncStuffWithEachItem,
function onDone() {
// once the 1st iteration finished, time to move to the 2nd array
asyncIterate(
_sizesArray2,
doAsyncStuffWithEachItem,
function onDone2() {
// great we are done
// What if we have _sizesArray3, 4, 5 ... ?
// Well, we can continue to nest more callback here
// but in that way we'll soon end up with callback hell
// and that's a big NoNo
}
)
}
);
Version 2, based on Promises:
To avoid callback hell, luckily we can use Promises. Something like this should do it:
const promisifiedAsyncIterate = (array, onEach) =>
new Promise((resolve) => {
asyncIterate(array, onEach, resolve);
});
and use it like:
promisifiedAsyncIterate(_sizeArray1, doAsyncStuffWithEachItem)
.then(() => promisifiedAsyncIterate(_sizeArray2, doAsyncStuffWithEachItem))
.then(() => promisifiedAsyncIterate(_sizeArray3, doAsyncStuffWithEachItem))
.then(() => promisifiedAsyncIterate(_sizeArray4, doAsyncStuffWithEachItem))
It could be abstracted and cleaned up even more, or even made completely dynamic – say you have an array of sizeArrays that you pass in to your function, but I think this is enough for now :). Hope it helps.
I'm trying to build an API for a single-page web app using AWS Lambda and the Serverless Framework. I want to use Redis Cloud for storage, mostly for its combination of speed and data persistence. I may use more Redis Cloud features in the future, so I'd prefer to avoid using ElastiCache for this. My Redis Cloud instance is running in the same AWS region as my function.
I have a function called related that takes a hashtag from a GET request to an API endpoint, and checks to see if there's an entry for it in the database. If it's there, it should return the results immediately. If not, it should query RiteTag, write the results to Redis, and then return the results to the user.
I'm pretty new to this, so I'm probably doing something adorably naive. Here's the event handler:
'use strict'
const lib = require('../lib/related')
module.exports.handler = function (event, context) {
lib.respond(event, (err, res) => {
if (err) {
return context.fail(err)
} else {
return context.succeed(res)
}
})
}
Here's the ../lib/related.js file:
var redis = require('redis')
var jsonify = require('redis-jsonify')
var rt = require('./ritetag')
var redisOptions = {
host: process.env.REDIS_URL,
port: process.env.REDIS_PORT,
password: process.env.REDIS_PASS
}
var client = jsonify(redis.createClient(redisOptions))
module.exports.respond = function (event, callback) {
var tag = event.hashtag.replace(/^#/, '')
var key = 'related:' + tag
client.on('connect', () => {
console.log('Connected:', client.connected)
})
client.on('end', () => {
console.log('Connection closed.')
})
client.on('ready', function () {
client.get(key, (err, res) => {
if (err) {
client.quit()
callback(err)
} else {
if (res) {
// Tag is found in Redis, so send results directly.
client.quit()
callback(null, res)
} else {
// Tag is not yet in Redis, so query Ritetag.
rt.hashtagDirectory(tag, (err, res) => {
if (err) {
client.quit()
callback(err)
} else {
client.set(key, res, (err) => {
if (err) {
callback(err)
} else {
client.quit()
callback(null, res)
}
})
}
})
}
}
})
})
}
All of this works as expected, to a point. If I run the function locally (using sls function run related), I have no problems whatsoever—tags are read from and written to the Redis database as they should be. However, when I deploy it (using sls dash deploy), it works the first time it's run after deployment, and then stops working. All subsequent attempts to run it simply return null to the browser (or Postman, or curl, or the web app). This is true regardless of whether the tag I use for testing is already in the database or not. If I then re-deploy, making no changes to the function itself, it works again—once.
On my local machine, the function first logs Connected: true to the console, then the results of the query, then Connection closed. On AWS, it logs Connected: true, then the results of the query, and that's it. On the second run, it logs Connection closed. and nothing else. On the third and all subsequent runs, it logs nothing at all. Neither environment ever reports any errors.
It seems pretty clear that the problem is with the connection to Redis. If I don't close it in the callbacks, then subsequent attempts to call the function just time out. I've also tried using redis.unref instead of redis.quit, but that didn't seem to make any difference.
Any help would be greatly appreciated.
I've now solved my own problem, and I hope I can be of help to someone experiencing this problem in the future.
There are two major considerations when connecting to a database like I did in the code above from a Lambda function:
Once context.succeed(), context.fail(), or context.done() is called, AWS may freeze any processes that haven't finished yet. This is what was causing AWS to log Connection closed on the second call to my API endpoint—the process was frozen just before Redis finished closing, then thawed on the next call, at which point it continued right where it left off, reporting that the connection was closed. Takeaway: if you want to close your database connection, make sure it's fully closed before you call one of those methods. You can do this by putting a callback in an event handler that's triggered by a connection close (.on('end'), in my case).
If you split your code into separate files and require them at the top of each file, like I did, Amazon will cache as many of those modules as possible in memory. If that's causing problems, try moving the require() calls inside a function instead of at the top of the file, then exporting that function. Those modules will then be re-imported whenever the function is run.
Here's my updated code. Note that I've also put my Redis configuration into a separate file, so I can import it into other Lambda functions without duplicating code.
The Event Handler
'use strict'
const lib = require('../lib/related')
module.exports.handler = function (event, context) {
lib.respond(event, (err, res) => {
if (err) {
return context.fail(err)
} else {
return context.succeed(res)
}
})
}
Redis Configuration
module.exports = () => {
const redis = require('redis')
const jsonify = require('redis-jsonify')
const redisOptions = {
host: process.env.REDIS_URL,
port: process.env.REDIS_PORT,
password: process.env.REDIS_PASS
}
return jsonify(redis.createClient(redisOptions))
}
The Function
'use strict'
const rt = require('./ritetag')
module.exports.respond = function (event, callback) {
const redis = require('./redis')()
const tag = event.hashtag.replace(/^#/, '')
const key = 'related:' + tag
let error, response
redis.on('end', () => {
callback(error, response)
})
redis.on('ready', function () {
redis.get(key, (err, res) => {
if (err) {
redis.quit(() => {
error = err
})
} else {
if (res) {
// Tag is found in Redis, so send results directly.
redis.quit(() => {
response = res
})
} else {
// Tag is not yet in Redis, so query Ritetag.
rt.hashtagDirectory(tag, (err, res) => {
if (err) {
redis.quit(() => {
error = err
})
} else {
redis.set(key, res, (err) => {
if (err) {
redis.quit(() => {
error = err
})
} else {
redis.quit(() => {
response = res
})
}
})
}
})
}
}
})
})
}
This works exactly as it should—and it's blazing fast, too.
I'm developing a RESTful API with Node.js, Express and MySql. No ORM used.
I want to reuse as much code as possible in order to handle requests for resources.
I'm going to implement a callback factory and I'd like to hear from you about my approach.
api.js
It is the main application file.
'use strict';
var
http = require('http'),
express = require('express'),
app = express(),
env = app.get('env'),
config = require('./config')[env],
pool = require('./pool')(config.database), /* node-mysql pool */
people = require('./routes/people')(pool); /* route handlers for people resource */
app.use(express.json());
app.use(express.urlencoded());
app.use(app.router);
app.get('/people', people.findAll); /* middleware handling request for the resource */
http.createServer(app).listen(8000);
routes/people.js
It is the file containing code to handle requests for /peopleresource.
'use strict';
module.exports = {
var
async = require('async'),
CallbackFactory = require('../CallbackFactory'),
people = {};
people.findAll = function (req, res, next) {
async.waterfall(
[
CallbackFactory.createCallback('getPoolConnection', pool),
CallbackFactory.createCallback('lastQuery', 'SELECT * FROM person'),
CallbackFactory.createCallback('json', res)
],
CallbackFactory.getCallback('next', next);
);
};
return people;
};
CallbackFactory.js
The module exports the createCallback method only.
The first argument to createCallback is always the callback name.
Other arguments are specific to the callback to create.
'use strict';
var
factoryMethods: {
getPoolConnection: function (pool) {
return function (callback) {
pool.getConnection(function (err, connection) {
callback(err, connection);
};
};
},
lastQuery: function (sql, values) {
return function (connection, callback) {
connection.query(sql, values, function (err, result) {
connection.release();
callback(err, result);
});
};
},
json: function (res) {
return function (result, callback) {
res.json(result);
callback();
};
},
next: function (next) {
return function (err) {
if (err) {
return next(err);
}
};
},
};
module.exports = {
createCallback: function () {
var
args = Array.prototype.slice.call(arguments),
name = args.shift();
return factoryMethods[name].apply(null, args);
}
};
Conclusion
I googled and searched SO in order to find approaches that are useful to code reuse.
I didn't find anything. Maybe it is not the correct/best approach. What do you think about it?
I think you're on the wrong track here. Callback factory is not a common pattern because it's adding a layer of abstraction without reducing complexity in any way. Phrased differently, think of async.waterfall as already providing the abstraction layer that you're looking for. Or, if you like named functions, that I recommend async.auto, which also automatically determines which functions can operation in serial and parallel based on the dependencies you specify.
I would recommend having async.waterfall or async.auto call your functions directly. If you want additional abstraction and less code, use an ORM like Sequelize. But I see your factory complicating understanding your code for anyone else who comes along later.