Please note that I am a total noob when it comes to Node.js.
I have created a WebSockets app that will send server status and usage every second when a client connects (through basic authentications so that only the authorized clients can contact).
I want to add disk usage info to the data and is using the nodejs-disks module for that. And this is the function I've created to get the info I need:
function getDiskStats()
{
var output = {
count: 0,
space: {
total: 0,
used: 0,
free: 0
}
};
df.drives(function(err, drives)
{
df.drivesDetail(drives, function(err, data)
{
output.count = data.length;
for(var i = 0; i < data.length; i++)
{
output.space.total += parseInt(data[i].total);
output.space.used += parseInt(data[i].used);
output.space.free += parseInt(data[i].available)
}
});
});
return output;
}
But when the output is returned, everything is 0 and if I log variable in the console right after the for loop I get the values. My guess is that its because of the async method nodejs-disks uses to get the data. I am not sure what I should do next and I HAVE google a lot but couldn't find a good solution.
Thanks!
Like #mevernom said, you are returning your object before any of the asynchronous objects have a chance to work.
You can modify your function to work asynchronously by making your getDiskStats function take a callback that it calls when everything else is done.
function getDiskStats(callback) {
var output = {
count: 0,
space: {
total: 0,
used: 0,
free: 0
}
};
df.drives(function(err, drives) {
if (err) return callback(err); // stop on error
df.drivesDetail(drives, function(err, data) {
if (err) return callback(err); // stop on error
output.count = data.length;
for(var i = 0; i < data.length; i++) {
output.space.total += parseInt(data[i].total);
output.space.used += parseInt(data[i].used);
output.space.free += parseInt(data[i].available)
}
// done processing, call callback
callback(null, output);
});
});
}
You would then have to use your function like
getDiskStats(function(err, value) {
// do stuff!
});
Because your getDistStats returned immediately, before callbacks finished their tasks.
If you are not comfortable with asynchronous nature, perhaps you could try async module.
Related
I need to loop through a json and make multiple getitem calls to dynamo db. My issue is that node js flies through the code not waiting for the function return so I can't create an xml that has a header, multiple calculated lines, and then a footer.
In the sample below, the !make footer would be written long before the function taxrate finishes. How can I force the script to wait for the taxrate function to finish?
!make xml header
for(i=0; i<linelength; i++)
{
business_unit = '100'
invoice_line = 1
total = 100
taxrate(business_unit, invoice_line, total);
!write line xml
}
!make xml footer
function taxrate(business_unit, i, gross_total) {
const params = {
Key: {
"tax_rate": {
S: business_unit
}
},
TableName:"tax_table"
};
dynamodb.getItem(params,function(err, data){
if(err) {
console.log("call error");
console.log(err);
} else {
console.log(data.Item.tax.N);
return(data.Item.tax.N);
}
});
There are several patterns for solving this problem; the first one is one you've already got in your code, which is the callback function. The function you are passing to dynamodb.GetItem() is a callback that executes after some other code is run, so that the result (data) can be accessed.
A more modern way is to use Promises, which take a bit of time to understand. It's worth doing this though, so that you can understand what's happening behind the scenes when you use the newest way, which is async and await, which would look like this:
exports.lambdaHandler = async (event, context) => {
!make xml header
for(i=0; i<linelength; i++)
{
business_unit = '100'
invoice_line = 1
total = 100
await taxrate(business_unit, invoice_line, total);
!write line xml
}
!make xml footer
}
async function taxrate(business_unit, i, gross_total) {
const params = {
Key: {
"tax_rate": {
S: business_unit
}
},
TableName:"tax_table"
};
try {
let data = await dynamodb.getItem(params).promise();
return data.Item.tax.N;
} catch(err) {
console.log("call error");
console.log(err);
}
}
Notice that both lambdaHandler() and taxrate() have been turned into async functions. In the for loop, we've now awaited the result of taxrate(). The callback to getItem has been replaced with a method that returns a promise.
I'm trying to loop & sort a large number of data ( the whole ethereum blockchain lol )
I'm trying to create a record of all transactions for every address.
Obviously this is a very intensive process and I'm not sure how to make it more efficient beyond what I have (which isn't that efficient)
It starts out quick but I'm thinking now it has slowed because of the lookup for the address in the txs object.
Any help opinions / help is greatly appreciated.
https://giphy.com/gifs/3o6fJ7KWqxESY9okk8
var txs = {};
var i = 0;
// Loop over blocks
(function loop () {
setTimeout(function () {
// Get current block
var block = web3.eth.getBlock(i, true, (error, block) => {
// debugger;
// Loop over transactions in block
for(var j = 0; j < block.transactions.length; j++) {
// debugger;
if(txs[block.transactions[j].to]) {
txs[block.transactions[j].to].transactions.push(block.transactions[j]);
} else if (txs[block.transactions[j].to]) {
txs[block.transactions[j].from].transactions.push(block.transactions[j]);
} else {
txs[block.transactions[j].to] = {
transactions: [block.transactions[j]]
}
txs[block.transactions[j].from] = {
transactions: [block.transactions[j]]
}
}
}
});
i++
if (i < highestBlock) {
loop();
}
}, 50);
})();
I think that your code has an error in it located at "else-if", it seems that you should use the txs[block.transactions[j].from] property instead of the txs[block.transactions[j].to]. If you simply want to accomplish a recursive pattern you could use the setImmediate function or the process.nextTick method. If you use node.js v6+ you could use a Map instead of the object.
So in Node.js let's say I have the following code:
for (var i = 0; i < 1000; i++) {
someAsynchronousFunction(function(err,res) {
// Do a bunch of stuff
callback(null,res);
});
}
But I want this to run synchronously. I know this is not advised in Node JS, but I am just trying to understand the language. I tried implementing the following solution, but it just ends up hanging during runtime:
for (var i = 0; i < 1000; i++) {
var going = true;
someAsynchronousFunction(function(err,res) {
// Do a bunch of stuff
callback(null,res);
going = false;
});
while (going) {
}
}
What is going wrong and what is the correct way to do this?
One of the best way to do that is to use the async library.
async.timesSeries(1000, function(n, next){
someAsynchronousFunction(next);
});
Or you can do that with async.series() function.
.times() documentation : http://caolan.github.io/async/docs.html#.timesSeries
Another way to do this is using Promises to generate a sequential execution of them thanks to Array#reduce :
// Function that returns Promise that is fllfiled after a second.
function asyncFunc (x){
return new Promise((rs, rj)=>{
setTimeout( ()=>{
console.log('Hello ', x);
rs();
}, 1000)
});
}
// Generate an array filed with values : [0, 1, 2, 3, ...]
Array.from({length : 1000}, (el, i)=> i)
// loop througth the array chaining the promises.
.reduce( (promise, value) =>
promise.then(asyncFunc.bind(null, value))
, Promise.resolve(null));
This is the code am running which returns the Range Maximum call stack size exceeded error.
// to insert 10000 values on to mongodb using node.js
var MongoClient = require('mongodb').MongoClient;
var mongoServer = require('mongodb').Server;
var serverOptions = {
'auto_reconnect': true,
'poolSize': 100
};
var i=0;
var async =require('async');
var mongoClient = new MongoClient(new mongoServer('localhost', 27017, serverOptions));
var db = mongoClient.db('test');
var collection = db.collection('new_file_test');
mongoClient.open(function (err, mongoClient)
{
if(err){console.log(err)};
function start(i,call)
{
if(i<10000) {
call(start);
}
}
function pass(callback)
{
Insert(save);
i++;
callback(i,pass);
}
start(i,pass);
});
function Insert(callback) {
console.log("Inserting" );
var doc={
'trip_paramid':i,
'tripid':'116',
'lattitude':'12.8929183',
'longitude':'77.63627',
'speed':'2',
'heading':'0',
'altitude':'80469',
'address':'qwertyasdfgxcvbn',
'engine_status':'Normal',
'oil_pressure': '83.12',
'water_temp': '28',
'fuel_content':'0',
'brake':'Normal',
'creation_time':'2013-08-31 23:22:17',
'brakelight_status':'Normal',
'battery_status':'12.68',
'event_code':'8',
'dbinsert_time':'2013-08-31 23:24:59',
'gsm_status':'-51',
'cell_id':'45',
'vehicle_id':'123456',
'distance':'0'}
callback(doc);
}
function save(doc)
{
collection.insert(doc, function(err)
{
if (err)
{
console.log('Error occured');
}
else
console.log("Saved");
});
}
If the condition is to insert 1000 rows it works fine and the error throws only when the condition goes beyond 10000.
Looping over 10000 times and performing insert is really a bad idea. But still you can do with async library which might help you fix the issue. I have came across this situation before and i used async.queue to overcome the issue.
Async.js module.
The problem comes from the recursive loop you made:
function start(i, call) {
if (i < 10000) {
call(start);
}
}
function pass(callback) {
Insert(save);
i++;
callback(i, pass);
}
start(i, pass);
You should change it to something like this:
for (var i = 0; i < 10000; i++) {
Insert(save);
}
Simplifying your code you have this:
var i = 0;
function pass() {
if (i < 10000) {
Insert(save);
pass(i);
}
i++;
}
pass();
The problem comes from the part that you are calling this function recursively, and since javascript doesn't have tail recursion elimination, the callstack keeps growing. V8(nodejs javascript engine) has it's limits, the callstack once reached to the maximum defined size the error will be thrown.
You can also have look at the following questions for more information:
Maximum call stack size exceeded error
JavaScript recursion: Maximum call stack size exceeded
This is all about fixing Maximum call stack size exceeded error. But 10000 looks like a huge number. I just ran that and it took about 3 seconds on my machine, to finish the loop using monk. Using mongo shell it took about 1 second. If you are running a server, when the loop is running your application is unresponsive.
I suggest instead, insert in batches, and use node's setImmediate function to schedule the next batch to be run after pending I/O events(like handling new web requests):
function insert10000(i) {
insert100();
i++;
if (i < 100) {
setImmidiate(insert10000, i);
}
}
function insert100() {
for (var i = 0; i < 100; i++) {
Insert(save);
}
}
And since we came on the topic of batching insert calls, collection.insert method, supports an array of documents instead of just one to be inserted.
So when we currently have something like following:
collection.insert(doc1);
collection.insert(doc2);
It can be changed to this:
collection.insert([doc1, doc2]);
And that actually is faster. So you can change the code to this:
function insert10000(i) {
insert100(i);
i++;
if (i < 100) {
setImmediate(insert10000, i);
}
}
function insert100(i) {
var docs = [];
for (var l = i + 1000; i < l; i++) {
docs.push({
'trip_paramid':i,
'tripid':'116',
'lattitude':'12.8929183',
'longitude':'77.63627',
'speed':'2',
'heading':'0',
'altitude':'80469',
'address':'qwertyasdfgxcvbn',
'engine_status':'Normal',
'oil_pressure': '83.12',
'water_temp': '28',
'fuel_content':'0',
'brake':'Normal',
'creation_time':'2013-08-31 23:22:17',
'brakelight_status':'Normal',
'battery_status':'12.68',
'event_code':'8',
'dbinsert_time':'2013-08-31 23:24:59',
'gsm_status':'-51',
'cell_id':'45',
'vehicle_id':'123456',
'distance':'0'
});
}
collection.insert(docs, function(err) {
if (err) {
console.log('Error occurred', err);
}
});
}
I measured this, it was faster twice faster than the original case.
I am newbie to nodejs.It's very hard to handle callbacks at nodejs level. I have code like this,
getItems(request,function(jsonObject){
var itemData={};
var itemDetails=new Array();
for(var i=0;i < jsonObject.length;i++){
getItemDetails(jsonObject[i].value.item_id,function(jsonObject){
itemDetails.push(jsonObject);
});
}
itemData["itemDetails"]=itemDetails;
response.contentType("application/json");
response.send({"data":itemData});
});
while executing the above code, the for loop is continuing with out getting callback from getItemDetails method and response sent to client. My requirement is the loop will wait until getting the call back from the getItemDetails then response should send.
I have tried with process.nextTick(), but i am unable to find where i have to use that process.nextTick().. Please anybody provide suggestions.
Thanks in advance.
You need to send the response only after you get all the items, so modify your code like so:
getItems(request,function(jsonObject) {
var itemData = {},
itemDetails = [],
itemsLeft = len = jsonObject.length,
i;
function sendResponse(itemDetails) {
itemData["itemDetails"] = itemDetails;
response.contentType("application/json");
response.send({ "data": itemData });
}
for (i = 0; i < len; i++) {
getItemDetails(jsonObject[i].value.item_id, function(jsonObject) {
itemDetails.push(jsonObject);
// send response after all callbacks have been executed
if (!--itemsLeft) {
sendResponse(itemDetails);
}
});
}
});
Note: I've used itemLeft here since it's a more generic way to solve these kind of problems, but Ianzz approach is also ok since you can compare the length of the two arrays.
You can't get the loop to wait, but you can modify your code to get the behavior you are expecting:
getItems(request,function(outerJsonObject){
var itemData={};
var itemDetails=new Array();
for(var i=0;i < outerJsonObject.length;i++){
getItemDetails(jsonObject[i].value.item_id,function(innerJsonObject){
itemDetails.push(innerJsonObject);
if (itemDetails.length == outerJsonObject.length) {
// got all item details
itemData["itemDetails"]=itemDetails;
response.contentType("application/json");
response.send({"data":itemData});
}
});
}
});