I'm trying to loop & sort a large number of data ( the whole ethereum blockchain lol )
I'm trying to create a record of all transactions for every address.
Obviously this is a very intensive process and I'm not sure how to make it more efficient beyond what I have (which isn't that efficient)
It starts out quick but I'm thinking now it has slowed because of the lookup for the address in the txs object.
Any help opinions / help is greatly appreciated.
https://giphy.com/gifs/3o6fJ7KWqxESY9okk8
var txs = {};
var i = 0;
// Loop over blocks
(function loop () {
setTimeout(function () {
// Get current block
var block = web3.eth.getBlock(i, true, (error, block) => {
// debugger;
// Loop over transactions in block
for(var j = 0; j < block.transactions.length; j++) {
// debugger;
if(txs[block.transactions[j].to]) {
txs[block.transactions[j].to].transactions.push(block.transactions[j]);
} else if (txs[block.transactions[j].to]) {
txs[block.transactions[j].from].transactions.push(block.transactions[j]);
} else {
txs[block.transactions[j].to] = {
transactions: [block.transactions[j]]
}
txs[block.transactions[j].from] = {
transactions: [block.transactions[j]]
}
}
}
});
i++
if (i < highestBlock) {
loop();
}
}, 50);
})();
I think that your code has an error in it located at "else-if", it seems that you should use the txs[block.transactions[j].from] property instead of the txs[block.transactions[j].to]. If you simply want to accomplish a recursive pattern you could use the setImmediate function or the process.nextTick method. If you use node.js v6+ you could use a Map instead of the object.
Related
Please find below code
function get_btc(address) {
address_transaction(address, user_id, coin_key, deposite_txn_fee, function(callback) {
for (var j = 0; j < callback.response.data.txs.length; j++) {
let user_id = callback.user_id;
//some code//
}
});
}
get_label_info(function(err, data) {
for (var i = 0; i < data.length; i++) {
let address = data[i].address;
deposite_model.get_coin_info(function(err, data1) {
var coin_name = data1[0].coin_code;
const return_functions = get_switch(coin_name);
if (return_functions) {
obj[return_functions](address);
}
})
}
});
function all_completed() {
console.log('all functions has been completed');
}
By the help of above mentioned code i want to excecute all_completed loop when all functions has been completly done.
At the initial start get_label_info function is excuted then controller go on to get_btc function.
Please help me how could i run all_completed functions after all functions completed run.
I'll assume you are using es6, and that you know what a Promise is in that context. In that case wrap all your callback based things in a Promise that resolves when the callback completes. Then, in your loop, push all your Promises into an array variable. Finally call Promise.all with that array as an argument and call then on the result to encapsulate the code you want ti run after they all complete (resolve).
So in Node.js let's say I have the following code:
for (var i = 0; i < 1000; i++) {
someAsynchronousFunction(function(err,res) {
// Do a bunch of stuff
callback(null,res);
});
}
But I want this to run synchronously. I know this is not advised in Node JS, but I am just trying to understand the language. I tried implementing the following solution, but it just ends up hanging during runtime:
for (var i = 0; i < 1000; i++) {
var going = true;
someAsynchronousFunction(function(err,res) {
// Do a bunch of stuff
callback(null,res);
going = false;
});
while (going) {
}
}
What is going wrong and what is the correct way to do this?
One of the best way to do that is to use the async library.
async.timesSeries(1000, function(n, next){
someAsynchronousFunction(next);
});
Or you can do that with async.series() function.
.times() documentation : http://caolan.github.io/async/docs.html#.timesSeries
Another way to do this is using Promises to generate a sequential execution of them thanks to Array#reduce :
// Function that returns Promise that is fllfiled after a second.
function asyncFunc (x){
return new Promise((rs, rj)=>{
setTimeout( ()=>{
console.log('Hello ', x);
rs();
}, 1000)
});
}
// Generate an array filed with values : [0, 1, 2, 3, ...]
Array.from({length : 1000}, (el, i)=> i)
// loop througth the array chaining the promises.
.reduce( (promise, value) =>
promise.then(asyncFunc.bind(null, value))
, Promise.resolve(null));
Or quite possibly I am doing it wrong, in fact, more than likely I am doing it wrong.
Have a table which contains a "tree" of skill, starting at the root level and may be as deep as ten levels (only two so far), but I want to return it as one big fat JSON structure, so I want to ask the database for each set of data, build my structure then ask for the next level.
Of course if I just send of my requests using mongoose, they will come back at any time, as they are all nice asyncronous calls. Normally a good things.
Looking at the documentation for Mongoose(using 4.1.1) it seems like it has a promise built in, but whenever I try to use it the api call throws a hissy fit and I get a 500 back.
Here is my simple function:
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
for (var i = 0; i<10; i++){
var returnData = [];
console.log("Lets get level " + i );
var query = Skill.find({level: i });//The query function
var promise = query.exec; //The promise?
promise.then(function(doc) { //Totally blows up at this point
console.log("Something came back")
return "OK";
});
}
}
The Mongoose documentation on the subject can be found here
http://mongoosejs.com/docs/api.html#promise_Promise
var promise = query.exec;
// =>
var promise = query.exec()
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
var p;
for (var i = 0; i < 10; i ++) {
if (i == 0 ) {
p = Skill.find({level:i}).exec();
} else {
p.then(function (){
return Skill.find({level:i}).exec()
})
}
p.then(function (data) {
//deal with your data
})
}
p.then(function () {
// deal with response
})
}
This is the code am running which returns the Range Maximum call stack size exceeded error.
// to insert 10000 values on to mongodb using node.js
var MongoClient = require('mongodb').MongoClient;
var mongoServer = require('mongodb').Server;
var serverOptions = {
'auto_reconnect': true,
'poolSize': 100
};
var i=0;
var async =require('async');
var mongoClient = new MongoClient(new mongoServer('localhost', 27017, serverOptions));
var db = mongoClient.db('test');
var collection = db.collection('new_file_test');
mongoClient.open(function (err, mongoClient)
{
if(err){console.log(err)};
function start(i,call)
{
if(i<10000) {
call(start);
}
}
function pass(callback)
{
Insert(save);
i++;
callback(i,pass);
}
start(i,pass);
});
function Insert(callback) {
console.log("Inserting" );
var doc={
'trip_paramid':i,
'tripid':'116',
'lattitude':'12.8929183',
'longitude':'77.63627',
'speed':'2',
'heading':'0',
'altitude':'80469',
'address':'qwertyasdfgxcvbn',
'engine_status':'Normal',
'oil_pressure': '83.12',
'water_temp': '28',
'fuel_content':'0',
'brake':'Normal',
'creation_time':'2013-08-31 23:22:17',
'brakelight_status':'Normal',
'battery_status':'12.68',
'event_code':'8',
'dbinsert_time':'2013-08-31 23:24:59',
'gsm_status':'-51',
'cell_id':'45',
'vehicle_id':'123456',
'distance':'0'}
callback(doc);
}
function save(doc)
{
collection.insert(doc, function(err)
{
if (err)
{
console.log('Error occured');
}
else
console.log("Saved");
});
}
If the condition is to insert 1000 rows it works fine and the error throws only when the condition goes beyond 10000.
Looping over 10000 times and performing insert is really a bad idea. But still you can do with async library which might help you fix the issue. I have came across this situation before and i used async.queue to overcome the issue.
Async.js module.
The problem comes from the recursive loop you made:
function start(i, call) {
if (i < 10000) {
call(start);
}
}
function pass(callback) {
Insert(save);
i++;
callback(i, pass);
}
start(i, pass);
You should change it to something like this:
for (var i = 0; i < 10000; i++) {
Insert(save);
}
Simplifying your code you have this:
var i = 0;
function pass() {
if (i < 10000) {
Insert(save);
pass(i);
}
i++;
}
pass();
The problem comes from the part that you are calling this function recursively, and since javascript doesn't have tail recursion elimination, the callstack keeps growing. V8(nodejs javascript engine) has it's limits, the callstack once reached to the maximum defined size the error will be thrown.
You can also have look at the following questions for more information:
Maximum call stack size exceeded error
JavaScript recursion: Maximum call stack size exceeded
This is all about fixing Maximum call stack size exceeded error. But 10000 looks like a huge number. I just ran that and it took about 3 seconds on my machine, to finish the loop using monk. Using mongo shell it took about 1 second. If you are running a server, when the loop is running your application is unresponsive.
I suggest instead, insert in batches, and use node's setImmediate function to schedule the next batch to be run after pending I/O events(like handling new web requests):
function insert10000(i) {
insert100();
i++;
if (i < 100) {
setImmidiate(insert10000, i);
}
}
function insert100() {
for (var i = 0; i < 100; i++) {
Insert(save);
}
}
And since we came on the topic of batching insert calls, collection.insert method, supports an array of documents instead of just one to be inserted.
So when we currently have something like following:
collection.insert(doc1);
collection.insert(doc2);
It can be changed to this:
collection.insert([doc1, doc2]);
And that actually is faster. So you can change the code to this:
function insert10000(i) {
insert100(i);
i++;
if (i < 100) {
setImmediate(insert10000, i);
}
}
function insert100(i) {
var docs = [];
for (var l = i + 1000; i < l; i++) {
docs.push({
'trip_paramid':i,
'tripid':'116',
'lattitude':'12.8929183',
'longitude':'77.63627',
'speed':'2',
'heading':'0',
'altitude':'80469',
'address':'qwertyasdfgxcvbn',
'engine_status':'Normal',
'oil_pressure': '83.12',
'water_temp': '28',
'fuel_content':'0',
'brake':'Normal',
'creation_time':'2013-08-31 23:22:17',
'brakelight_status':'Normal',
'battery_status':'12.68',
'event_code':'8',
'dbinsert_time':'2013-08-31 23:24:59',
'gsm_status':'-51',
'cell_id':'45',
'vehicle_id':'123456',
'distance':'0'
});
}
collection.insert(docs, function(err) {
if (err) {
console.log('Error occurred', err);
}
});
}
I measured this, it was faster twice faster than the original case.
I am newbie to nodejs.It's very hard to handle callbacks at nodejs level. I have code like this,
getItems(request,function(jsonObject){
var itemData={};
var itemDetails=new Array();
for(var i=0;i < jsonObject.length;i++){
getItemDetails(jsonObject[i].value.item_id,function(jsonObject){
itemDetails.push(jsonObject);
});
}
itemData["itemDetails"]=itemDetails;
response.contentType("application/json");
response.send({"data":itemData});
});
while executing the above code, the for loop is continuing with out getting callback from getItemDetails method and response sent to client. My requirement is the loop will wait until getting the call back from the getItemDetails then response should send.
I have tried with process.nextTick(), but i am unable to find where i have to use that process.nextTick().. Please anybody provide suggestions.
Thanks in advance.
You need to send the response only after you get all the items, so modify your code like so:
getItems(request,function(jsonObject) {
var itemData = {},
itemDetails = [],
itemsLeft = len = jsonObject.length,
i;
function sendResponse(itemDetails) {
itemData["itemDetails"] = itemDetails;
response.contentType("application/json");
response.send({ "data": itemData });
}
for (i = 0; i < len; i++) {
getItemDetails(jsonObject[i].value.item_id, function(jsonObject) {
itemDetails.push(jsonObject);
// send response after all callbacks have been executed
if (!--itemsLeft) {
sendResponse(itemDetails);
}
});
}
});
Note: I've used itemLeft here since it's a more generic way to solve these kind of problems, but Ianzz approach is also ok since you can compare the length of the two arrays.
You can't get the loop to wait, but you can modify your code to get the behavior you are expecting:
getItems(request,function(outerJsonObject){
var itemData={};
var itemDetails=new Array();
for(var i=0;i < outerJsonObject.length;i++){
getItemDetails(jsonObject[i].value.item_id,function(innerJsonObject){
itemDetails.push(innerJsonObject);
if (itemDetails.length == outerJsonObject.length) {
// got all item details
itemData["itemDetails"]=itemDetails;
response.contentType("application/json");
response.send({"data":itemData});
}
});
}
});