Leaflet.js draw line along rivers between two points - node.js

I am looking to draw a line along a few rivers (they merge so the river name technically changes) with Leaflet.js. I am currently using a Mapbox custom map style to display the map but I'm at a loss of how to "draw" a line along these rivers, from one marker to another.
EDIT
Thanks to #ghybs who pointed me in the right direction (below).
I now have this code which works perfectly for getting the data. However. The problem is that the nodes are not "in order". I'd like the nodes to be in order with regards to the river, so that I can draw the line. Currently, because they are not sequential, the line is all over the place.
The code is utilising Request to get the data, hence the calls are asynchronous. I think this is what's leading to the ordering issue.
var request = require("request");
var parseString = require("xml2js").parseString;
var fs = require("fs");
var results = [];
request("https://www.openstreetmap.org/api/0.6/relation/5806846", function(error, response, body){
// var body = fs.readFileSync("relation.xml");
var total_requests = 0;
var completed_requests = 0;
parseString(body, function(err, result){
var ways = result.osm.relation[0].member;
console.log("Initial requests: " + ways.length);
total_requests += ways.length;
for (var i = 0; i < ways.length; i++) {
var way = ways[i].$.ref;
(function(way, i){
setTimeout(function(){
request("https://www.openstreetmap.org/api/0.6/way/"+way, function(error, response, body){
completed_requests++;
if (error) {
console.log(error);
console.log("https://www.openstreetmap.org/api/0.6/way/" + way + " failed");
}
else {
parseString(body, function(err, result){
var nodes = result.osm.way[0].nd;
console.log("Total requests " + + nodes.length);
total_requests += nodes.length;
for (var i2 = 0; i2 < nodes.length; i2++){
var node = nodes[i2].$.ref;
(function(node, i){
setTimeout(function(){
request("https://www.openstreetmap.org/api/0.6/node/"+node, function(error, response, body){
completed_requests++;
if (error) {
console.log(error);
console.log("https://www.openstreetmap.org/api/0.6/node/" + node + " failed");
}
else {
parseString(body, function(err, result){
var lat = result.osm.node[0].$.lat;
var long = result.osm.node[0].$.lon;
results.push([lat, long]);
});
console.log(total_requests + "/" + completed_requests);
if (completed_requests == total_requests){
console.log("Done");
console.log("Got " + results.length + " results");
fs.writeFile("little_ouse.json", JSON.stringify(results), function(err) {
if (err) {
return console.log(err);
}
console.log("The file was saved");
});
}
}
});
}, i * 1000);
})(node, i2);
}
});
}
});
}, i * 1000);
})(way, i)
}
});
});

It sounds like you would like to extract your rivers paths from the OSM database (which is what Mapbox studio uses to let you customize your basemap style).
On OpenStreetMap main website, you have a big "Export" button at the top. You can use it to extract all the data contained in a given bounding box, including the coordinates of the paths for your rivers.
Then you can use other tools to convert to GeoJSON and keep only the data related to your rivers (e.g. http://geojson.io/).
Once you have your data as GeoJSON, you can easily display it on Leaflet using L.geoJson(myGeoJSONdata).addTo(map)

Related

Node.js + Cheerio : Request inside a loop

I'm using cheerio, request and Node.js.
When I run the script below, it outputs names in a wrong order. I believe that it's caused by asynchronous nature of it, how can I make it work in the "right" order? Do I need to use a sync package or is there a way to change it in a way so it'll work in a sync way?
app.get('/returned', function (req, res) {
for (var y = 0; y < 10; y++) {
var url = "http://example.com" + y + "/person.html";
request(url, function (err, resp, body) {
$ = cheerio.load(body);
var links = $('#container');
var name = links.find('span[itemprop="name"]').html(); // name
if (name == null) {
console.log("returned null");
} else {
console.log(name);
}
});
}
});
Promise makes this relatively easy:
app.get('/returned', function (req, res) {
let urls = [];
for (let y = 0; y < 10; y++) {
urls.push('http://example.com' + y + '/person.html');
}
Promise.all(urls.map(function (url) {
return new Promise(resolve, reject) {
request(url, function (err, resp, body) {
if (err) {return reject(err);}
let $ = cheerio.load(body);
let links = $('#container');
let name = links.find('span[itemprop="name"]').html(); // name
resolve({name: name, links: links, url: url});
});
});
}).then(function (result) {
result.forEach(function (obj) {
if (obj.name == null) {
console.log(obj.url, "returned null");
} else {
console.log(obj.url, obj.name);
}
});
}).catch(function (err) {
console.log(err);
});
});
I started by creating an array of urls to get, then I mapped that to an array of promises. When each of the requests are complete, i resolved the promise with the name, url, and links. When all promises were complete, I then looped over the result which will will be in the original order. This runs in parallel.
Nope, you shouldn't have to use a sync package. IMO the cleanest way is to use a mature 3rd party library.
I'd recommend async.
The async.series method would execute all request functions in the order they are given, then allow you to register a callback to fire when all requests have been made, or when an error has occurred.
https://github.com/caolan/async#seriestasks-callback

Function returning undefined Node JS

I am currently trying to iterate through an array of JSON elements, parse and add the data I need into a specially formatted string, and once conditions are met, initiate the uploading of this data.
The problem that I am running into, however, is my variable 'deviceOutString' is being returned as undefined, leaving me with a string of 'undefined' written as many time as there are JSON elements in the array. I know that the return from the 'checkDuplicates' function is correct because before returning the value, the logs show that the value is correct.
I have attached my code below, please let me know if you have any ideas.
Thanks!
Old Code (updated below)
var i=0;
var parsedJson = JSON.parse(storedData) ;
var storedDataSize = parsedJson.length;
console.log('Stored Data Size: '+storedDataSize);
var async = require('async');
async.each(parsedJson, function( subElemJson, callback1) {
async.series([
function(callback){
console.log('dstring: ' + deviceOutString);
console.log('i : ' + i);
var subElemJsonPretty = JSON.stringify(subElemJson,null,0) ;
var date = subElemJson['date'];
deviceOutString += checkDuplicates(subElemJson, deviceOutString);
console.log('theLoop*DString: ' + deviceOutString);
callback(null, 'one');
},
function(callback){
if((i == storedDataSize - 1 || count == 225) && storedDataSize > 0) {
writeDCC(deviceOutString);
count = 0;
makeList();
}
i++;
callback(null, 'two');
setTimeout(function () { callback1(); }, 500);
}
]);
}); }
Updated New Code
function theLoop(storedData) {
var deviceOutString = '<list>';
var temp;
try {
var i=0;
var parsedJson = JSON.parse(storedData) ;
var storedDataSize = parsedJson.length;
console.log('Stored Data Size: '+storedDataSize);
var async = require('async');
var delayed = require('delayed');
async.each(parsedJson, function( subElemJson, callback1) {
async.series([
function(callback){
var subElemJsonPretty = JSON.stringify(subElemJson,null,0) ;
var date = subElemJson.date;
console.log('THIS IS THE DATE: '+date);
temp = checkDuplicates(subElemJson, deviceOutString);
console.log('This is the temp: ' + temp);
callback(null, temp);
}
], function(results){
console.log('*****Results are In*****: ' + results);
deviceOutString =+ temp;
if((i == storedDataSize - 1 || count == 225) && storedDataSize > 0) {
writeDCC(deviceOutString);
count = 0;
deviceOutString = '<list>';
}
i++;
callback1(results);
});
},
function(err){
if( err ) {
console.log('A file failed to process');
} else {
console.log('All files have been processed successfully');
}
});
} catch (error) {
console.info('Exception parsing ' + '\n\n' + error);
return;
}
}
So a few things
1: var date = subElemJson['date']; accessing object properties via array syntax is a bad practice. Nit picky but hey :P try var data = subElemJson.date; instead.
2: deviceOutString isn't defined anywhere in the code you provided.
3: Both async.series and async.each are going to want a callback function for when each is finished. that's the whole point of calling callback(null, 'one'); -- that you pass a value to the "results" array in the final async.series callback. You are calling setTimeout(function() { callback1(); }, 500); in the wrong place (also arbitrarily putting it behind a timeout?).
The proper async.series formatting is thus:
async.series([
function(callback) {
// do stuff
callback(null, someValue);
},
function(callback) {
// do other stuff
callback(null, someOtherValue);
}
], function(results) {
// all the stuffs are done
console.log(results); <-- results is an array containing "someValue" and "someOtherValue" from the iterations above
callback1(results);
});
Also, async.each is in the same boat -- it expects you to pass a "every thing I'm looping through has completed now!" function at the end.
Async docs on .each() (scroll down for docs on .series()): https://github.com/caolan/async#each

New to node.js and I'm not sure if I'm doing this correct

So, the code works, but the indent level is insane... With all the callbacks in node, how exactly should I be coding?
"use strict";
var crypto = require('crypto'),
fs = require('fs'),
mmm = require('mmmagic'),
Magic = require('mmmagic').Magic,
path = require('path');
console.log('Init controller: ' + path.basename(__filename));
exports.help = function () {
var help;
help = "POST http://server/images\n";
help += " Upload image for storage.\n";
help += " <image> - The image file to upload\n";
help += " <title> - The title of the image, no more than 50 characters\n";
help += " <desc> - The description of the image, no more than 1024 characters\n";
return help;
}
exports.post = function (req, res) {
var image = req.files.image;
if (typeof(image) == 'undefined') {
res.status(400).send("{error:'Upload error'}");
return;
}
var magic = new Magic(mmm.MAGIC_MIME_TYPE);
magic.detectFile(image.path, function(err, result) {
if (err) {
res.status(400).send("{error:'Upload mime error'}");
} else {
var mime = result.toLowerCase().split('/');
if (mime[0] != 'image') {
res.status(400).send("{error:'Upload not an image', mime: '" + result + "'}");
} else {
// Read the image file
fs.readFile(image.path, function (err, data) {
if (err) {
res.status(400).send("{error:'Upload read error'}");
} else {
var hash = crypto.createHash('md5').update(data).digest("hex");
req.app.models.image.count({'hash': hash}, function (err, count) {
if (err) {
res.status(400).send("{error:'ORM Error: '" + JSON.stringify(err) + "'}");
} else {
if (count > 0) {
res.status(400).send("{error:'Image already exists'}");
} else {
var hash = crypto.createHash('md5').update(data).digest("hex");
var newPath = path.join(req.app.tempDir, hash);
fs.writeFile(newPath, data, function (err) {
if (err) {
res.status(400).send("{error:'Upload write error'}");
} else {
// Save the image
req.app.models.image.create([{
'hash' : hash,
'mime' : mime,
title : '',
description : ''
}], function(err, images) {
if (err) {
fs.unlink(newPath);
res.status(400).send("{error:'" + err.message + "'}");
} else {
res.status(200).send("{id:'" + images[0].id + "}");
}
});
}
});
}
}
});
}
});
}
}
});
}
http://callbackhell.com/ has a guide to adjusting to asynchronous programming.
Some follow-ups from the comments:
There are projects such as Node Fibers and Iced CoffeeScript if making asynchronous code look like top-down synchronous code appeals to you and the being misled by that illusion doesn't make you very nervous. However, I strongly recommend working with regular javascript and async programming with callbacks (not promises) until the light bulb goes on before exploring solutions to a "problem" you don't really understand.
Asynchronous code is a bunch of bits of code that execute when the OS is done with I/O. That's why it doesn't read top-down - because it doesn't execute top-down. It executes whenever the IO is done which is why it scales the way it does.
Don't be misled by beginner snippets. Go look at code by the masters like TJ Holowaychuk before deciding that asynchronous code is unreadable. Callback hell is a beginner phenomenon. It is extremely prevalent. Almost everyone goes through it as a phase, but it's not particularly difficult to get beyond, especially given the alternative of mastering multithreaded locking and semaphores and the debugging thereof. Async code that reads well is often just straight-up better-designed code all around. But yes, it lacks the top-down orderliness of synchronous code and that means more jumping around in the source.
<shamelessPlug>
I went through the process of callback spaghetti --> promise spaghetti --> wrote my own promise management library
That library is available here
... and this describes the above process in a bit more detail
</shamelessPlug>
In any case, the general answer is yes - you will eventually wind up with a mess of callbacks. To stop that from happening, you are best served by a callback management system or promises.
Which method is the best? It really is up to you, and what you prefer.
I found that concept of promises helps here a lot. There are many implementations of promises in JavaScript, but idea is same. You can start from this one:
https://github.com/kriszyp/node-promise
Come one guys. Do you really think that you need a library for that. You can handle it with pure javascript. Here is the code rewritten:
var response = null,
request = null;
var fileDetected = function(err, result) {
if (err) {
response.status(400).send("{error:'Upload mime error'}");
} else {
var mime = result.toLowerCase().split('/');
if (mime[0] != 'image') {
response.status(400).send("{error:'Upload not an image', mime: '" + result + "'}");
} else {
// Read the image file
fs.readFile(image.path, onReadFile);
}
}
}
var onReadFile = function(err, data) {
if (err) {
response.status(400).send("{error:'Upload read error'}");
} else {
var hash = crypto.createHash('md5').update(data).digest("hex");
request.app.models.image.count({'hash': hash}, onImageCount);
}
}
var onImageCount = function(err, count) {
if (err) {
response.status(400).send("{error:'ORM Error: '" + JSON.stringify(err) + "'}");
} else {
if (count > 0) {
response.status(400).send("{error:'Image already exists'}");
} else {
var hash = crypto.createHash('md5').update(data).digest("hex");
var newPath = path.join(request.app.tempDir, hash);
fs.writeFile(newPath, data, onFileWrite);
}
}
}
var onFileWrite = function(err) {
if (err) {
response.status(400).send("{error:'Upload write error'}");
} else {
// Save the image
request.app.models.image.create([{
'hash' : hash,
'mime' : mime,
title : '',
description : ''
}], function(err, images) {
if (err) {
fs.unlink(newPath);
response.status(400).send("{error:'" + err.message + "'}");
} else {
response.status(200).send("{id:'" + images[0].id + "}");
}
});
}
}
exports.post = function (req, res) {
request = req;
response = res;
var image = request.files.image;
if (typeof(image) == 'undefined') {
response.status(400).send("{error:'Upload error'}");
return;
}
var magic = new Magic(mmm.MAGIC_MIME_TYPE);
magic.detectFile(image.path, fileDetected);
}
The good part is that by separating everything in different functions you actually split your logic into blocks. The name of the newly created function speaks about the purpose of the block.
If you need to comment that something is "load the image" or "save the image", extracting those into separate functions would help the readability, decrease the indent level, and remove the need for comments.
I.e. instead of writing
// ...
petAHorse(function(horsesResponse) {
// ...
})
// ...
you can write
function horsePettedHandler(horsesResponse) {
// ...
}
// ...
petAHorse(horsePettedHandler);
// ...
Instead of promises I prefer to use node-sync lib. Because it allow one great thing: you don't must wrap async libs functions for promises, you can just use it with special syntax. Like this:
var result = thirdPartyLibAsyncFunction.sync(null, 2, 3);
It work with your own code the same way.

Nodejs CSV data export system for users

I need to allow users to export their data in csv format. I have written app in nodejs. The export data for users can be huge. So i was wondering How to handle such situation in nodejs. Should i user process.nexttick or child process api of nodejs? Also are there any good module available for nodejs to convert data from mysql to csv.
read line by line from your mysql-db, and append line by line to your file
i dont know that much about the mysqlmodule, so i'm assuming here each line is just an array, therefore the 'row.join(';')'. if thats not the case (maybe its an object), you should fix that.
var fs = require('fs');
var connection = require('mysql').createConnection({yourdbsettingshere});
function processRow (row) {
fs.appendFile('your-file.csv', row.join(';'), function (err) {
connection.resume();
});
}
var query = connection.query('SELECT * FROM WHATEVER');
query
.on('error', function(err) {
// do something when an error happens
})
.on('fields', function(fields) {
processRow(fields);
})
.on('result', function(row) {
// Pausing the connnection is useful if your processing involves I/O
connection.pause();
processRow(row, function (err) {
connection.resume();
});
})
.on('end', function() {
// now you can mail your user
});
if you have a lot of requests, you could use the compute-cluster module for distributing your workload
The accepted answer is not working because CSV files are separated by , not ;. Also there is no newline character \n after the end of each row and the fields object contains information about the column attributes not the data rows. results contains the rows resulted from query. Hence I wrote my own code for generating CSV files. If you need more explanation then please comment, I will provide.
pool.query('SELECT * FROM category', function (error, results, fields) {
var reportFile = Date.now();
fs.closeSync(fs.openSync(__dirname + '/../reports/' + reportFile + '.csv', 'w'));
var attributes = [];
var row = [];
for(var x = 0; x<fields.length; x++) attributes.push(fields[x].name);
fs.appendFile(__dirname + '/../reports/' + reportFile + '.csv', attributes.join(','), function (err) {
if(err) console.log('Error appending fields', err);
fs.appendFileSync(__dirname + '/../reports/' + reportFile + '.csv', '\n');
for(var x = 0; x<results.length; x++) {
row = [];
for(var y = 0; y<attributes.length; y++){
row.push(results[x][attributes[y]]);
}
fs.appendFileSync(__dirname + '/../reports/' + reportFile + '.csv', row.join(','));
fs.appendFileSync(__dirname + '/../reports/' + reportFile + '.csv', '\n');
}
req.reportFile = reportFile;
next();
});
});

recoding nested for loop to use async in node.js

I'm new to Node.js and Async coding. I need to write the equivalent of a nested for loop which will work with Node. I understand that my question is very similar to the one posted here: nested loops asynchronusly in nodejs, next loop must start only after one gets completed, but even after looking at that post in detail, I was unable to fix my code.
I am working with an XML feed. The 'parser' uses the xml2js package. The
loop runs exactly as expected if I remove the sql query (for which I'm using the mysql node package), but when I put the sql query in, then all the orders get processed first, the the "DONE" is output, and then the query fails as it tries to look up items for just the last order repeatedly.
I've tried replacing the for loops with async.forEach loops, but this did not help.
Any help or advice on how to recode this in a way more idiomatic to node would be greatly appreciated.
Many thanks!
Sixhobbits
parser.parseString(data, function (err, result) {
if(err) throw(err);
var numOrders = result['Root']['Orders'][0]['Order'].length;
var curr, currItem, currOrdId, items, sellersCode;
console.log("Logging IDs of", numOrders, "orders");
// for each order
for (var j=0; j<numOrders; j++){
//current order
curr = result['Root']['Orders'][0]['Order'][j];
currOrdId = curr['OrderId'][0]
items = curr['Items'][0]['Item'];
console.log("Order ID:", currOrdId, "--",items.length, "Items");
// for each item
for (var k=0; k<items.length; k++){
currItem = items[k];
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,function(err,rows,fields){
if (err) throw(err);
console.log(" Item ID :",currItem['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
});
}//for
}//for
console.log("DONE");
});//parseString
You were on the right track by looking to use async.forEach. Here's how you would rework this code to use that:
parser.parseString(data, function (err, result) {
if(err) throw(err);
var numOrders = result['Root']['Orders'][0]['Order'].length;
var currOrdId, items, sellersCode;
console.log("Logging IDs of", numOrders, "orders");
// for each order
async.forEach(result['Root']['Orders'][0]['Order'], function (curr, callback1) {
currOrdId = curr['OrderId'][0];
items = curr['Items'][0]['Item'];
console.log("Order ID:", currOrdId, "--",items.length, "Items");
async.forEach(items, function (currItem, callback2) {
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,function(err,rows,fields){
console.log(" Item ID :",currItem['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
callback2(err);
});
}, callback1);
}, function (err) {
console.log("DONE");
});
});//parseString
Each iteration of async.forEach must call its callback parameter when all of its async processing has completed. You've got two levels in this case which makes it a little more difficult to keep track of in your head, but it's the same concept.
This is a classic closure-in-a-loop problem. You need to break the closure by passing currItem as an argument:
for (var k=0; k<items.length; k++){
currItem = items[k];
sellersCode = currItem['SellersProductCode'][0];
var sqlQuery = 'select data_index, fulltext_id, cataloginventory_stock_item.product_id from catalogsearch_fulltext inner join cataloginventory_stock_item where catalogsearch_fulltext.data_index like "' + sellersCode + '|%"' + 'and cataloginventory_stock_item.item_id = catalogsearch_fulltext.product_id';
var query = connection.query(sqlQuery,(function(CI){
return function(err,rows,fields){
if (err) throw(err);
console.log(" Item ID :",CI['ItemId'][0]);
console.log(" Full Text ID :", rows[0]['fulltext_id']);
console.log(" Product ID :", rows[0]['product_id']);
}
})(currItem)); // Break closure by passing currItem as argument
}//for
I realize this is an old post, but you might find this function useful
eachKVAsync = function(elements,userInfo,onKeyValue,ondone) {
var onDone = ondone;
var ret = null;
var done=false;
var isArray = typeof elements.forEach===$f$;
var keys = isArray ? null : [],
values = isArray ? elements : [];
if (keys) {
for (var k in elements) {
keys.push(k);
values.push(elements[k]);
}
}
var aborted=false;
var endLoop = function (userInfo){
aborted=true;
if (onDone) {
onDone(userInfo,aborted);
onDone = null;
}
}
var i = 0;
var iterate = function (userInfo) {
if (i < values.length) {
var ix=i;
i++;
onKeyValue((keys?keys[ix]:i),values[ix],userInfo,iterate,endLoop);
} else {
if (onDone) {
onDone(userInfo,aborted);
onDone = null;
return;
}
}
}
iterate(userInfo);
},
use example
eachKVAsync(
elements, {
aValue: 2004
},
function onItem(key, value, info, nextItem, endLoop) {
if (value.isOk) {
info.aValue += value.total;
setTimeout(nextItem,1000,info);
} else {
endLoop(info);
}
},
function afterLastItem(info, loopEndedEarly) {
if (!loopEndedEarly) {
console.log(info.aValue);
}
}
);

Resources