Need help fixing memory leak for elixir and sweet_xml library - memory-leaks

Question

I'm a newbie in elixir.
I've got the following lib/osm.ex file
defmodule Osm do
import SweetXml
def hello do
:world
end
def main(args) do
args |> parse_args |> process
end
defp parse_args(args) do
{options, _, _} = OptionParser.parse(args, switches: [osm_file: :string, help: :boolean])
options
end
def output_help() do
IO.puts "Usage: osm [OPTION]"
IO.puts ""
IO.puts " --osm-file an osm-file to import"
IO.puts " --help outputs this help-page"
end
def process([]) do
IO.puts "No arguments given"
end
def process(options) do
if options[:help] do
output_help()
else
case options do
[osm_file: _] ->
process_osm_file(options[:osm_file])
end
end
end
def process_osm_file(file) do
counts = %{:nodes => 0, :ways => 0, :relations => 0}
cond do
String.ends_with?(file, ".pbf") ->
IO.puts "parse osm-pbf."
String.ends_with?(file, ".osm.bz2") ->
IO.puts "extract and parse osm-xml."
String.ends_with?(file, ".osm") ->
IO.puts "parse osm-xml."
File.stream!(file)
|> stream_tags([:node, :way, :relation], discard: [:node, :way, :relation])
|> Stream.map(fn
{_, node} ->
process_element(node, counts)
end)
|> Enum.reduce(fn element, result ->
result_modified = %{result |
nodes: result[:nodes] + element[:nodes],
ways: result[:ways] + element[:ways],
relations: result[:relations] + element[:relations]
}
cond do
rem(result_modified[:nodes], 1000) == 0 ->
IO.write "\rnodes: " <> to_string(result_modified[:nodes]) <> "; ways: " <> to_string(result_modified[:ways]) <> "; relations: " <> to_string(result_modified[:relations]) <> "; mem: " <> to_string(:erlang.memory(:total))
true -> true
end
result_modified
end)
|> Stream.run
IO.puts ""
true ->
IO.puts "invalid osm-file extension."
end
end
defp process_element(doc, counts) do
case doc |> xmlElement(:name) do
:node ->
doc |> xmap(
id: ~x"./#id"i,
lat: ~x"./#lat"f,
lon: ~x"./#lon"f,
tags: [
~x"./tag"l,
key: ~x"./#k"s,
value: ~x"./#v"s
]
) |> process_node(counts)
:way ->
doc |> xmap(
id: ~x"./#id"i,
nd: [
~x"./nd"l,
ref: ~x"./#ref"i
],
tags: [
~x"./tag"l,
key: ~x"./#k"s,
value: ~x"./#v"s
]
) |> process_way(counts)
:relation ->
doc |> xmap(
id: ~x"./#id"i,
member: [
~x"./member"l,
type: ~x"./#type"s,
ref: ~x"./#ref"s,
role: ~x"./#role"s
],
tags: [
~x"./tag"l,
key: ~x"./#k"s,
value: ~x"./#v"s
]
) |> process_relation(counts)
_ ->
IO.puts "unhandled element"
end
end
defp process_node(node, counts) do
_ = node
Map.put(counts, :nodes, counts[:nodes] + 1)
end
defp process_way(way, counts) do
_ = way
Map.put(counts, :ways, counts[:ways] + 1)
end
defp process_relation(relation, counts) do
_ = relation
Map.put(counts, :relations, counts[:relations] + 1)
end
end
and the following mix.exs file
defmodule Osm.MixProject do
use Mix.Project
def project do
[
app: :osm,
version: "0.1.0",
elixir: "~> 1.7",
start_permanent: Mix.env() == :prod,
escript: [main_module: Osm],
deps: deps()
]
end
def application do
[
extra_applications: [:logger]
]
end
defp deps do
[
{:sweet_xml, github: 'kbrw/sweet_xml', app: false}
]
end
end
I compile it with mix escript.build
I've downloaded the berlin-latest.osm.bz2 file and extracted the berlin-latest.osm file.
If I call ./osm --osm-file=berlin-latest.osm
The script parses the xml-data and counts the nodes, ways and relations correctly, but the memory consumption is increasing until the end.
Is there a memory leak in the SweetXml library or am I doing something wrong?

Answer 1

I do not see something that would create a memory leak in your code.
I did the following test: I removed progressively all the code using SweetXml, and it is when I withdrew the first part using SweetXml (i.e: stream_tags([:node, :way, :relation], discard: [:node, :way, :relation])) that the memory leak disappeared. This clearly indicates that the memory consumption comes from SweetXml
Reading the source code of SweetXml.stream_tags/3 function, might bring you some answers. I haven't yet understood from where the leak comes from.
Edit: after a thorough inspection of the source code, I still haven't found the origin of the leak. I am starting to thing it is something even deeper, maybe linked to the way erlang VM works.