~ihabunek/triglav

2743a4bdcc554260a932e0c210d5d9b50ec23a27 — Ivan Habunek 1 year, 20 days ago 8c019f6
Make osm import usable from release
4 files changed, 111 insertions(+), 94 deletions(-)

M lib/mix/tasks/triglav/import_osm.ex
A lib/triglav/import/osm.ex
M lib/triglav/release.ex
M mix.exs
M lib/mix/tasks/triglav/import_osm.ex => lib/mix/tasks/triglav/import_osm.ex +2 -93
@@ 1,106 1,15 @@
defmodule Mix.Tasks.Triglav.ImportOsm do
  use Mix.Task

  alias Triglav.Repo

  @shortdoc "Imports the latest OSM data for Croatia from Geofabrik"

  @moduledoc """
  Imports the latest OSM data for Croatia from Geofabrik

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  @impl Mix.Task
  def run(args) do
    Application.put_env(:triglav, :minimal, true)
    {:ok, _} = Application.ensure_all_started(:triglav)

    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    {opts, _rest} = OptionParser.parse!(args, strict: [force: :boolean])

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq or opts[:force] do
      url = "https://download.geofabrik.de/europe/croatia-latest.osm.pbf"
      IO.puts("Downloading #{url}")
      download(url, "croatia-latest.osm.pbf")

      System.cmd("osm2pgsql", [
        "--hstore-all",
        "--create",
        "--slim",
        "--database",
        "triglav_dev",
        "croatia-latest.osm.pbf"
      ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN type text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN ref text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN is_zet boolean")

      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")
      Repo.query("UPDATE planet_osm_rels SET type = tags_hstore->'type'")
      Repo.query("UPDATE planet_osm_rels SET ref = tags_hstore->'ref'")
      Repo.query("UPDATE planet_osm_rels SET is_zet = lower(tags_hstore->'operator') in ('zet', 'zagrebački električni tramvaj')")

      Repo.query("CREATE INDEX idx_planet_osm_rels_is_zet ON planet_osm_rels(is_zet)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_type ON planet_osm_rels(type)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_ref ON planet_osm_rels(ref)")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data. Use --force option to import anyway.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url, target) do
    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
    Triglav.Import.Osm.run(opts[:force])
  end
end

A lib/triglav/import/osm.ex => lib/triglav/import/osm.ex +97 -0
@@ 0,0 1,97 @@
defmodule Triglav.Import.Osm do
  @moduledoc """
  Imports the latest OSM data for Croatia from Geofabrik

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  alias Triglav.Repo

  def run(force \\ false) do
    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq or force do
      url = "https://download.geofabrik.de/europe/croatia-latest.osm.pbf"
      IO.puts("Downloading #{url}")
      download(url, "croatia-latest.osm.pbf")

      System.cmd("osm2pgsql", [
        "--hstore-all",
        "--create",
        "--slim",
        "--database",
        "triglav_dev",
        "croatia-latest.osm.pbf"
      ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN type text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN ref text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN is_zet boolean")

      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")
      Repo.query("UPDATE planet_osm_rels SET type = tags_hstore->'type'")
      Repo.query("UPDATE planet_osm_rels SET ref = tags_hstore->'ref'")
      Repo.query("UPDATE planet_osm_rels SET is_zet = lower(tags_hstore->'operator') in ('zet', 'zagrebački električni tramvaj')")

      Repo.query("CREATE INDEX idx_planet_osm_rels_is_zet ON planet_osm_rels(is_zet)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_type ON planet_osm_rels(type)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_ref ON planet_osm_rels(ref)")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data. Use --force option to import anyway.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url, target) do
    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
  end
end

M lib/triglav/release.ex => lib/triglav/release.ex +11 -0
@@ 14,6 14,11 @@ defmodule Triglav.Release do
    {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
  end

  def import_osm(force \\ false) do
    start_app()
    Triglav.Import.Osm.run(force)
  end

  defp repos do
    Application.fetch_env!(@app, :ecto_repos)
  end


@@ 21,4 26,10 @@ defmodule Triglav.Release do
  defp load_app do
    Application.load(@app)
  end

  defp start_app do
    load_app()
    Application.put_env(@app, :minimal, true)
    Application.ensure_all_started(@app)
  end
end

M mix.exs => mix.exs +1 -1
@@ 20,7 20,7 @@ defmodule Triglav.MixProject do
  def application do
    [
      mod: {Triglav.Application, []},
      extra_applications: [:logger, :runtime_tools]
      extra_applications: [:logger, :runtime_tools, :inets]
    ]
  end