~ihabunek/triglav

ref: 726038f028b29f3b9c04f00c4b08a1588f12873a triglav/lib/mix/tasks/triglav/import_osm.ex -rw-r--r-- 3.3 KiB
726038f0Ivan Habunek Add --force option to osm import task 1 year, 3 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
defmodule Mix.Tasks.Triglav.ImportOsm do
  use Mix.Task

  alias Triglav.Repo

  @shortdoc "Imports the latest OSM data for Croatia from Geofabrik"

  @moduledoc """
  Imports the latest OSM data for Croatia from Geofabrik

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  @impl Mix.Task
  def run(args) do
    {:ok, _} = Application.ensure_all_started(:triglav)

    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    {opts, _rest} = OptionParser.parse!(args, strict: [force: :boolean])

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq or opts[:force] do
      url = "https://download.geofabrik.de/europe/croatia-latest.osm.pbf"
      IO.puts("Downloading #{url}")
      download(url, "croatia-latest.osm.pbf")

      System.cmd("osm2pgsql", [
        "--hstore-all",
        "--create",
        "--slim",
        "--database",
        "triglav_dev",
        "croatia-latest.osm.pbf"
      ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN type text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN ref text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN is_zet boolean")

      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")
      Repo.query("UPDATE planet_osm_rels SET type = tags_hstore->'type'")
      Repo.query("UPDATE planet_osm_rels SET ref = tags_hstore->'ref'")
      Repo.query("UPDATE planet_osm_rels SET is_zet = lower(tags_hstore->'operator') in ('zet', 'zagrebački električni tramvaj')")

      Repo.query("CREATE INDEX idx_planet_osm_rels_is_zet ON planet_osm_rels(is_zet)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_type ON planet_osm_rels(type)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_ref ON planet_osm_rels(ref)")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data. Use --force option to import anyway.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url, target) do
    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
  end
end