~ihabunek/triglav

ref: 806eea20fdc339530019214ba443a0d151e85acf triglav/lib/triglav/import/osm.ex -rw-r--r-- 3.7 KiB
806eea20Ivan Habunek Open JOSM links in the background, add indicator 1 year, 1 month ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
defmodule Triglav.Import.Osm do
  @moduledoc """
  Imports the latest OSM data for Croatia from Geofabrik

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  alias Triglav.Repo

  def run(force \\ false) do
    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq or force do
      download("https://download.geofabrik.de/europe/croatia-latest.osm.pbf")
      download("https://download.geofabrik.de/europe/croatia-latest.osm.pbf.md5")

      with {_, 0} <- System.cmd("md5sum", ["--check", "croatia-latest.osm.pbf.md5"]) do
        IO.puts("Checksum OK.")
      else
        _ -> raise "Checksum validation failed."
      end

      database =
        :triglav
        |> Application.fetch_env!(Triglav.Repo)
        |> Keyword.fetch!(:url)
        |> URI.parse()
        |> Map.fetch!(:path)
        |> String.trim_leading("/")

      {_, 0} = System.cmd("osm2pgsql", [
        "--hstore-all",
        "--create",
        "--slim",
        "--database",
        database,
        "croatia-latest.osm.pbf"
      ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN type text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN ref text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN is_zet boolean")

      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")
      Repo.query("UPDATE planet_osm_rels SET type = tags_hstore->'type'")
      Repo.query("UPDATE planet_osm_rels SET ref = tags_hstore->'ref'")
      Repo.query("UPDATE planet_osm_rels SET is_zet = lower(tags_hstore->'operator') in ('zet', 'zagrebački električni tramvaj')")

      Repo.query("CREATE INDEX idx_planet_osm_rels_is_zet ON planet_osm_rels(is_zet)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_type ON planet_osm_rels(type)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_ref ON planet_osm_rels(ref)")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")
      File.rm("croatia-latest.osm.pbf.md5")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data. Use --force option to import anyway.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url) do
    IO.puts("Downloading #{url}")

    target =
      url
      |> URI.parse()
      |> Map.fetch!(:path)
      |> Path.basename()

    if File.exists?(target) do
      File.rm(target)
    end

    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
  end
end