~ihabunek/triglav

ref: 0266d54b9e454b6f482eba8ed374c9c905e5857c triglav/lib/triglav/import/osm.ex -rw-r--r-- 3.7 KiB
0266d54bIvan Habunek Improve zet import 1 year, 1 month ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
defmodule Triglav.Import.Osm do
  @moduledoc """
  Imports the latest OSM data for Croatia from Geofabrik

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  alias Triglav.Repo

  def run(force \\ false) do
    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq or force do
      download("https://download.geofabrik.de/europe/croatia-latest.osm.pbf")
      download("https://download.geofabrik.de/europe/croatia-latest.osm.pbf.md5")

      with {_, 0} <- System.cmd("md5sum", ["--check", "croatia-latest.osm.pbf.md5"]) do
        IO.puts("Checksum OK.")
      else
        _ -> raise "Checksum validation failed."
      end

      database =
        :triglav
        |> Application.fetch_env!(Triglav.Repo)
        |> Keyword.fetch!(:url)
        |> URI.parse()
        |> Map.fetch!(:path)
        |> String.trim_leading("/")

      {_, 0} =
        System.cmd("osm2pgsql", [
          "--hstore-all",
          "--create",
          "--slim",
          "--database",
          database,
          "croatia-latest.osm.pbf"
        ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN type text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN ref text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN is_zet boolean")

      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")
      Repo.query("UPDATE planet_osm_rels SET type = tags_hstore->'type'")
      Repo.query("UPDATE planet_osm_rels SET ref = tags_hstore->'ref'")

      Repo.query(
        "UPDATE planet_osm_rels SET is_zet = lower(tags_hstore->'operator') in ('zet', 'zagrebački električni tramvaj')"
      )

      Repo.query("CREATE INDEX idx_planet_osm_rels_is_zet ON planet_osm_rels(is_zet)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_type ON planet_osm_rels(type)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_ref ON planet_osm_rels(ref)")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")
      File.rm("croatia-latest.osm.pbf.md5")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data. Use --force option to import anyway.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url) do
    IO.puts("Downloading #{url}")

    target =
      url
      |> URI.parse()
      |> Map.fetch!(:path)
      |> Path.basename()

    if File.exists?(target) do
      File.rm(target)
    end

    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
  end
end