~ihabunek/triglav

ref: 72d1f411368cbf732c3e8286a4781c313234fa5f triglav/lib/mix/task/triglav/import_osm.ex -rw-r--r-- 2.3 KiB
72d1f411Ivan Habunek Add route master validation 1 year, 4 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
defmodule Mix.Tasks.Triglav.ImportOsm do
  use Mix.Task

  alias Triglav.Repo

  @shortdoc "Imports OSM data for Croatia from Geofabrik"

  @impl Mix.Task
  def run(_args) do
    {:ok, _} = Application.ensure_all_started(:triglav)

    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq do
      url = "https://download.geofabrik.de/europe/croatia-latest.osm.pbf"
      IO.puts("Downloading #{url}")
      download(url, "croatia-latest.osm.pbf")

      System.cmd("osm2pgsql", [
        "--hstore-all",
        "--create",
        "--slim",
        "--database",
        "triglav_dev",
        "croatia-latest.osm.pbf"
      ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url, target) do
    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
  end
end