~ihabunek/triglav

ref: 806eea20fdc339530019214ba443a0d151e85acf triglav/lib/triglav/import/zet.ex -rw-r--r-- 3.3 KiB
806eea20Ivan Habunek Open JOSM links in the background, add indicator 1 year, 1 month ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
defmodule Triglav.Import.Zet do
  @moduledoc """
  Imports the latest GTFS data from ZET.

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  alias Triglav.Repo
  alias Triglav.Schemas.Zet.FeedInfo

  def run(force \\ false) do
    url = get_download_url()

    filename =
      url
      |> URI.parse()
      |> Map.get(:path)
      |> Path.basename()
      |> String.replace(" ", "")

    temp_dir = Path.join([System.tmp_dir!(), "triglav"])
    File.mkdir_p!(temp_dir)

    IO.puts("Downloading: #{url}")
    target = Path.join([temp_dir, filename])
    download(url, target)

    IO.puts("Decompressing...")
    {_, 0} = System.cmd("unzip", ["-o", target, "-d", temp_dir])

    [header, row | _] =
      temp_dir
      |> Path.join("feed_info.txt")
      |> File.read!()
      |> String.split("\n")
      |> Enum.map(&String.split(&1, ","))

    version_index = Enum.find_index(header, &(&1 == "feed_version"))
    web_version = Enum.at(row, version_index) |> String.trim("\"")
    local_version = get_local_version()

    IO.puts("Local version: #{local_version}")
    IO.puts("  Web version: #{web_version}")

    if force or is_nil(local_version) or
         String.to_integer(web_version) > String.to_integer(local_version) do
      IO.puts("Updating...")
      File.cd!(temp_dir)
      setup_db_env()
      run_sql("priv/gtfs/schema.sql")
      run_sql("priv/gtfs/load.sql")
      run_sql("priv/gtfs/indices.sql")
    else
      IO.puts("You already have the latest data. Use --force option to import anyway.")
    end
  end

  defp get_local_version() do
    if zet_schema_exists() do
      FeedInfo
      |> Repo.one()
      |> Map.get(:feed_version)
    end
  end

  defp zet_schema_exists() do
    {:ok, %{rows: [[exists?]]}} =
      Repo.query("""
        SELECT EXISTS (
         SELECT FROM information_schema.tables
         WHERE table_schema = 'zet'
         AND table_name = 'feed_info'
      );
      """)

    exists?
  end

  defp run_sql(path) do
    IO.puts("Running: #{path}")
    path = Application.app_dir(:triglav, path)
    {_, 0} = System.cmd("psql", ["-f", path])
  end

  defp get_download_url() do
    html = get("https://www.zet.hr/odredbe/datoteke-u-gtfs-formatu/669")

    Regex.run(~r/https:\/\/www.zet.hr\/UserDocsImages\/[^"]+/, html)
    |> List.first()
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url, target) do
    # Encode whitespace in path
    url =
      url
      |> URI.parse()
      |> Map.update!(:path, &URI.encode(&1))
      |> URI.to_string()

    if File.exists?(target) do
      File.rm(target)
    end

    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))

    IO.puts("Saved to: #{target}")
  end

  defp setup_db_env() do
    Application.fetch_env!(:triglav, Triglav.Repo)
    |> Keyword.get(:url)
    |> Ecto.Repo.Supervisor.parse_url()
    |> Enum.each(fn
      {:hostname, hostname} -> System.put_env("PGHOST", hostname)
      {:database, database} -> System.put_env("PGDATABASE", database)
      {:username, username} -> System.put_env("PGUSER", username)
      {:password, password} -> System.put_env("PGPASSWORD", password)
      {:port, port} -> System.put_env("PGPORT", to_string(port))
    end)
  end
end