~ihabunek/triglav

4863efffb34f071435fc1bb637ffb581d394dad9 — Ivan Habunek 8 months ago 700b548
Remove legacy code for processing osm2pgsql data
13 files changed, 14 insertions(+), 374 deletions(-)

D lib/mix/tasks/triglav/import_osm.ex
D lib/triglav/data_import.ex
D lib/triglav/import/osm.ex
M lib/triglav/release.ex
D lib/triglav/schemas/imported_data.ex
D lib/triglav/schemas/osm/point.ex
D lib/triglav/schemas/osm/relation.ex
D lib/triglav/schemas/osm/way.ex
M lib/triglav/zet/errors.ex
D lib/triglav/zet/osm.ex
M lib/triglav_web/controllers/zet/routes_controller.ex
M lib/triglav_web/templates/zet/routes/index.html.eex
A priv/repo/migrations/20210207162740_drop_imported_data.exs
D lib/mix/tasks/triglav/import_osm.ex => lib/mix/tasks/triglav/import_osm.ex +0 -14
@@ 1,14 0,0 @@
defmodule Mix.Tasks.Triglav.ImportOsm do
  use Mix.Task

  @shortdoc "Imports the latest OSM data for Croatia from Geofabrik"

  @impl Mix.Task
  def run(args) do
    Application.put_env(:triglav, :repo_only, true)
    {:ok, _} = Application.ensure_all_started(:triglav)

    {opts, _rest} = OptionParser.parse!(args, strict: [force: :boolean])
    Triglav.Import.Osm.run(opts[:force])
  end
end

D lib/triglav/data_import.ex => lib/triglav/data_import.ex +0 -17
@@ 1,17 0,0 @@
defmodule Triglav.DataImport do
  alias Triglav.Repo
  alias Triglav.Schemas.ImportedData

  def load_state() do
    case Repo.one(ImportedData) do
      nil -> save_state()
      %ImportedData{} = data -> data
    end
  end

  def save_state(attrs \\ %{}) do
    %ImportedData{id: 1}
    |> ImportedData.changeset(attrs)
    |> Repo.insert!(on_conflict: :replace_all, conflict_target: [:id])
  end
end
\ No newline at end of file

D lib/triglav/import/osm.ex => lib/triglav/import/osm.ex +0 -127
@@ 1,127 0,0 @@
defmodule Triglav.Import.Osm do
  @moduledoc """
  Imports the latest OSM data for Croatia from Geofabrik

  See:
  https://download.geofabrik.de/europe/croatia.html
  """

  alias Triglav.Repo

  def run(force \\ false) do
    web_state = get_web_state()
    db_state = Triglav.DataImport.load_state()

    %{osm_sequence_number: db_seq, osm_timestamp: db_ts} = db_state
    %{osm_sequence_number: web_seq, osm_timestamp: web_ts} = web_state

    if db_seq do
      IO.puts(" Local data: seq ##{db_seq} from #{db_ts}")
    else
      IO.puts(" Local data: none")
    end

    IO.puts("Remote data: seq ##{web_seq} from #{web_ts}\n")

    if is_nil(db_seq) or web_seq > db_seq or force do
      download("https://download.geofabrik.de/europe/croatia-latest.osm.pbf")
      download("https://download.geofabrik.de/europe/croatia-latest.osm.pbf.md5")

      with {_, 0} <- System.cmd("md5sum", ["--check", "croatia-latest.osm.pbf.md5"]) do
        IO.puts("Checksum OK.")
      else
        _ -> raise "Checksum validation failed."
      end

      database =
        :triglav
        |> Application.fetch_env!(Triglav.Repo)
        |> Keyword.fetch!(:url)
        |> URI.parse()
        |> Map.fetch!(:path)
        |> String.trim_leading("/")

      {_, 0} =
        System.cmd("osm2pgsql", [
          "--hstore-all",
          "--create",
          "--slim",
          "--database",
          database,
          "croatia-latest.osm.pbf"
        ])

      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN tags_hstore hstore")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN type text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN ref text")
      Repo.query("ALTER TABLE planet_osm_rels ADD COLUMN is_zet boolean")

      Repo.query("UPDATE planet_osm_rels SET tags_hstore = tags::hstore")
      Repo.query("UPDATE planet_osm_rels SET type = tags_hstore->'type'")
      Repo.query("UPDATE planet_osm_rels SET ref = tags_hstore->'ref'")

      Repo.query(
        "UPDATE planet_osm_rels SET is_zet = lower(tags_hstore->'operator') in ('zet', 'zagrebački električni tramvaj')"
      )

      Repo.query("CREATE INDEX idx_planet_osm_rels_is_zet ON planet_osm_rels(is_zet)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_type ON planet_osm_rels(type)")
      Repo.query("CREATE INDEX idx_planet_osm_rels_ref ON planet_osm_rels(ref)")

      IO.puts("Saving state...")
      Triglav.DataImport.save_state(web_state)

      IO.puts("Deleting PBF archive...")
      File.rm("croatia-latest.osm.pbf")
      File.rm("croatia-latest.osm.pbf.md5")

      IO.puts("Done.")
    else
      IO.puts("You already have the latest OSM data. Use --force option to import anyway.")
    end
  end

  defp get_web_state() do
    state =
      get("http://download.geofabrik.de/europe/croatia-updates/state.txt")
      |> String.split(~r"\n")
      |> Enum.reject(&String.starts_with?(&1, "#"))
      |> Enum.reject(&(&1 == ""))
      |> Enum.map(&String.split(&1, "="))
      |> Enum.map(&List.to_tuple/1)
      |> Enum.into(%{}, fn {k, v} -> {k, v} end)

    sequence_number = String.to_integer(state["sequenceNumber"])

    {:ok, timestamp, 0} =
      state["timestamp"]
      |> String.replace(~r"\\", "")
      |> DateTime.from_iso8601()

    %{osm_sequence_number: sequence_number, osm_timestamp: timestamp}
  end

  defp get(url) do
    {:ok, {{'HTTP/1.1', 200, 'OK'}, _headers, body}} =
      :httpc.request(:get, {to_charlist(url), []}, [], [])

    to_string(body)
  end

  defp download(url) do
    IO.puts("Downloading #{url}")

    target =
      url
      |> URI.parse()
      |> Map.fetch!(:path)
      |> Path.basename()

    if File.exists?(target) do
      File.rm(target)
    end

    {:ok, :saved_to_file} =
      :httpc.request(:get, {to_charlist(url), []}, [], stream: to_charlist(target))
  end
end

M lib/triglav/release.ex => lib/triglav/release.ex +0 -5
@@ 14,11 14,6 @@ defmodule Triglav.Release do
    {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version))
  end

  def import_osm(force \\ false) do
    start_repo()
    Triglav.Import.Osm.run(force)
  end

  def import_zet(opts \\ []) do
    start_repo()
    Triglav.Import.Zet.run(opts)

D lib/triglav/schemas/imported_data.ex => lib/triglav/schemas/imported_data.ex +0 -14
@@ 1,14 0,0 @@
defmodule Triglav.Schemas.ImportedData do
  use Ecto.Schema
  import Ecto.Changeset

  schema "imported_data" do
    field :osm_sequence_number, :integer
    field :osm_timestamp, :utc_datetime
  end

  def changeset(row, params \\ %{}) do
    row
    |> cast(params, [:osm_sequence_number, :osm_timestamp])
  end
end

D lib/triglav/schemas/osm/point.ex => lib/triglav/schemas/osm/point.ex +0 -19
@@ 1,19 0,0 @@
defmodule Triglav.Schemas.Osm.Point do
  use Ecto.Schema

  @derive {Inspect, only: [:id]}
  @primary_key false

  schema "planet_osm_point" do
    field :id, :integer, primary_key: true, source: :osm_id
    field :name, :string
    field :tags, :map
    field :way, Geo.PostGIS.Geometry

    field :way_4326, Geo.PostGIS.Geometry, virtual: true
  end

  def get_tag(point, name), do: Map.get(point.tags, name)
  def has_tag?(point, name), do: Map.has_key?(point.tags, name)
  def has_tags?(point, names), do: Enum.all?(names, &has_tag?(point, &1))
end

D lib/triglav/schemas/osm/relation.ex => lib/triglav/schemas/osm/relation.ex +0 -31
@@ 1,31 0,0 @@
defmodule Triglav.Schemas.Osm.Relation do
  use Ecto.Schema

  @derive {Inspect, only: [:id, :tags]}
  @primary_key false

  schema "planet_osm_rels" do
    field :id, :integer, primary_key: true
    field :way_off, :integer
    field :rel_off, :integer
    field :parts, {:array, :integer}
    field :members, {:array, :string}
    field :tags_array, {:array, :string}, source: :tags

    # Added columns
    field :tags, :map, source: :tags_hstore
    field :type, :string
    field :ref, :string
    field :is_zet, :boolean
  end

  def get_tag(relation, name), do: Map.get(relation.tags, name)
  def has_tag?(relation, name), do: Map.has_key?(relation.tags, name)
  def has_tags?(relation, names), do: Enum.all?(names, &has_tag?(relation, &1))
  def type(relation), do: get_tag(relation, "type")
  def ref(relation), do: get_tag(relation, "ref")
  def is_route(relation), do: type(relation) == "route"
  def is_route_master(relation), do: type(relation) == "route_master"
  def is_roundtrip(relation), do: get_tag(relation, "roundtrip") == "yes"
  def is_ptv2(relation), do: get_tag(relation, "public_transport:version") == "2"
end

D lib/triglav/schemas/osm/way.ex => lib/triglav/schemas/osm/way.ex +0 -47
@@ 1,47 0,0 @@
defmodule Triglav.Schemas.Osm.Way do
  use Ecto.Schema
  alias Triglav.Schemas.Osm.Way

  @derive {Inspect, only: [:id]}
  @primary_key false
  @type t :: %__MODULE__{}

  schema "planet_osm_ways" do
    field :id, :integer, primary_key: true
    field :nodes, {:array, :integer}
    field :tags, {:array, :string}
  end

  @spec adjacent?(t, t) :: boolean()
  def adjacent?(%Way{} = one, %Way{} = other) do
    cond do
      roundabout?(one) ->
        roundabout_adjacent?(one, other)

      roundabout?(other) ->
        roundabout_adjacent?(other, one)

      true ->
        linear_adjacent?(one, other)
    end
  end

  defp roundabout_adjacent?(roundabout, way) do
    List.first(way.nodes) in roundabout.nodes or
      List.last(way.nodes) in roundabout.nodes
  end

  defp linear_adjacent?(one, other) do
    List.first(one.nodes) == List.first(other.nodes) or
      List.first(one.nodes) == List.last(other.nodes) or
      List.last(one.nodes) == List.first(other.nodes) or
      List.last(one.nodes) == List.last(other.nodes)
  end

  @spec roundabout?(t()) :: boolean()
  def roundabout?(%Way{} = way) do
    way.tags
    |> Enum.chunk_every(2)
    |> Enum.any?(&(&1 == ["junction", "roundabout"]))
  end
end

M lib/triglav/zet/errors.ex => lib/triglav/zet/errors.ex +0 -3
@@ 1,8 1,5 @@
defmodule Triglav.Zet.Errors do
  alias Triglav.Schemas.Error
  alias Triglav.Repo

  import Ecto.Query

  def render(%Error{key: "missing_route_master"}) do
    "Missing route_master relation"

D lib/triglav/zet/osm.ex => lib/triglav/zet/osm.ex +0 -90
@@ 1,90 0,0 @@
defmodule Triglav.Zet.Osm do
  alias Triglav.Repo
  alias Triglav.Schemas.Osm.Point
  alias Triglav.Schemas.Osm.Relation
  alias Triglav.Schemas.Osm.Way

  import Ecto.Query
  import Geo.PostGIS

  def list_public_transport_relations(opts \\ []) do
    pt_relations()
    |> filter_by_ref(opts)
    |> Repo.all()
    |> Enum.map(&process_members/1)
  end

  @doc """
  For a given list of relations returns all included members of the :way variety
  which do not have a role (to skip stations entered as ways). Returns them as
  a map indexed by way ID for easier lookup.
  """
  @spec list_ways([Relation.t()]) :: Way.t()
  def list_ways(relations) do
    ids =
      relations
      |> Enum.map(fn r ->
        r.members
        |> Enum.filter(&(&1.type == :way and &1.role == ""))
        |> Enum.map(& &1.id)
      end)
      |> List.flatten()

    from(w in Way, where: w.id in ^ids)
    |> Repo.all()
  end

  @doc """
  Returns a list of all nodes with platform* roles which are members of any of
  the given relations.
  """
  @spec list_platform_nodes([Relation.t()]) :: [Point.t()]
  def list_platform_nodes(relations) do
    ids = platform_node_ids(relations)

    from(p in Point,
      select: %{p | way_4326: st_transform(p.way, 4326)},
      where: p.id in ^ids,
      order_by: [asc: :name]
    )
    |> Repo.all()
  end

  defp platform_node_ids(relations) do
    relations
    |> Enum.map(fn r ->
      r.members
      |> Enum.filter(&(&1.type == :node and String.starts_with?(&1.role, "platform")))
      |> Enum.map(& &1.id)
    end)
    |> List.flatten()
  end

  # Private

  defp pt_relations() do
    from(r in Relation,
      where: r.type in ["route", "route_master"] and r.is_zet
    )
  end

  defp filter_by_ref(query, ref: ref), do: where(query, [r], r.ref == ^ref)
  defp filter_by_ref(query, _), do: query

  # Consider moving this to the database
  defp process_members(rel) do
    Map.update!(rel, :members, fn members ->
      members
      |> Enum.chunk_every(2)
      |> Enum.map(&List.to_tuple/1)
      |> Enum.map(fn {element, role} ->
        {type, id} = element_type(element)
        %{type: type, id: id, role: role}
      end)
    end)
  end

  defp element_type("n" <> id), do: {:node, String.to_integer(id)}
  defp element_type("r" <> id), do: {:relation, String.to_integer(id)}
  defp element_type("w" <> id), do: {:way, String.to_integer(id)}
end

M lib/triglav_web/controllers/zet/routes_controller.ex => lib/triglav_web/controllers/zet/routes_controller.ex +3 -3
@@ 1,7 1,7 @@
defmodule TriglavWeb.Zet.RoutesController do
  use TriglavWeb, :controller

  alias Triglav.DataImport
  alias Triglav.Import.Geofabrik
  alias Triglav.Schemas.Osmosis.Relation
  alias Triglav.Zet.Gtfs
  alias Triglav.Zet.Osmosis


@@ 9,7 9,7 @@ defmodule TriglavWeb.Zet.RoutesController do
  def index(conn, _params) do
    routes = Gtfs.list_routes(with_errors: true)
    relations = Osmosis.list_public_transport_relations(members: true)
    osm_info = DataImport.load_state()
    {:ok, osm_state} = Geofabrik.local_state()
    gtfs_info = Gtfs.get_feed_info()

    # Relations which were not matched to any known route


@@ 22,7 22,7 @@ defmodule TriglavWeb.Zet.RoutesController do
    render(conn, "index.html",
      routes: annotate_routes(routes, relations),
      unmached_relations: unmached_relations,
      osm_info: osm_info,
      osm_state: osm_state,
      gtfs_info: gtfs_info
    )
  end

M lib/triglav_web/templates/zet/routes/index.html.eex => lib/triglav_web/templates/zet/routes/index.html.eex +4 -4
@@ 52,12 52,12 @@
        </td>
      </tr>
      <tr>
        <th>Sequence</th>
        <td><%= @osm_info.osm_sequence_number %></td>
        <th>Imported at</th>
        <td><%= @osm_state.timestamp %></td>
      </tr>
      <tr>
        <th>Imported at</th>
        <td><%= @osm_info.osm_timestamp %></td>
        <th>Sequence no.</th>
        <td><%= @osm_state.sequence_number %></td>
      </tr>
    </table>
  </div>

A priv/repo/migrations/20210207162740_drop_imported_data.exs => priv/repo/migrations/20210207162740_drop_imported_data.exs +7 -0
@@ 0,0 1,7 @@
defmodule Triglav.Repo.Migrations.DropImportedData do
  use Ecto.Migration

  def change do
    drop table("imported_data")
  end
end