~ihabunek/triglav

837d6e21f3fc80e932ea631bb402f346769c4154 — Ivan Habunek 3 months ago 7bdb88d
Reorganize validator to use new data
M lib/mix/tasks/triglav/validate_routes.ex => lib/mix/tasks/triglav/validate_routes.ex +5 -2
@@ 65,8 65,11 @@ defmodule Mix.Tasks.Triglav.ValidateRoutes do
        %{history: history} = results = do_validate(osm_state, zet_feed_info)

        Logger.info("Routes validated")
        Logger.info("Found #{history.created_count} new error(s)")
        Logger.info("Found #{history.resolved_count} resolved error(s)")

        Logger.info(
          "Found #{history.count} errors " <>
            "(#{history.created_count} new, #{history.resolved_count} resolved)"
        )

        if dry_run do
          Logger.info("Dry run. Not persisting errors.")

M lib/triglav/derived/public_transport.ex => lib/triglav/derived/public_transport.ex +16 -1
@@ 12,6 12,22 @@ defmodule Triglav.Derived.PublicTransport do
  alias Triglav.Zet.Gtfs
  alias Triglav.Zet.Osmosis

  import Ecto.Query
  import Geo.PostGIS

  def list_trips() do
    from(t in Trip) |> Repo.all()
  end

  def list_platforms() do
    from(p in Platform,
      join: t in Trip,
      on: t.relation_id == p.relation_id,
      select: %{p | distance_from_route: st_distance_in_meters(p.geometry, t.geometry)}
    )
    |> Repo.all()
  end

  def generate() do
    relations = Osmosis.list_public_transport_relations(members: true, type: "route")
    relations_map = Map.new(relations, &{&1.id, &1})


@@ 35,7 51,6 @@ defmodule Triglav.Derived.PublicTransport do
      |> Enum.map(&generate_platform(&1, stops_map, routes_map, relations_map))

    persist(trips, platforms)
    nil
  end

  defp persist(trips, platforms) do

M lib/triglav/geo_json.ex => lib/triglav/geo_json.ex +2 -1
@@ 56,7 56,8 @@ defmodule Triglav.GeoJSON do
  @spec ways_to_multilinestring([Way.t()]) :: map()
  def ways_to_multilinestring(ways) do
    %Geo.MultiLineString{
      coordinates: Enum.map(ways, & &1.linestring.coordinates)
      coordinates: Enum.map(ways, & &1.linestring.coordinates),
      srid: List.first(ways).linestring.srid
    }
  end
end

M lib/triglav/schemas/public_transport/platform.ex => lib/triglav/schemas/public_transport/platform.ex +4 -0
@@ 5,6 5,7 @@ defmodule Triglav.Schemas.PublicTransport.Platform do
  use Ecto.Schema

  alias Triglav.Schemas.Osmosis
  alias Triglav.Schemas.PublicTransport.Trip
  alias Triglav.Schemas.Zet

  @type t :: %__MODULE__{}


@@ 19,5 20,8 @@ defmodule Triglav.Schemas.PublicTransport.Platform do
    belongs_to :way, Osmosis.Way
    field :sequence_id, :integer, primary_key: true
    field :geometry, Geo.PostGIS.Geometry

    has_one :trip, Trip, foreign_key: :relation_id, references: :relation_id
    field :distance_from_route, :float, virtual: true
  end
end

M lib/triglav/zet/validator.ex => lib/triglav/zet/validator.ex +98 -47
@@ 8,35 8,88 @@ defmodule Triglav.Zet.Validator do
  alias Triglav.Schemas.Zet.Route
  alias Triglav.Zet.Gtfs
  alias Triglav.Zet.Osmosis
  alias Triglav.Derived.PublicTransport
  alias Triglav.Schemas.PublicTransport.Platform

  defmodule ValidationData do
    @moduledoc """
    Contains data loaded from the database and used for validation.
    """

    @type t :: %__MODULE__{
            platforms_by_relation_id: %{integer() => [Platform.t()]},
            relations: [Relation.t()],
            relations_by_ref: %{binary() => [Relation.t()]},
            routes: [Route.t()],
            trips_map: %{integer() => Trip.t()},
            ways_map: %{integer() => Way.t()}
          }

    defstruct [
      :platforms_by_relation_id,
      :relations,
      :relations_by_ref,
      :routes,
      :trips_map,
      :ways_map
    ]
  end

  @spec validate_all_routes() :: [Error.t()]
  def validate_all_routes() do
  def load_data() do
    routes = Gtfs.list_routes()
    relations = Osmosis.list_public_transport_relations(members: true)
    relation_stop_ids = relations |> Enum.map(& &1.id) |> Osmosis.list_gtfs_stop_ids()
    ways = Osmosis.list_ways(relations) |> Map.new(&{&1.id, &1})
    relations_by_ref = Enum.group_by(relations, & &1.tags["ref"])
    trips = PublicTransport.list_trips()
    trips_map = trips |> Map.new(&{&1.relation_id, &1})
    platforms = PublicTransport.list_platforms()
    platforms_by_relation_id = Enum.group_by(platforms, & &1.relation_id)

    # Ways are only used for finding where routes are broken, so only fetch ways
    # for broken relations.
    broken_relations_ids = trips |> Enum.filter(&(not &1.exact)) |> Enum.map(& &1.relation_id)
    ways = relations |> Enum.filter(&(&1.id in broken_relations_ids)) |> Osmosis.list_ways()
    ways_map = Map.new(ways, &{&1.id, &1})

    %ValidationData{
      platforms_by_relation_id: platforms_by_relation_id,
      relations: relations,
      relations_by_ref: relations_by_ref,
      routes: routes,
      trips_map: trips_map,
      ways_map: ways_map
    }
  end

    for route <- routes do
      route_relations = Enum.filter(relations, &(Relation.ref(&1) == route.id))
      validate_route(route, route_relations, ways, relation_stop_ids)
    end
  @spec validate_all_routes() :: [Error.t()]
  def validate_all_routes() do
    data = load_data()

    data.routes
    |> Enum.map(&validate_route(&1, data))
    |> List.flatten()
  end

  @spec validate_route(
          Route.t(),
          [Relation.t()],
          %{integer() => Way.t()},
          %{integer() => [integer()]}
        ) :: [Error.t()]
  def validate_route(route, relations, ways, relation_stop_ids) do
  @spec validate_route(Route.t(), ValidationData.t()) :: [Error.t()]
  def validate_route(route, data) do
    relations = Enum.filter(data.relations, &(Relation.ref(&1) == route.id))

    route_errors =
      relations
      |> Enum.filter(&Relation.is_route/1)
      |> Enum.map(&validate_route_relation(route, &1, data))

    route_master_errors =
      relations
      |> Enum.filter(&Relation.is_route_master/1)
      |> Enum.map(&validate_route_master_relation(route, &1))

    [
      validate_has_relations(route, relations),
      validate_has_route_master(route, relations),
      validate_routes_are_contained_in_route_master(route, relations),
      validate_routes_master_route_has_no_unknown_members(route, relations),
      Enum.map(relations, &validate_route_master_relation(route, &1)),
      Enum.map(relations, &validate_route_relation(route, &1, ways, relation_stop_ids))
      route_errors,
      route_master_errors
    ]
    |> List.flatten()
    |> Enum.filter(& &1)


@@ 98,10 151,7 @@ defmodule Triglav.Zet.Validator do

  # ROUTE MASTER RELATION

  defp validate_route_master_relation(
         %Route{} = route,
         %Relation{tags: %{"type" => "route_master"}} = relation
       ) do
  defp validate_route_master_relation(%Route{} = route, %Relation{} = relation) do
    required_tags = ["type", "ref", "name", "operator"]
    allowed_tags = required_tags ++ ["route_master", "network"]



@@ 123,18 173,14 @@ defmodule Triglav.Zet.Validator do

  defp validate_route_master_relation(_, _), do: nil

  @spec validate_route_relation(Route.t(), Relation.t(), ValidationData.t()) :: [Error.t()]
  defp validate_route_relation(
         %Route{} = route,
         %Relation{tags: %{"type" => "route"}} = relation,
         ways,
         relation_stop_ids
         %ValidationData{} = data
       ) do
    ordered_ways =
      relation.members
      |> Enum.filter(&(&1.member_type == "W" and &1.member_role == ""))
      |> Enum.map(fn member -> Map.fetch!(ways, member.member_id) end)

    router_result = Triglav.Osm.Router.route_ways(ordered_ways)
    trip = Map.get(data.trips_map, relation.id)
    platforms = Map.get(data.platforms_by_relation_id, relation.id, [])

    required_tags = ["type", "route", "ref", "operator", "from", "to", "name"]



@@ 153,14 199,12 @@ defmodule Triglav.Zet.Validator do
      validate_required_tags(route, relation, required_tags),
      validate_allowed_tags(route, relation, allowed_tags),
      validate_tag_values(route, relation, expected_tags),
      validate_continuous_ways(route, relation, router_result),
      validate_has_gtfs_stop_ids(route, relation, relation_stop_ids),
      validate_continuous_ways(route, relation, trip, data),
      validate_has_gtfs_stop_ids(route, relation, platforms),
      validate_member_roles(route, relation)
    ]
  end

  defp validate_route_relation(_, _, _, _), do: nil

  defp validate_is_ptv2(route, relation) do
    if !Relation.is_ptv2(relation) do
      Error.relation_not_updated_to_ptv2(route, relation)


@@ 190,25 234,32 @@ defmodule Triglav.Zet.Validator do
    end
  end

  defp validate_continuous_ways(route, relation, router_result) do
    case router_result do
      {:ok, _} ->
        nil
  # If trip has an exact linestring, that means route is complete, so only
  # run this check if exact is false.
  defp validate_continuous_ways(route, relation, trip, data) do
    if not trip.exact do
      ordered_ways =
        relation.members
        |> Enum.filter(&(&1.member_type == "W" and &1.member_role == ""))
        |> Enum.map(fn member -> Map.fetch!(data.ways_map, member.member_id) end)

      {:error, {:ways_not_connected, _w1, w2}} ->
        Error.broken_route(route, relation, w2.id)
      router_result = Triglav.Osm.Router.route_ways(ordered_ways)

      case router_result do
        {:ok, _} ->
          nil

        {:error, {:ways_not_connected, _w1, w2}} ->
          Error.broken_route(route, relation, w2.id)
      end
    end
  end

  defp validate_has_gtfs_stop_ids(route, relation, relation_stop_ids) do
    stop_ids = Map.get(relation_stop_ids, relation.id)
  defp validate_has_gtfs_stop_ids(route, relation, platforms) do
    empty_count = Enum.count(platforms, &is_nil(&1.stop_id))

    if is_list(stop_ids) do
      empty_count = Enum.count(stop_ids, &is_nil/1)

      if empty_count > 0 do
        Error.relation_missing_gtfs_stop_ids(route, relation, empty_count)
      end
    if empty_count > 0 do
      Error.relation_missing_gtfs_stop_ids(route, relation, empty_count)
    end
  end