~ihabunek/triglav

5725e4080922882af88029a2e7dffdfc2f1b9911 — Ivan Habunek 3 months ago 26f2c3b
Validate and regenerate derived data after update
M lib/mix/tasks/triglav/import_zet.ex => lib/mix/tasks/triglav/import_zet.ex +23 -1
@@ 1,6 1,11 @@
defmodule Mix.Tasks.Triglav.ImportZet do
  use Mix.Task

  alias Triglav.Derived.PublicTransport
  alias Mix.Tasks.Triglav.ValidateRoutes

  require Logger

  @shortdoc "Imports GTFS data for ZET routes"

  @impl Mix.Task


@@ 9,6 14,23 @@ defmodule Mix.Tasks.Triglav.ImportZet do
    {:ok, _} = Application.ensure_all_started(:triglav)

    {opts, _} = OptionParser.parse!(args, strict: [force: :boolean])
    Triglav.Import.Zet.run(opts)

    case Triglav.Import.Zet.run(opts) do
      {:ok, :no_update_available} ->
        Logger.info("Done. No update available")

      {:ok, {:updated_to, feed}} ->
        Logger.info("Updated to #{feed.version} #{feed.start_date} #{feed.end_date}")

        Logger.info("Regenerating derived data")
        PublicTransport.generate()

        Logger.info("Running validator")
        ValidateRoutes.validate_routes()

        # Currently errors are not returned
        # {:error, error} ->
        #   Logger.error("Update failed. Error:\n#{inspect(error)}")
    end
  end
end

M lib/mix/tasks/triglav/osmosis_update.ex => lib/mix/tasks/triglav/osmosis_update.ex +21 -1
@@ 1,12 1,32 @@
defmodule Mix.Tasks.Triglav.OsmosisUpdate do
  use Mix.Task

  alias Triglav.Derived.PublicTransport
  alias Mix.Tasks.Triglav.ValidateRoutes

  require Logger

  @shortdoc "Updates the osmosis data to latest available."

  @impl Mix.Task
  def run(_args) do
    Application.put_env(:triglav, :repo_only, true)
    {:ok, _} = Application.ensure_all_started(:triglav)
    :ok = Triglav.Import.Osmosis.update()

    case Triglav.Import.Osmosis.update() do
      {:ok, :no_update_available} ->
        Logger.info("Done. No update available")

      {:ok, {:updated_to, state}} ->
        Logger.info("Updated to #{state.sequence_no} #{state.timestamp}")
        Logger.info("Regenerating derived data")
        PublicTransport.generate()

        Logger.info("Running validator")
        ValidateRoutes.validate_routes()

      {:error, error} ->
        Logger.error("Update failed. Error:\n#{inspect(error)}")
    end
  end
end

M lib/triglav/import/osmosis.ex => lib/triglav/import/osmosis.ex +5 -3
@@ 6,6 6,7 @@ defmodule Triglav.Import.Osmosis do
  https://download.geofabrik.de/europe/croatia.html
  """

  alias Triglav.Derived.PublicTransport
  alias Triglav.Import.Geofabrik
  alias Triglav.Repo



@@ 78,7 79,8 @@ defmodule Triglav.Import.Osmosis do
    end
  end

  @spec update() :: :ok | {:error, term}
  @spec update() ::
          {:ok, :no_update_available} | {:ok, {:updated_to, OsmState.t()}} | {:error, term}
  def update() do
    Logger.info("Checking Geofabrik for updates...")



@@ 90,10 92,10 @@ defmodule Triglav.Import.Osmosis do
      if web_state.sequence_number > local_state.sequence_number do
        Logger.info("New data available. Updating.")
        apply_updates(local_state, web_state)
        :ok
        {:ok, {:updated_to, web_state}}
      else
        Logger.info("You already have the latest data")
        :ok
        {:ok, :no_update_available}
      end
    end
  end

M lib/triglav/import/zet.ex => lib/triglav/import/zet.ex +20 -7
@@ 9,9 9,13 @@ defmodule Triglav.Import.Zet do
  alias Triglav.Repo
  alias Triglav.Schemas.Zet.FeedInfo

  require Logger

  @type option :: {:force, boolean()}
  @type options :: [option]
  @type option :: :force

  @spec run(options) ::
          {:ok, :no_update_available} | {:ok, {:updated_to, FeedInfo.t()}} | {:error, term}
  def run(opts \\ []) do
    force = Keyword.get(opts, :force, false)



@@ 23,11 27,16 @@ defmodule Triglav.Import.Zet do
    local_version = if local_info, do: String.to_integer(local_info.version)
    upgrade_available? = is_nil(local_info) or web_version > local_version

    IO.puts("Local version: #{local_version || "none"}")
    IO.puts("  Web version: #{web_version}")
    Logger.info("Local version: #{local_version || "none"}")
    Logger.info("  Web version: #{web_version}")

    if force or upgrade_available? do
      IO.puts("Updating...")
      Logger.info("Updating...")

      # TODO: better error handling:
      # - make functions return {:ok, _} or {:error, _}
      # - use Ecto.Multi to run every step in a transaction
      # - email on error?

      temp_dir = get_temp_dir()
      extract_archive(web_feed_archive, temp_dir)


@@ 39,14 48,18 @@ defmodule Triglav.Import.Zet do
      run_sql("priv/gtfs/transform.sql")

      File.rm_rf!(temp_dir)

      {:ok, {:updated_to, web_info}}
    else
      IO.puts("You already have the latest data. Use :force option to import anyway.")
      Logger.info("You already have the latest data. Use :force option to import anyway.")

      {:ok, :no_update_available}
    end
  end

  def get_web_feed_archive() do
    url = download_url()
    IO.puts("Downloading: #{url}")
    Logger.info("Downloading: #{url}")

    {:ok, {{_http, 200, 'OK'}, _headers, content}} =
      :httpc.request(:get, {url, []}, [], body_format: :binary)


@@ 110,7 123,7 @@ defmodule Triglav.Import.Zet do
  end

  defp run_sql(path) do
    IO.puts("Running: #{path}")
    Logger.info("Running: #{path}")
    path = Application.app_dir(:triglav, path)
    {_, 0} = System.cmd("psql", ["-f", path])
  end

M lib/triglav/release.ex => lib/triglav/release.ex +2 -8
@@ 15,8 15,8 @@ defmodule Triglav.Release do
  end

  def import_zet(opts \\ []) do
    start_repo()
    Triglav.Import.Zet.run(opts)
    load_app()
    Mix.Tasks.Triglav.ImportZet.run(opts)
  end

  def osmosis_init() do


@@ 46,10 46,4 @@ defmodule Triglav.Release do
  defp load_app do
    Application.load(@app)
  end

  defp start_repo do
    load_app()
    Application.put_env(@app, :repo_only, true)
    Application.ensure_all_started(@app)
  end
end

M lib/triglav/schemas/osm_state.ex => lib/triglav/schemas/osm_state.ex +2 -0
@@ 2,6 2,8 @@ defmodule Triglav.Schemas.OsmState do
  use Ecto.Schema
  import Ecto.Changeset

  @type t :: %__MODULE__{}

  schema "osm_state" do
    field :sequence_number, :integer
    field :timestamp, :utc_datetime