A lib/triglav/poi.ex => lib/triglav/poi.ex +84 -0
@@ 0,0 1,84 @@
+defmodule Triglav.Poi do
+ alias Triglav.Poi.{Node, Mapping, Source}
+ alias Triglav.Poi.Sources
+ alias Triglav.Repo
+
+ import Ecto.Changeset
+ import Ecto.Query
+
+ require Logger
+
+ @sources [
+ Sources.Crodux,
+ Sources.Tifon
+ ]
+
+ def update_all() do
+ tasks = for source <- @sources, do: Task.async(fn -> update(source) end)
+ Task.await_many(tasks, :timer.seconds(60))
+ end
+
+ def update(sources) when is_list(sources) do
+ for source <- sources do
+ update(source)
+ end
+ end
+
+ def update(source_module) when is_atom(source_module) do
+ Repo.transact(
+ fn ->
+ source = Sources.get_or_create!(source_module)
+
+ with {:ok, nodes} <- source_module.fetch() do
+ delete_mappings(source)
+ delete_nodes(source)
+ node_count = persist_nodes(source, nodes)
+ mappings = source_module.find_mappings()
+ mapping_count = persist_mappings(source, mappings)
+
+ source
+ |> change(%{node_count: node_count, mapping_count: mapping_count})
+ |> Repo.update()
+ end
+ end,
+ timeout: :timer.seconds(60)
+ )
+ end
+
+ def regenerate(source_module) do
+ Repo.transact(
+ fn ->
+ source = Sources.get_or_create!(source_module)
+ mappings = source_module.find_mappings()
+ delete_mappings(source)
+ persist_mappings(source, mappings)
+ end,
+ timeout: :timer.seconds(60)
+ )
+ end
+
+ @spec nodes(Source.t()) :: [Node.t()]
+ def nodes(source) do
+ Repo.all(from(Node, where: [source_id: ^source.id], order_by: [asc: :name]))
+ end
+
+ defp delete_nodes(source) do
+ Repo.delete_all(where(Node, source_id: ^source.id))
+ end
+
+ defp delete_mappings(source) do
+ Repo.delete_all(where(Mapping, source_id: ^source.id))
+ end
+
+ defp persist_nodes(source, nodes) do
+ {count, nil} = Repo.insert_all(Node, nodes)
+ Logger.info("#{source.name}: Inserted #{count} nodes")
+ count
+ end
+
+ defp persist_mappings(source, mappings) do
+ {count, nil} = Repo.insert_all(Mapping, mappings)
+ Logger.info("#{source.name}: Inserted #{count} mappings")
+ count
+ end
+end
A lib/triglav/poi/mapping.ex => lib/triglav/poi/mapping.ex +15 -0
@@ 0,0 1,15 @@
+defmodule Triglav.Poi.Mapping do
+ alias Triglav.Poi.{Node, Source}
+ alias Triglav.Schemas.Osmosis
+ use Ecto.Schema
+
+ @type t() :: %__MODULE__{}
+
+ schema "poi_mappings" do
+ belongs_to :source, Source
+ belongs_to :node, Node
+ belongs_to :osm_node, Osmosis.Node
+ belongs_to :osm_way, Osmosis.Way
+ field :distance, :float
+ end
+end
A lib/triglav/poi/node.ex => lib/triglav/poi/node.ex +15 -0
@@ 0,0 1,15 @@
+defmodule Triglav.Poi.Node do
+ use Ecto.Schema
+
+ @type t() :: %__MODULE__{}
+ @type id() :: pos_integer()
+
+ schema "poi_nodes" do
+ belongs_to :source, Triglav.Poi.Source
+ field :geometry, Geo.PostGIS.Geometry
+ field :name, :string
+ field :ref, :string
+ field :tags, {:map, :string}
+ has_one :mapping, Triglav.Poi.Mapping
+ end
+end
A lib/triglav/poi/source.ex => lib/triglav/poi/source.ex +40 -0
@@ 0,0 1,40 @@
+defmodule Triglav.Poi.Source do
+ use Ecto.Schema
+
+ alias Triglav.Poi.{Node, Mapping}
+
+ @type t() :: %__MODULE__{}
+ @type id() :: pos_integer()
+
+ schema "poi_sources" do
+ field :name, :string
+ field :slug, :string
+ field :node_count, :integer
+ field :mapping_count, :integer
+ timestamps()
+
+ has_many :nodes, Node
+ has_many :mappings, Mapping
+ end
+
+ @type poi_node :: %{
+ source_id: id(),
+ geometry: Geo.geometry(),
+ name: String.t(),
+ ref: String.t(),
+ tags: %{String.t() => String.t()}
+ }
+
+ @type mapping :: %{
+ source_id: id(),
+ node_id: Node.id(),
+ osm_node_id: integer() | nil,
+ osm_way_id: integer() | nil,
+ distance: float()
+ }
+
+ @callback slug() :: String.t()
+ @callback name() :: String.t()
+ @callback fetch() :: {:ok, [poi_node()]} | {:error, any()}
+ @callback find_mappings() :: [mapping()]
+end
A lib/triglav/poi/sources.ex => lib/triglav/poi/sources.ex +28 -0
@@ 0,0 1,28 @@
+defmodule Triglav.Poi.Sources do
+ alias Triglav.Poi.Source
+ alias Triglav.Poi.Sources
+ alias Triglav.Repo
+ import Ecto.Query
+
+ @spec all() :: [Source.t()]
+ def all(), do: Source |> order_by(asc: :name) |> Repo.all()
+
+ @spec get!(atom) :: Source.t()
+ def get!(slug), do: Repo.get_by!(Source, slug: slug)
+
+ @spec crodux() :: Source.t()
+ def crodux(), do: get_or_create!(Sources.Crodux)
+
+ @spec tifon() :: Source.t()
+ def tifon(), do: get_or_create!(Sources.Tifon)
+
+ def get_or_create!(source) do
+ %Source{
+ name: source.name(),
+ slug: source.slug()
+ }
+ |> Repo.insert!(on_conflict: :nothing, conflict_target: :slug)
+
+ Repo.one!(where(Source, slug: ^source.slug()))
+ end
+end
A lib/triglav/poi/sources/crodux.ex => lib/triglav/poi/sources/crodux.ex +184 -0
@@ 0,0 1,184 @@
+defmodule Triglav.Poi.Sources.Crodux do
+ use Tesla, only: [:get]
+
+ plug Tesla.Middleware.Retry,
+ delay: :timer.seconds(1),
+ max_retries: 5,
+ should_retry: fn
+ {:ok, %{status: status}} when status != 200 -> true
+ {:ok, _} -> false
+ {:error, _} -> true
+ end
+
+ plug Tesla.Middleware.JSON
+ plug Tesla.Middleware.Timeout, timeout: :timer.seconds(10)
+ plug Tesla.Middleware.Logger, debug: false
+ plug Triglav.Tesla.Middleware.ErrrorOnStatus
+
+ alias Triglav.Poi
+ alias Triglav.Poi.Source
+ alias Triglav.Poi.Sources
+ alias Triglav.Repo
+ alias Triglav.Schemas.Osmosis
+ alias Triglav.Helpers.MapUtils
+
+ import Ecto.Query
+ import Geo.PostGIS
+ import Triglav.Query
+
+ require Logger
+
+ @behaviour Source
+
+ @impl Source
+ def name(), do: "Crodux"
+
+ @impl Source
+ def slug(), do: "crodux"
+
+ @impl Source
+ def fetch() do
+ url = "https://crodux-derivati.hr/wp-json/acf/v3/benzinski_servis/?per_page=500"
+
+ with {:ok, response} <- get(url) do
+ {:ok, parse(response.body)}
+ end
+ end
+
+ def parse(body) do
+ source = Sources.get_or_create!(__MODULE__)
+
+ body
+ |> Enum.map(&parse_node(source, &1))
+ |> Enum.sort_by(& &1.name)
+ end
+
+ @impl Source
+ def find_mappings() do
+ source = Sources.get_or_create!(__MODULE__)
+
+ node_mappings =
+ Repo.all(
+ from(poi_node in Poi.Node,
+ where: poi_node.source_id == ^source.id,
+ join: osm_node in Osmosis.Node,
+ on:
+ tag(osm_node, "amenity") == "fuel" and
+ (tag(osm_node, "operator") == "Crodux" or tag(osm_node, "name") == "Crodux") and
+ st_distance_in_meters(osm_node.geom, poi_node.geometry) < 100,
+ select: {poi_node.id, osm_node.id, st_distance_in_meters(osm_node.geom, poi_node.geometry)},
+ order_by: [asc: 1, asc: 3]
+ )
+ )
+ # If multiple nodes are matched keep the closest one only
+ |> Enum.dedup_by(fn {node_id, _, _} -> node_id end)
+ |> Enum.map(fn {node_id, osm_node_id, distance} ->
+ %{
+ source_id: source.id,
+ node_id: node_id,
+ osm_node_id: osm_node_id,
+ distance: distance
+ }
+ end)
+
+ way_mappings =
+ Repo.all(
+ from(poi_node in Poi.Node,
+ where: poi_node.source_id == ^source.id,
+ join: osm_way in Osmosis.Way,
+ on:
+ tag(osm_way, "amenity") == "fuel" and
+ (tag(osm_way, "operator") == "Crodux" or tag(osm_way, "name") == "Crodux") and
+ st_distance_in_meters(osm_way.linestring, poi_node.geometry) < 100,
+ select: {poi_node.id, osm_way.id, st_distance_in_meters(osm_way.linestring, poi_node.geometry)},
+ order_by: [asc: 1, asc: 3]
+ )
+ )
+ # If multiple nodes are matched keep the closest one only
+ |> Enum.dedup_by(fn {node_id, _, _} -> node_id end)
+ |> Enum.map(fn {node_id, osm_way_id, distance} ->
+ %{
+ source_id: source.id,
+ node_id: node_id,
+ osm_way_id: osm_way_id,
+ distance: distance
+ }
+ end)
+
+ Enum.concat(node_mappings, way_mappings)
+ end
+
+ def parse_node(source, record) do
+ address = parse_address(record["acf"]["adresa"])
+ %{"lat" => lat, "lng" => lng} = record["acf"]["koordinate"]
+ {lat, ""} = Float.parse(lat)
+ {lng, ""} = Float.parse(lng)
+
+ %{
+ source_id: source.id,
+ ref: to_string(record["id"]),
+ name: record["acf"]["title"],
+ geometry: %Geo.Point{coordinates: {lng, lat}, srid: 4326},
+ tags:
+ %{
+ # TODO: opening_hours
+ amenity: "fuel",
+ name: "Crodux",
+ official_name: record["acf"]["title"],
+ operator: "Crodux",
+ phone: record["acf"]["telefon"],
+ website: "https://crodux-derivati.hr/"
+ }
+ |> Map.merge(address)
+ |> MapUtils.remove_blank()
+ }
+ end
+
+ defp parse_address(address) do
+ [street_housenumber, postcode_city] =
+ address
+ # Replace notes in brackets
+ |> String.replace(~r"\(.+\)"U, "")
+ # Replace nonbreaking space
+ |> String.replace(<<160::utf8>>, " ")
+ |> String.split("<br />")
+ |> Enum.map(&String.trim/1)
+ |> Enum.reject(&(&1 == ""))
+
+ [postcode, city] =
+ postcode_city
+ |> String.replace("Požega Osječka 34 000", "34000 Požega")
+ |> String.replace("Pula 52100", "52100 Pula")
+ |> String.replace("Grad Novigrad - Cittanova", "Novigrad")
+ |> String.replace("Sv.Križ", "Sveti Križ Začretje")
+ |> String.replace("Sv. Križ", "Sveti Križ Začretje")
+ # Remove space between post code digits
+ |> String.replace(~r"(\d{2}) (\d{3})", "\\1\\2")
+ |> String.split(~r"\s+", parts: 2, trim: true)
+
+ [street_housenumber | _subplace] =
+ street_housenumber
+ |> String.trim(",")
+ |> String.split(",")
+
+ [housenumber | street] =
+ street_housenumber
+ |> String.replace(~r" AC .+", "")
+ |> String.replace(~r"Autocesta.+", "")
+ |> String.replace(~r" 18 i", " 18i")
+ |> String.replace(~r" 346 a", " 346a")
+ |> String.replace(~r"Rupa (Istok|Zapad)", "")
+ |> String.replace(~r"br.", "")
+ |> String.split(~r"\s+")
+ |> Enum.reverse()
+
+ street = Enum.reverse(street) |> Enum.join(" ")
+
+ %{
+ "addr:housenumber": housenumber,
+ "addr:street": street,
+ "addr:city": city,
+ "addr:postcode": postcode
+ }
+ end
+end
A lib/triglav/poi/sources/tifon.ex => lib/triglav/poi/sources/tifon.ex +214 -0
@@ 0,0 1,214 @@
+defmodule Triglav.Poi.Sources.Tifon do
+ use Tesla, only: [:post]
+
+ plug Tesla.Middleware.FormUrlencoded
+ plug Tesla.Middleware.JSON
+
+ plug Tesla.Middleware.Retry,
+ delay: :timer.seconds(1),
+ max_retries: 5,
+ should_retry: fn
+ {:ok, %{status: status}} when status != 200 -> true
+ {:ok, _} -> false
+ {:error, _} -> true
+ end
+
+ plug Tesla.Middleware.Timeout, timeout: 10_000
+ plug Triglav.Tesla.Middleware.ErrrorOnStatus
+ plug Triglav.Tesla.Middleware.Logger
+
+ alias Triglav.Helpers.MapUtils
+ alias Triglav.Poi
+ alias Triglav.Poi.Source
+ alias Triglav.Poi.Sources
+ alias Triglav.Repo
+ alias Triglav.Schemas.Osmosis
+
+ import Ecto.Query
+ import Geo.PostGIS
+ import Triglav.Query
+
+ require Logger
+
+ @behaviour Source
+
+ @impl Source
+ def name(), do: "Tifon"
+
+ @impl Source
+ def slug(), do: "tifon"
+
+ @impl Source
+ def fetch() do
+ source = Sources.tifon()
+
+ with {:ok, records} <- fetch_list() do
+ Enum.reduce_while(records, [], fn record, acc ->
+ case fetch_details(record) do
+ {:ok, details} ->
+ {:cont, [parse_node(source, record, details) | acc]}
+
+ {:error, error} ->
+ Logger.error(error)
+ {:halt, {:error, error}}
+ end
+ end)
+ end
+ |> case do
+ {:error, error} -> {:error, error}
+ list when is_list(list) -> {:ok, Enum.sort_by(list, & &1.name)}
+ end
+ end
+
+ @impl Source
+ def find_mappings() do
+ source = Sources.tifon()
+
+ node_mappings =
+ Repo.all(
+ from(poi_node in Poi.Node,
+ where: poi_node.source_id == ^source.id,
+ join: osm_node in Osmosis.Node,
+ on:
+ tag(osm_node, "amenity") == "fuel" and
+ (tag(osm_node, "operator") == "Tifon" or tag(osm_node, "name") == "Tifon") and
+ st_distance_in_meters(osm_node.geom, poi_node.geometry) < 100,
+ select: {poi_node.id, osm_node.id, st_distance_in_meters(osm_node.geom, poi_node.geometry)},
+ order_by: [asc: 1, asc: 3]
+ )
+ )
+ # If multiple nodes are matched keep the closest one only
+ |> Enum.dedup_by(fn {node_id, _, _} -> node_id end)
+ |> Enum.map(fn {node_id, osm_node_id, distance} ->
+ %{
+ source_id: source.id,
+ node_id: node_id,
+ osm_node_id: osm_node_id,
+ distance: distance
+ }
+ end)
+
+ way_mappings =
+ Repo.all(
+ from(poi_node in Poi.Node,
+ where: poi_node.source_id == ^source.id,
+ join: osm_way in Osmosis.Way,
+ on:
+ tag(osm_way, "amenity") == "fuel" and
+ (tag(osm_way, "operator") == "Tifon" or tag(osm_way, "name") == "Tifon") and
+ st_distance_in_meters(osm_way.linestring, poi_node.geometry) < 100,
+ select: {poi_node.id, osm_way.id, st_distance_in_meters(osm_way.linestring, poi_node.geometry)},
+ order_by: [asc: 1, asc: 3]
+ )
+ )
+ # If multiple ways are matched keep the closest one only
+ |> Enum.dedup_by(fn {poi_id, _, _} -> poi_id end)
+ |> Enum.map(fn {node_id, osm_way_id, distance} ->
+ %{
+ source_id: source.id,
+ node_id: node_id,
+ osm_way_id: osm_way_id,
+ distance: distance
+ }
+ end)
+
+ Enum.concat(node_mappings, way_mappings)
+ end
+
+ defp fetch_list() do
+ url = "https://pretrazivacpostaja.tifon.hr/hr/portlet/routing/along_latlng.json"
+
+ with {:ok, response} <- post(url, %{country: "Croatia", lat: 45.813, lng: 15.9779}) do
+ {:ok, response.body}
+ end
+ end
+
+ defp fetch_details(record) do
+ url = "https://pretrazivacpostaja.tifon.hr/hr/portlet/routing/station_info.json"
+
+ with {:ok, response} <- post(url, %{id: record["id"]}) do
+ {:ok, response.body}
+ end
+ end
+
+ defp parse_node(source, record, details) do
+ {street, house_number} = parse_address(record["address"])
+
+ {lat, ""} = Float.parse(record["lat"])
+ {lng, ""} = Float.parse(record["lng"])
+
+ name =
+ record["name"]
+ |> String.split()
+ |> Enum.map(&String.capitalize/1)
+ |> Enum.join(" ")
+
+ %{
+ source_id: source.id,
+ ref: record["id"],
+ name: name,
+ geometry: %Geo.Point{coordinates: {lng, lat}, srid: 4326},
+ tags:
+ MapUtils.remove_blank(%{
+ "addr:city": record["city"],
+ "addr:housenumber": house_number,
+ "addr:postcode": record["postcode"],
+ "addr:street": street,
+ amenity: "fuel",
+ mobile: details["fs"]["fs_mobile_num"],
+ name: "Tifon",
+ official_name: record["name"],
+ opening_hours: parse_opening_hours(details),
+ operator: "Tifon",
+ phone: details["fs"]["fs_phone_num"],
+ website: "https://www.tifon.hr/"
+ })
+ }
+ end
+
+ defp parse_opening_hours(details) do
+ # TODO: check how to handle this
+ if details["fs"]["opn_hrs_wtr_wd"] != details["fs"]["opn_hrs_smr_wd"] or
+ details["fs"]["opn_hrs_wtr_sat"] != details["fs"]["opn_hrs_smr_sat"] or
+ details["fs"]["opn_hrs_wtr_sun"] != details["fs"]["opn_hrs_smr_sun"] do
+ Logger.warning("Summer working hours different from winter.")
+ end
+
+ weekday = details["fs"]["opn_hrs_wtr_wd"]
+ saturday = details["fs"]["opn_hrs_wtr_sat"]
+ sunday = details["fs"]["opn_hrs_wtr_sun"]
+
+ cond do
+ weekday != saturday and saturday != sunday ->
+ "Mo-Fr #{weekday}; Sa #{saturday}; Su #{sunday}"
+
+ weekday == saturday and saturday != sunday ->
+ "Mo-Sa #{weekday}; Su #{sunday}"
+
+ weekday != saturday and saturday == sunday ->
+ "Mo-Fr #{weekday}; Sa-Su #{sunday}"
+
+ weekday == saturday and saturday == sunday ->
+ "Mo-Su #{weekday}"
+ end
+ end
+
+ defp parse_address(address) do
+ parts =
+ address
+ |> String.replace("; A1 (Dir. Split)", "")
+ |> String.replace("; A6 (Dir. Rijeka - Zagreb)", "")
+ |> String.replace("; A1 (Dir. Zagreb)", "")
+ |> String.replace(" A1 (Dir. Split)", "")
+ |> String.replace(" A1 (Dir. Zagreb)", "")
+ |> String.replace("471 c", "471c")
+ |> String.replace("Ul. ", "Ulica ")
+ |> String.replace(" ul.", " ulica")
+ |> String.split()
+ |> Enum.reverse()
+
+ [number | street_parts] = parts
+ street = street_parts |> Enum.reverse() |> Enum.join(" ")
+ {street, number}
+ end
+end
A priv/repo/migrations/20221030114114_create_poi.exs => priv/repo/migrations/20221030114114_create_poi.exs +32 -0
@@ 0,0 1,32 @@
+defmodule Triglav.Repo.Migrations.CreatePoi do
+ use Ecto.Migration
+
+ def change do
+ create table(:poi_sources) do
+ add :name, :text, null: false
+ add :slug, :text, null: false
+ add :node_count, :integer
+ add :mapping_count, :integer
+ timestamps()
+ end
+
+ create table(:poi_nodes) do
+ add :source_id, references(:poi_sources), null: false
+ add :name, :text, null: false
+ add :ref, :text
+ add :geometry, :geometry, null: false
+ add :tags, :map, null: false, default: %{}
+ end
+
+ create table(:poi_mappings) do
+ add :source_id, references(:poi_sources), null: false
+ add :node_id, references(:poi_nodes), null: false
+ add :osm_way_id, :bigint
+ add :osm_node_id, :bigint
+ add :distance, :float, null: false
+ end
+
+ create unique_index(:poi_sources, [:slug])
+ create unique_index(:poi_nodes, [:source_id, :ref])
+ end
+end