~sbaildon/paginator

3508d6ad77a95ac1faf15d5fd7f959fab3e17da2 — Theodor Gherzan 9 months ago d379366 + f0d1084
Merge pull request #166 from mattnenterprise/expression-cursor-v3

Add expression based cursor v3
M README.md => README.md +45 -0
@@ 140,6 140,51 @@ cursor_before = metadata.before
IO.puts "total count: #{metadata.total_count}"
```

## Dynamic expressions

```elixir
  query =
    from(
      f in Post,
      # Alias for fragment must match witch cursor field name in fetch_cursor_value_fun and cursor_fields
      select_merge: %{
        rank_value:
          fragment("ts_rank(document, plainto_tsquery('simple', ?)) AS rank_value", ^q)
      },
      where: fragment("document @@ plainto_tsquery('simple', ?)", ^q),
      order_by: [
        desc: fragment("rank_value"),
        desc: f.id
      ]
    )
    query
    |> Repo.paginate(
      limit: 30,
      fetch_cursor_value_fun: fn
        # Here we build the rank_value for each returned row
        schema, :rank_value ->
          {:ok, %{rows: [[rank_value]]}} =
            Repo.query("SELECT ts_rank($1, plainto_tsquery('simple', $2))", [
              schema.document,
              q
            ])
          rank_value
        schema, field ->
          Paginator.default_fetch_cursor_value(schema, field)
      end,
      cursor_fields: [
        {:rank_value, # Here we build the rank_value that will be used in the where clause
         fn ->
           dynamic(
             [x],
             fragment("ts_rank(document, plainto_tsquery('simple', ?))", ^q)
           )
         end},
        :id
      ]
    )
```

## Security Considerations

`Repo.paginate/4` will throw an `ArgumentError` should it detect an executable term in the cursor parameters passed to it (`before`, `after`).

M lib/paginator.ex => lib/paginator.ex +6 -0
@@ 309,6 309,12 @@ defmodule Paginator do
       }) do
    cursor_fields
    |> Enum.map(fn
      {{cursor_field, func}, _order} when is_atom(cursor_field) and is_function(func) ->
        {cursor_field, fetch_cursor_value_fun.(schema, cursor_field)}

      {cursor_field, func} when is_atom(cursor_field) and is_function(func) ->
        {cursor_field, fetch_cursor_value_fun.(schema, cursor_field)}

      {cursor_field, _order} ->
        {cursor_field, fetch_cursor_value_fun.(schema, cursor_field)}


M lib/paginator/config.ex => lib/paginator/config.ex +7 -0
@@ 96,6 96,13 @@ defmodule Paginator.Config do
        when is_atom(schema) and is_atom(field) and value in @order_directions ->
          {schema, field}

        {{field, func}, value}
        when is_function(func) and is_atom(field) and value in @order_directions ->
          field

        {field, func} when is_function(func) and is_atom(field) ->
          field

        field when is_atom(field) ->
          field


M lib/paginator/ecto/query.ex => lib/paginator/ecto/query.ex +18 -6
@@ 36,9 36,9 @@ defmodule Paginator.Ecto.Query do
    where(query, [{q, 0}], ^filters)
  end

  defp build_where_expression(query, [{column, order}], values, cursor_direction) do
    value = Map.get(values, column)
    {q_position, q_binding} = column_position(query, column)
  defp build_where_expression(query, [{field, order} = column], values, cursor_direction) do
    value = column_value(column, values)
    {q_position, q_binding} = column_position(query, field)

    DynamicFilterBuilder.build!(%{
      sort_order: order,


@@ 50,9 50,9 @@ defmodule Paginator.Ecto.Query do
    })
  end

  defp build_where_expression(query, [{column, order} | fields], values, cursor_direction) do
    value = Map.get(values, column)
    {q_position, q_binding} = column_position(query, column)
  defp build_where_expression(query, [{field, order} = column | fields], values, cursor_direction) do
    value = column_value(column, values)
    {q_position, q_binding} = column_position(query, field)

    filters = build_where_expression(query, fields, values, cursor_direction)



@@ 66,6 66,14 @@ defmodule Paginator.Ecto.Query do
    })
  end

  defp column_value({{field, func}, _order}, values) when is_function(func) and is_atom(field) do
    Map.get(values, field)
  end

  defp column_value({column, _order}, values) do
    Map.get(values, column)
  end

  defp maybe_where(query, %Config{
         after: nil,
         before: nil


@@ 102,6 110,10 @@ defmodule Paginator.Ecto.Query do
    |> filter_values(cursor_fields, before_values, :before)
  end

  # With custom column handler
  defp column_position(_query, {_, handler} = column) when is_function(handler),
    do: {0, column}

  # Lookup position of binding in query aliases
  defp column_position(query, {binding_name, column}) do
    case Map.fetch(query.aliases, binding_name) do

M lib/paginator/ecto/query/asc_nulls_first.ex => lib/paginator/ecto/query/asc_nulls_first.ex +11 -10
@@ 2,6 2,7 @@ defmodule Paginator.Ecto.Query.AscNullsFirst do
  @behaviour Paginator.Ecto.Query.DynamicFilterBuilder

  import Ecto.Query
  import Paginator.Ecto.Query.FieldOrExpression

  @impl Paginator.Ecto.Query.DynamicFilterBuilder
  def build_dynamic_filter(%{direction: :after, value: nil, next_filters: true}) do


@@ 11,23 12,23 @@ defmodule Paginator.Ecto.Query.AscNullsFirst do
  def build_dynamic_filter(args = %{direction: :after, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      (is_nil(field(query, ^args.column)) and ^args.next_filters) or
        not is_nil(field(query, ^args.column))
      (^field_or_expr_is_nil(args) and ^args.next_filters) or
        not (^field_or_expr_is_nil(args))
    )
  end

  def build_dynamic_filter(args = %{direction: :after, next_filters: true}) do
    dynamic(
      [{query, args.entity_position}],
      field(query, ^args.column) > ^args.value
      ^field_or_expr_greater(args)
    )
  end

  def build_dynamic_filter(args = %{direction: :after}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) > ^args.value
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_greater(args)
    )
  end



@@ 38,23 39,23 @@ defmodule Paginator.Ecto.Query.AscNullsFirst do
  def build_dynamic_filter(args = %{direction: :before, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      is_nil(field(query, ^args.column)) and ^args.next_filters
      ^field_or_expr_is_nil(args) and ^args.next_filters
    )
  end

  def build_dynamic_filter(args = %{direction: :before, next_filters: true}) do
    dynamic(
      [{query, args.entity_position}],
      field(query, ^args.column) < ^args.value or is_nil(field(query, ^args.column))
      ^field_or_expr_less(args) or ^field_or_expr_is_nil(args)
    )
  end

  def build_dynamic_filter(args = %{direction: :before}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) < ^args.value or
        is_nil(field(query, ^args.column))
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_less(args) or
        ^field_or_expr_is_nil(args)
    )
  end
end

M lib/paginator/ecto/query/asc_nulls_last.ex => lib/paginator/ecto/query/asc_nulls_last.ex +11 -10
@@ 2,6 2,7 @@ defmodule Paginator.Ecto.Query.AscNullsLast do
  @behaviour Paginator.Ecto.Query.DynamicFilterBuilder

  import Ecto.Query
  import Paginator.Ecto.Query.FieldOrExpression

  @impl Paginator.Ecto.Query.DynamicFilterBuilder
  def build_dynamic_filter(%{direction: :after, value: nil, next_filters: true}) do


@@ 11,23 12,23 @@ defmodule Paginator.Ecto.Query.AscNullsLast do
  def build_dynamic_filter(args = %{direction: :after, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      is_nil(field(query, ^args.column)) and ^args.next_filters
      ^field_or_expr_is_nil(args) and ^args.next_filters
    )
  end

  def build_dynamic_filter(args = %{direction: :after, next_filters: true}) do
    dynamic(
      [{query, args.entity_position}],
      field(query, ^args.column) > ^args.value or is_nil(field(query, ^args.column))
      ^field_or_expr_greater(args) or ^field_or_expr_is_nil(args)
    )
  end

  def build_dynamic_filter(args = %{direction: :after}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) > ^args.value or
        is_nil(field(query, ^args.column))
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_greater(args) or
        ^field_or_expr_is_nil(args)
    )
  end



@@ 38,20 39,20 @@ defmodule Paginator.Ecto.Query.AscNullsLast do
  def build_dynamic_filter(args = %{direction: :before, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      (is_nil(field(query, ^args.column)) and ^args.next_filters) or
        not is_nil(field(query, ^args.column))
      (^field_or_expr_is_nil(args) and ^args.next_filters) or
        not (^field_or_expr_is_nil(args))
    )
  end

  def build_dynamic_filter(args = %{direction: :before, next_filters: true}) do
    dynamic([{query, args.entity_position}], field(query, ^args.column) < ^args.value)
    dynamic([{query, args.entity_position}], ^field_or_expr_less(args))
  end

  def build_dynamic_filter(args = %{direction: :before}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) < ^args.value
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_less(args)
    )
  end
end

M lib/paginator/ecto/query/desc_nulls_first.ex => lib/paginator/ecto/query/desc_nulls_first.ex +11 -10
@@ 2,6 2,7 @@ defmodule Paginator.Ecto.Query.DescNullsFirst do
  @behaviour Paginator.Ecto.Query.DynamicFilterBuilder

  import Ecto.Query
  import Paginator.Ecto.Query.FieldOrExpression

  @impl Paginator.Ecto.Query.DynamicFilterBuilder
  def build_dynamic_filter(%{direction: :before, value: nil, next_filters: true}) do


@@ 11,23 12,23 @@ defmodule Paginator.Ecto.Query.DescNullsFirst do
  def build_dynamic_filter(args = %{direction: :before, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      is_nil(field(query, ^args.column)) and ^args.next_filters
      ^field_or_expr_is_nil(args) and ^args.next_filters
    )
  end

  def build_dynamic_filter(args = %{direction: :before, next_filters: true}) do
    dynamic(
      [{query, args.entity_position}],
      field(query, ^args.column) > ^args.value or is_nil(field(query, ^args.column))
      ^field_or_expr_greater(args) or ^field_or_expr_is_nil(args)
    )
  end

  def build_dynamic_filter(args = %{direction: :before}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) > ^args.value or
        is_nil(field(query, ^args.column))
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_greater(args) or
        ^field_or_expr_is_nil(args)
    )
  end



@@ 38,20 39,20 @@ defmodule Paginator.Ecto.Query.DescNullsFirst do
  def build_dynamic_filter(args = %{direction: :after, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      (is_nil(field(query, ^args.column)) and ^args.next_filters) or
        not is_nil(field(query, ^args.column))
      (^field_or_expr_is_nil(args) and ^args.next_filters) or
        not (^field_or_expr_is_nil(args))
    )
  end

  def build_dynamic_filter(args = %{direction: :after, next_filters: true}) do
    dynamic([{query, args.entity_position}], field(query, ^args.column) < ^args.value)
    dynamic([{query, args.entity_position}], ^field_or_expr_less(args))
  end

  def build_dynamic_filter(args = %{direction: :after}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) < ^args.value
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_less(args)
    )
  end
end

M lib/paginator/ecto/query/desc_nulls_last.ex => lib/paginator/ecto/query/desc_nulls_last.ex +11 -10
@@ 2,6 2,7 @@ defmodule Paginator.Ecto.Query.DescNullsLast do
  @behaviour Paginator.Ecto.Query.DynamicFilterBuilder

  import Ecto.Query
  import Paginator.Ecto.Query.FieldOrExpression

  @impl Paginator.Ecto.Query.DynamicFilterBuilder
  def build_dynamic_filter(%{direction: :before, value: nil, next_filters: true}) do


@@ 11,23 12,23 @@ defmodule Paginator.Ecto.Query.DescNullsLast do
  def build_dynamic_filter(args = %{direction: :before, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      (is_nil(field(query, ^args.column)) and ^args.next_filters) or
        not is_nil(field(query, ^args.column))
      (^field_or_expr_is_nil(args) and ^args.next_filters) or
        not (^field_or_expr_is_nil(args))
    )
  end

  def build_dynamic_filter(args = %{direction: :before, next_filters: true}) do
    dynamic(
      [{query, args.entity_position}],
      field(query, ^args.column) > ^args.value
      ^field_or_expr_greater(args)
    )
  end

  def build_dynamic_filter(args = %{direction: :before}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) > ^args.value
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_greater(args)
    )
  end



@@ 38,23 39,23 @@ defmodule Paginator.Ecto.Query.DescNullsLast do
  def build_dynamic_filter(args = %{direction: :after, value: nil}) do
    dynamic(
      [{query, args.entity_position}],
      is_nil(field(query, ^args.column)) and ^args.next_filters
      ^field_or_expr_is_nil(args) and ^args.next_filters
    )
  end

  def build_dynamic_filter(args = %{direction: :after, next_filters: true}) do
    dynamic(
      [{query, args.entity_position}],
      field(query, ^args.column) < ^args.value or is_nil(field(query, ^args.column))
      ^field_or_expr_less(args) or ^field_or_expr_is_nil(args)
    )
  end

  def build_dynamic_filter(args = %{direction: :after}) do
    dynamic(
      [{query, args.entity_position}],
      (field(query, ^args.column) == ^args.value and ^args.next_filters) or
        field(query, ^args.column) < ^args.value or
        is_nil(field(query, ^args.column))
      (^field_or_expr_equal(args) and ^args.next_filters) or
        ^field_or_expr_less(args) or
        ^field_or_expr_is_nil(args)
    )
  end
end

A lib/paginator/ecto/query/field_or_expression.ex => lib/paginator/ecto/query/field_or_expression.ex +35 -0
@@ 0,0 1,35 @@
defmodule Paginator.Ecto.Query.FieldOrExpression do
  import Ecto.Query

  def field_or_expr_is_nil(%{column: {_, handler}}) do
    dynamic([{query, args.entity_position}], is_nil(^handler.()))
  end

  def field_or_expr_is_nil(args) do
    dynamic([{query, args.entity_position}], is_nil(field(query, ^args.column)))
  end

  def field_or_expr_equal(%{column: {_, handler}, value: value}) do
    dynamic([{query, args.entity_position}], ^handler.() == ^value)
  end

  def field_or_expr_equal(args) do
    dynamic([{query, args.entity_position}], field(query, ^args.column) == ^args.value)
  end

  def field_or_expr_less(%{column: {_, handler}, value: value}) do
    dynamic([{query, args.entity_position}], ^handler.() < ^value)
  end

  def field_or_expr_less(args) do
    dynamic([{query, args.entity_position}], field(query, ^args.column) < ^args.value)
  end

  def field_or_expr_greater(%{column: {_, handler}, value: value}) do
    dynamic([{query, args.entity_position}], ^handler.() > ^value)
  end

  def field_or_expr_greater(args) do
    dynamic([{query, args.entity_position}], field(query, ^args.column) > ^args.value)
  end
end

M test/paginator_test.exs => test/paginator_test.exs +133 -3
@@ 1013,12 1013,101 @@ defmodule PaginatorTest do
    end
  end

  test "expression based field is passed to cursor_fields" do
    base_customer_name = "Bob"

    list = create_customers_with_similar_names(base_customer_name)

    {:ok, customer_3} = Enum.fetch(list, 3)

    %Page{entries: entries, metadata: metadata} =
      base_customer_name
      |> customers_with_tsvector_rank()
      |> Repo.paginate(
        after: encode_cursor(%{rank_value: customer_3.rank_value, id: customer_3.id}),
        limit: 3,
        cursor_fields: [
          {:rank_value,
           fn ->
             dynamic(
               [x],
               fragment(
                 "ts_rank(setweight(to_tsvector('simple', name), 'A'), plainto_tsquery('simple', ?))",
                 ^base_customer_name
               )
             )
           end},
          :id
        ]
      )

    last_entry = List.last(entries)
    first_entry = List.first(entries)

    assert metadata == %Metadata{
             after: encode_cursor(%{rank_value: last_entry.rank_value, id: last_entry.id}),
             before: encode_cursor(%{rank_value: first_entry.rank_value, id: first_entry.id}),
             limit: 3
           }
  end

  test "expression based field when combined with UUID field" do
    base_customer_name = "Bob"

    create_customers_with_similar_names(base_customer_name)

    list = base_customer_name |> customers_with_tsvector_rank() |> Repo.all()
    {:ok, customer_3} = Enum.fetch(list, 3)

    %Page{entries: entries, metadata: metadata} =
      base_customer_name
      |> customers_with_tsvector_rank()
      |> Repo.paginate(
        after:
          encode_cursor(%{
            rank_value: customer_3.rank_value,
            internal_uuid: customer_3.internal_uuid
          }),
        limit: 3,
        cursor_fields: [
          {:rank_value,
           fn ->
             dynamic(
               [x],
               fragment(
                 "ts_rank(setweight(to_tsvector('simple', name), 'A'), plainto_tsquery('simple', ?))",
                 ^base_customer_name
               )
             )
           end},
          :internal_uuid
        ]
      )

    last_entry = List.last(entries)
    first_entry = List.first(entries)

    assert metadata == %Metadata{
             after:
               encode_cursor(%{
                 rank_value: last_entry.rank_value,
                 internal_uuid: last_entry.internal_uuid
               }),
             before:
               encode_cursor(%{
                 rank_value: first_entry.rank_value,
                 internal_uuid: first_entry.internal_uuid
               }),
             limit: 3
           }
  end

  defp to_ids(entries), do: Enum.map(entries, & &1.id)

  defp create_customers_and_payments(_context) do
    c1 = insert(:customer, %{name: "Bob"})
    c2 = insert(:customer, %{name: "Alice"})
    c3 = insert(:customer, %{name: "Charlie"})
    c1 = insert(:customer, %{name: "Bob", internal_uuid: Ecto.UUID.generate()})
    c2 = insert(:customer, %{name: "Alice", internal_uuid: Ecto.UUID.generate()})
    c3 = insert(:customer, %{name: "Charlie", internal_uuid: Ecto.UUID.generate()})

    a1 = insert(:address, city: "London", customer: c1)
    a2 = insert(:address, city: "New York", customer: c2)


@@ 1045,6 1134,26 @@ defmodule PaginatorTest do
     payments: {p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12}}
  end

  defp create_customers_with_similar_names(base_customer_name) do
    1..10
    |> Enum.map(fn i ->
      {:ok, %{rows: [[rank_value]]}} =
        Repo.query(
          "SELECT ts_rank(setweight(to_tsvector('simple', $1), 'A'), plainto_tsquery('simple', $2))",
          [
            "#{base_customer_name} #{i}",
            base_customer_name
          ]
        )

      insert(:customer, %{
        name: "#{base_customer_name} #{i}",
        internal_uuid: Ecto.UUID.generate(),
        rank_value: rank_value
      })
    end)
  end

  defp payments_by_status(status, direction \\ :asc) do
    from(
      p in Payment,


@@ 1114,6 1223,27 @@ defmodule PaginatorTest do
    )
  end

  defp customers_with_tsvector_rank(q) do
    from(f in Customer,
      select_merge: %{
        rank_value:
          fragment(
            "ts_rank(setweight(to_tsvector('simple', name), 'A'), plainto_tsquery('simple', ?)) AS rank_value",
            ^q
          )
      },
      where:
        fragment(
          "setweight(to_tsvector('simple', name), 'A') @@ plainto_tsquery('simple', ?)",
          ^q
        ),
      order_by: [
        asc: fragment("rank_value"),
        asc: f.internal_uuid
      ]
    )
  end

  defp encode_cursor(value) do
    Cursor.encode(value)
  end

M test/support/customer.ex => test/support/customer.ex +2 -0
@@ 6,6 6,8 @@ defmodule Paginator.Customer do
  schema "customers" do
    field(:name, :string)
    field(:active, :boolean)
    field(:internal_uuid, :binary_id)
    field(:rank_value, :float, virtual: true)

    has_many(:payments, Paginator.Payment)
    has_one(:address, Paginator.Address)

M test/support/factory.ex => test/support/factory.ex +1 -0
@@ 6,6 6,7 @@ defmodule Paginator.Factory do
  def customer_factory do
    %Customer{
      name: "Bob",
      internal_uuid: Ecto.UUID.generate(),
      active: true
    }
  end

M test/support/test_migration.ex => test/support/test_migration.ex +1 -0
@@ 5,6 5,7 @@ defmodule Paginator.TestMigration do
    create table(:customers) do
      add(:name, :string)
      add(:active, :boolean)
      add(:internal_uuid, :uuid, null: false)

      timestamps()
    end