Paginate superuser users/projects lists with server-side search

CHANGELOG.md

@@ -21,6 +21,9 @@ and this project adheres to
 
 - Editors can now provision and merge sandboxes; merge checks editor+ role on
   the target project [#4384](https://github.com/OpenFn/lightning/issues/4384)
+- Superuser users and projects settings lists now use server-side pagination,
+  sorting, and search for better performance
+  [#2913](https://github.com/OpenFn/lightning/issues/2913)
 - Show specific workflow names in sandbox merge dialog when target project has
   diverged, instead of generic warning message
   [#4001](https://github.com/OpenFn/lightning/issues/4001)

lib/lightning/accounts.ex

@@ -11,6 +11,7 @@ defmodule Lightning.Accounts do
 
   alias Ecto.Changeset
   alias Ecto.Multi
+  alias Lightning.Accounts.AdminSearchParams
   alias Lightning.Accounts.Events
   alias Lightning.Accounts.User
   alias Lightning.Accounts.UserBackupCode
@@ -114,6 +115,20 @@ defmodule Lightning.Accounts do
     Repo.all(User)
   end
 
+  @doc """
+  Returns a paginated list of users for the superuser admin table with
+  server-side filtering and sorting.
+  """
+  @spec list_users_for_admin(map()) :: Scrivener.Page.t()
+  def list_users_for_admin(params \\ %{}) do
+    params = AdminSearchParams.new(params)
+
+    User
+    |> filter_admin_users(params.filter)
+    |> order_admin_users(params.sort, params.dir)
+    |> Repo.paginate(AdminSearchParams.pagination_opts(params))
+  end
+
   @doc """
   Returns the list of users with the given emails
   """
@@ -160,6 +175,48 @@ defmodule Lightning.Accounts do
     if User.valid_password?(user, password), do: user
   end
 
+  defp filter_admin_users(query, ""), do: query
+
+  defp filter_admin_users(query, filter) do
+    search = "%#{filter}%"
+
+    where(
+      query,
+      [u],
+      ilike(u.first_name, ^search) or
+        ilike(u.last_name, ^search) or
+        ilike(u.email, ^search) or
+        ilike(fragment("?::text", u.role), ^search)
+    )
+  end
+
+  defp order_admin_users(query, "enabled", "asc"),
+    do: order_by(query, [u], desc: u.disabled)
+
+  defp order_admin_users(query, "enabled", "desc"),
+    do: order_by(query, [u], asc: u.disabled)
+
+  defp order_admin_users(query, "scheduled_deletion", "asc"),
+    do: order_by(query, [u], asc_nulls_last: u.scheduled_deletion)
+
+  defp order_admin_users(query, "scheduled_deletion", "desc"),
+    do: order_by(query, [u], desc_nulls_first: u.scheduled_deletion)
+
+  defp order_admin_users(query, "role", dir) do
+    direction = dir_to_atom(dir)
+    order_by(query, [u], [{^direction, fragment("?::text", u.role)}])
+  end
+
+  defp order_admin_users(query, sort, dir) do
+    direction = dir_to_atom(dir)
+    sort_field = String.to_existing_atom(sort)
+
+    order_by(query, [u], [{^direction, field(u, ^sort_field)}])
+  end
+
+  defp dir_to_atom("asc"), do: :asc
+  defp dir_to_atom("desc"), do: :desc
+
   @doc """
   Gets a single user.
 

lib/lightning/accounts/admin_search_params.ex

@@ -0,0 +1,118 @@
+defmodule Lightning.Accounts.AdminSearchParams do
+  @moduledoc """
+  Normalized query params for the superuser users table.
+  """
+
+  use Lightning.Schema
+
+  @primary_key false
+
+  @default_sort "email"
+  @allowed_sorts ~w(first_name last_name email role enabled support_user scheduled_deletion)
+  @default_page 1
+  @default_page_size 10
+  @max_page_size 100
+
+  @type t :: %__MODULE__{
+          filter: String.t(),
+          sort: String.t(),
+          dir: String.t(),
+          page: pos_integer(),
+          page_size: pos_integer()
+        }
+
+  embedded_schema do
+    field :filter, :string, default: ""
+    field :sort, :string, default: @default_sort
+    field :dir, :string, default: "asc"
+    field :page, :integer, default: @default_page
+    field :page_size, :integer, default: @default_page_size
+  end
+
+  def new(params \\ %{})
+  def new(%__MODULE__{} = params), do: params
+  def new(nil), do: new(%{})
+
+  def new(params) when is_map(params) do
+    params = stringify_param_keys(params)
+
+    %__MODULE__{}
+    |> cast(
+      %{
+        "filter" => normalize_filter(Map.get(params, "filter")),
+        "sort" => normalize_sort(Map.get(params, "sort")),
+        "dir" => normalize_dir(Map.get(params, "dir")),
+        "page" => parse_positive_int(Map.get(params, "page"), @default_page),
+        "page_size" =>
+          Map.get(params, "page_size")
+          |> parse_positive_int(@default_page_size)
+          |> min(@max_page_size)
+      },
+      [:filter, :sort, :dir, :page, :page_size]
+    )
+    |> apply_action!(:validate)
+  end
+
+  def default_uri_params do
+    new()
+    |> to_uri_params()
+  end
+
+  def pagination_opts(%__MODULE__{} = params) do
+    [page: params.page, page_size: params.page_size]
+  end
+
+  def to_uri_params(%__MODULE__{} = params) do
+    %{
+      "filter" => params.filter,
+      "sort" => params.sort,
+      "dir" => params.dir,
+      "page" => Integer.to_string(params.page),
+      "page_size" => Integer.to_string(params.page_size)
+    }
+  end
+
+  def to_uri_params(params) when is_map(params) do
+    params
+    |> new()
+    |> to_uri_params()
+  end
+
+  defp normalize_sort(sort) when is_binary(sort) do
+    if sort in @allowed_sorts, do: sort, else: @default_sort
+  end
+
+  defp normalize_sort(sort) when is_atom(sort) do
+    sort
+    |> Atom.to_string()
+    |> normalize_sort()
+  end
+
+  defp normalize_sort(_), do: @default_sort
+
+  defp normalize_dir(dir) when dir in ["asc", :asc], do: "asc"
+  defp normalize_dir(dir) when dir in ["desc", :desc], do: "desc"
+  defp normalize_dir(_), do: "asc"
+
+  defp normalize_filter(nil), do: ""
+
+  defp normalize_filter(filter) do
+    filter
+    |> to_string()
+    |> String.trim()
+  end
+
+  defp stringify_param_keys(params) do
+    Map.new(params, fn {key, value} -> {to_string(key), value} end)
+  end
+
+  defp parse_positive_int(value, _default) when is_integer(value) and value > 0,
+    do: value
+
+  defp parse_positive_int(value, default) do
+    case Integer.parse(to_string(value || "")) do
+      {int, ""} when int > 0 -> int
+      _ -> default
+    end
+  end
+end

lib/lightning/projects.ex

@@ -18,6 +18,7 @@ defmodule Lightning.Projects do
   alias Lightning.Invocation.Dataclip
   alias Lightning.Invocation.Step
   alias Lightning.Projects
+  alias Lightning.Projects.AdminSearchParams
   alias Lightning.Projects.Audit
   alias Lightning.Projects.Events
   alias Lightning.Projects.Project
@@ -191,6 +192,23 @@ defmodule Lightning.Projects do
     Repo.all(from(p in Project, order_by: p.name))
   end
 
+  @doc """
+  Returns a paginated list of projects for the superuser admin table with
+  server-side filtering and sorting.
+  """
+  @spec list_projects_for_admin(map()) :: Scrivener.Page.t()
+  def list_projects_for_admin(params \\ %{}) do
+    params = AdminSearchParams.new(params)
+
+    Project
+    |> join(:left, [p], pu in assoc(p, :project_users), on: pu.role == :owner)
+    |> join(:left, [_p, pu], owner in assoc(pu, :user))
+    |> preload([_p, _pu, _owner], project_users: :user)
+    |> filter_admin_projects(params.filter)
+    |> order_admin_projects(params.sort, params.dir)
+    |> Repo.paginate(AdminSearchParams.pagination_opts(params))
+  end
+
   @doc """
   Lists all projects that have history retention
   """
@@ -220,6 +238,52 @@ defmodule Lightning.Projects do
     end
   end
 
+  defp filter_admin_projects(query, ""), do: query
+
+  defp filter_admin_projects(query, filter) do
+    search = "%#{filter}%"
+
+    where(
+      query,
+      [p, _pu, owner],
+      ilike(p.name, ^search) or
+        ilike(p.description, ^search) or
+        ilike(
+          fragment("concat_ws(' ', ?, ?)", owner.first_name, owner.last_name),
+          ^search
+        )
+    )
+  end
+
+  defp order_admin_projects(query, "scheduled_deletion", "asc"),
+    do: order_by(query, [p, _pu, _owner], asc_nulls_last: p.scheduled_deletion)
+
+  defp order_admin_projects(query, "scheduled_deletion", "desc"),
+    do: order_by(query, [p, _pu, _owner], desc_nulls_first: p.scheduled_deletion)
+
+  defp order_admin_projects(query, "owner", dir) do
+    direction = project_dir_to_atom(dir)
+
+    order_by(
+      query,
+      [_p, _pu, owner],
+      [
+        {^direction,
+         fragment("concat_ws(' ', ?, ?)", owner.first_name, owner.last_name)}
+      ]
+    )
+  end
+
+  defp order_admin_projects(query, sort, dir) do
+    direction = project_dir_to_atom(dir)
+    sort_field = String.to_existing_atom(sort)
+
+    order_by(query, [p, _pu, _owner], [{^direction, field(p, ^sort_field)}])
+  end
+
+  defp project_dir_to_atom("asc"), do: :asc
+  defp project_dir_to_atom("desc"), do: :desc
+
   @doc """
   Gets the project associated with a run.
   Traverses Run → WorkOrder → Workflow → Project.