diff --git a/docker-compose.yml b/docker-compose.yml index 0281aa4a4..cc0760670 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,8 @@ services: # be a bare `level` which will be used as a catch-all for all other log # events that do not match any of the previous filters. - PHILOMENA_LOG=${PHILOMENA_LOG-Ecto=debug,Exq=none,PhilomenaMedia.Objects=info,debug} + # Set this env var to `y` to regenerate the test snapshots + - ASSERT_VALUE_ACCEPT_DIFFS - MIX_ENV=dev - PGPASSWORD=postgres - ANONYMOUS_NAME_SALT=2fmJRo0OgMFe65kyAJBxPT0QtkVes/jnKDdtP21fexsRqiw8TlSY7yO+uFyMZycp diff --git a/docker/app/run-test b/docker/app/run-test index f845890f1..9a2284edd 100755 --- a/docker/app/run-test +++ b/docker/app/run-test @@ -1,35 +1,44 @@ -#!/usr/bin/env sh -set -e +#!/usr/bin/env bash + +set -euo pipefail + +. "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh" export MIX_ENV=test +cd /srv/philomena + # Always install mix dependencies -(cd /srv/philomena && mix deps.get) +step mix deps.get + +# Some tests depend on prettier +step npm ci --no-scripts # Run formatting check -mix format --check-formatted +step mix format --check-formatted # Sleep to allow OpenSearch to finish initializing # if it's not done doing whatever it does yet -echo -n "Waiting for OpenSearch" +info "Waiting for OpenSearch" -until wget -qO - opensearch:9200; do - echo -n "." +until step wget -qO - http://opensearch:9200; do sleep 2 done -echo - -# Create the database -mix ecto.create -mix ecto.load +# Create the database, or migrate if one already exists from a previous run +if step createdb -h postgres -U postgres philomena_test; then + step mix ecto.create + step mix ecto.load +elif [[ "$(mix ecto.migrations)" == *" down "* ]]; then + step mix ecto.migrate --all +fi # Test the application -mix test +step mix test "$@" # Security lint -mix sobelow --config -mix deps.audit +step mix sobelow --config +step mix deps.audit # Static analysis exec mix dialyzer diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex index 4d41252a4..78ed2b7eb 100644 --- a/lib/philomena/images/query.ex +++ b/lib/philomena/images/query.ex @@ -197,6 +197,13 @@ defmodule Philomena.Images.Query do end end + @type context :: [ + user: map(), + watch: boolean(), + filter: boolean() + ] + + @spec parse(Parser.options(), map(), String.t()) :: Parser.result() defp parse(fields, context, query_string) do case prepare_context(context, query_string) do {:ok, context} -> @@ -214,6 +221,7 @@ defmodule Philomena.Images.Query do defp fields_for(%{role: role}) when role in ~W(moderator admin), do: moderator_fields() defp fields_for(_), do: raise(ArgumentError, "Unknown user role.") + @spec compile(String.t(), context()) :: Parser.result() def compile(query_string, opts \\ []) do user = Keyword.get(opts, :user) watch = Keyword.get(opts, :watch, false) diff --git a/lib/philomena_query/parse/parser.ex b/lib/philomena_query/parse/parser.ex index ab1870055..4921472bf 100644 --- a/lib/philomena_query/parse/parser.ex +++ b/lib/philomena_query/parse/parser.ex @@ -53,6 +53,9 @@ defmodule PhilomenaQuery.Parse.Parser do @typedoc "Query in the search engine JSON query language." @type query :: map() + @typedoc "Result of calling the `parse/3` function." + @type result :: {:ok, query()} | {:error, String.t()} + @typedoc "Whether the default field is `:term` (not analyzed) or `:ngram` (analyzed)." @type default_field_type :: :term | :ngram @@ -126,25 +129,55 @@ defmodule PhilomenaQuery.Parse.Parser do @max_clause_count 512 + @typedoc "Options for the new/1 function." + @type options :: [ + # Booleans + bool_fields: [String.t()], + + # Dates + date_fields: [String.t()], + + # Floating point numbers + float_fields: [String.t()], + + # Signed integers + int_fields: [String.t()], + + # Numeric values (unsigned integers without fuzzing + # or range queries) + numeric_fields: [String.t()], + + # IP CIDR masks + ip_fields: [String.t()], + + # Wildcardable fields which are searched as the exact value + literal_fields: [String.t()], + + # Wildcardable fields which are searched as stemmed values + ngram_fields: [String.t()], + + # Fields which do not exist on the document and are created by a callback + custom_fields: [String.t()], + + # A map of custom field names to transform functions + transforms: %{String.t() => transform()}, + + # A map of field names to the names they should have in the search engine + aliases: %{String.t() => String.t()}, + + # A list of field names which do not have string downcasing applied + no_downcase_fields: [String.t()], + + # a map of field names to normalization functions (see `t:normalizer/0`) + normalizations: %{String.t() => normalizer()} + ] + @doc """ Creates a `Parser` suitable for safely parsing user-input queries. - Fields refer to attributes of the indexed document which will be searchable with - `m:PhilomenaQuery.Search`. - - Available options: - - `bool_fields` - a list of field names parsed as booleans - - `float_fields` - a list of field names parsed as floats - - `int_fields` - a list of field names parsed as signed integers - - `numeric_fields` - a list of field names parsed as numeric values (unsigned integers without fuzzing or range queries) - - `ip_fields` - a list of field names parsed as IP CIDR masks - - `literal_fields` - wildcardable fields which are searched as the exact value - - `ngram_fields` - wildcardable fields which are searched as stemmed values - - `custom_fields` - fields which do not exist on the document and are created by a callback - - `transforms` - a map of custom field names to transform functions - - `aliases` - a map of field names to the names they should have in the search engine - - `no_downcase_fields` - a list of field names which do not have string downcasing applied - - `normalizations` - a map of field names to normalization functions (see `t:normalizer/0`) + Fields refer to attributes of the indexed document which will be searchable + with `m:PhilomenaQuery.Search`. See the available options described in the + `t:options/0` typespec. ## Example @@ -160,7 +193,7 @@ defmodule PhilomenaQuery.Parse.Parser do Parser.new(options) """ - @spec new(keyword()) :: t() + @spec new(options()) :: t() def new(options) do parser = struct(Parser, options) @@ -205,10 +238,8 @@ defmodule PhilomenaQuery.Parse.Parser do {:error, "Imbalanced parentheses."} """ - @spec parse(t(), String.t(), context()) :: {:ok, query()} | {:error, String.t()} - def parse(parser, input, context \\ nil) - - def parse(%Parser{} = parser, input, context) do + @spec parse(t(), String.t(), context()) :: result() + def parse(%Parser{} = parser, input, context \\ nil) do parser = %{parser | __data__: context} with {:ok, input} <- coerce_string(input), diff --git a/mix.exs b/mix.exs index 53b58c28d..706e39930 100644 --- a/mix.exs +++ b/mix.exs @@ -93,7 +93,10 @@ defmodule Philomena.MixProject do # Fixes for Elixir v1.15+ {:canary, "~> 1.1", - github: "marcinkoziej/canary", ref: "704debde7a2c0600f78c687807884bf37c45bd79"} + github: "marcinkoziej/canary", ref: "704debde7a2c0600f78c687807884bf37c45bd79"}, + + # Automated testing + {:assert_value, "~> 0.10.5", only: [:dev, :test]} ] end diff --git a/mix.lock b/mix.lock index c62f7bf0c..e7ea9fb8f 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "assert_value": {:hex, :assert_value, "0.10.5", "b1d68243194ba3da490674ff53a9d9b8dfff411d5dec9fc113c9b82be76eeddf", [:mix], [], "hexpm", "ba89aecb2e886e55b2c7ef9973a153838976b2abd10a931fa2d41b74dfb27de6"}, "bandit": {:hex, :bandit, "1.6.11", "2fbadd60c95310eefb4ba7f1e58810aa8956e18c664a3b2029d57edb7d28d410", [:mix], [{:hpax, "~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "543f3f06b4721619a1220bed743aa77bf7ecc9c093ba9fab9229ff6b99eacc65"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.3.1", "9f2e7e00f661a6acfae1431f1bc590e28698aaecc962c2a7b33150dfe9289c3d", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "9f539e9d3828fad4ffc8152dadc0d27c6d78cb2853a9a1d6518cfe8a5adb7f50"}, "briefly": {:hex, :briefly, "0.5.1", "ee10d48da7f79ed2aebdc3e536d5f9a0c3e36ff76c0ad0d4254653a152b13a8a", [:mix], [], "hexpm", "bd684aa92ad8b7b4e0d92c31200993c4bc1469fc68cd6d5f15144041bd15cb57"}, diff --git a/scripts/philomena.sh b/scripts/philomena.sh index d14c3dec1..5c3a81075 100755 --- a/scripts/philomena.sh +++ b/scripts/philomena.sh @@ -101,6 +101,43 @@ function init { "$(dirname "${BASH_SOURCE[0]}")/init.sh" } +# Update the `queries.json` test snapshots after the implementation changes. +function update_search_syntax_tests { + # TODO: refactor this after the devcontainer PR is merged: + # https://github.com/philomena-dev/philomena/pull/528 + # shellcheck disable=SC2016 + test=' + set -euo pipefail + + . scripts/lib.sh + + export MIX_ENV=test + + step mix deps.get + + if step createdb -h postgres -U postgres philomena_test; then + step mix ecto.create + step mix ecto.load + elif [[ "$(mix ecto.migrations)" == *" down "* ]]; then + step mix ecto.migrate --all + fi + + step mix test test/philomena/images/query_test.exs + ' + + ASSERT_VALUE_ACCEPT_DIFFS=y step docker compose run \ + --remove-orphans \ + app bash -c "$test" + + # See the git diff for the updated snapshot in vscode if it's available. + if command -v code &>/dev/null; then + step git difftool \ + --no-prompt \ + --extcmd "code --wait --diff" \ + -- test/philomena/images/search-syntax.json + fi +} + subcommand="${1:-}" shift || true @@ -118,6 +155,7 @@ case "$subcommand" in # Shortcut for `philomena exec docker compose` compose) docker compose "$@" ;; + update-search-syntax-tests) update_search_syntax_tests "$@" ;; *) die "See the available sub-commands in ${BASH_SOURCE[0]}" ;; diff --git a/test/philomena/images/query_test.exs b/test/philomena/images/query_test.exs new file mode 100644 index 000000000..67d8ef113 --- /dev/null +++ b/test/philomena/images/query_test.exs @@ -0,0 +1,80 @@ +defmodule Philomena.Images.QueryTest do + alias Philomena.Labeled + alias Philomena.Users.User + alias Philomena.Filters.Filter + import Philomena.FiltersFixtures + import Philomena.TagsFixtures + + use Philomena.SearchSyntaxCase + + defp make_user(attrs) do + %User{ + # Pretend that all users have the same ID. This doesn't influence the parser + # logic because it doesn't load the users from the DB. + id: 1, + watched_tag_ids: [], + watched_images_query_str: "", + watched_images_exclude_str: "", + no_spoilered_in_watched: false, + current_filter: %Filter{spoilered_tag_ids: [], spoilered_complex_str: ""} + } + |> Map.merge(attrs) + end + + test "search syntax" do + users = [ + Labeled.new(:anon, nil), + Labeled.new(:user, make_user(%{role: "user"})), + Labeled.new(:assistant, make_user(%{role: "assistant"})), + Labeled.new(:moderator, make_user(%{role: "moderator"})), + Labeled.new(:admin, make_user(%{role: "admin"})) + ] + + for id <- 10..14 do + tag_fixture(%{id: id, name: "tag#{id}"}) + end + + system_filter = + system_filter_fixture(%{ + id: 100, + name: "System Filter", + spoilered_tag_list: "tag10,tag11", + hidden_tag_list: "tag12,tag13", + hidden_complex_str: "truly AND complex" + }) + + assert_search_syntax(%{ + compile: &Philomena.Images.Query.compile/2, + snapshot: "#{__DIR__}/search-syntax.json", + contexts: %{ + user: users, + watch: [true, false], + filter: [true, false] + }, + test_cases: [ + wildcard: [ + "*", + "artist:*", + "artist:artist1" + ], + operators: [ + "tag1 OR tag2", + "tag1 AND tag2", + "tag1 AND (tag2 OR tag3)" + ], + authenticated_user: [ + "my:faves", + "my:watched" + ], + system_filter: [ + "filter_id:#{system_filter.id}" + ], + invalid_filters: [ + "filter_id:invalid_id", + # non-existent filter + "filter_id:9999" + ] + ] + }) + end +end diff --git a/test/philomena/images/search-syntax.json b/test/philomena/images/search-syntax.json new file mode 100644 index 000000000..3bc02fabb --- /dev/null +++ b/test/philomena/images/search-syntax.json @@ -0,0 +1,259 @@ +{ + "wildcard": [ + { + "philomena": "*", + "opensearch": { + "match_all": {} + } + }, + { + "philomena": "artist:*", + "opensearch": { + "wildcard": { + "namespaced_tags.name": "artist:*" + } + } + }, + { + "philomena": "artist:artist1", + "opensearch": { + "term": { + "namespaced_tags.name": "artist:artist1" + } + } + } + ], + "operators": [ + { + "philomena": "tag1 OR tag2", + "opensearch": { + "bool": { + "should": [ + { + "term": { + "namespaced_tags.name": "tag1" + } + }, + { + "term": { + "namespaced_tags.name": "tag2" + } + } + ] + } + } + }, + { + "philomena": "tag1 AND tag2", + "opensearch": { + "bool": { + "must": [ + { + "term": { + "namespaced_tags.name": "tag1" + } + }, + { + "term": { + "namespaced_tags.name": "tag2" + } + } + ] + } + } + }, + { + "philomena": "tag1 AND (tag2 OR tag3)", + "opensearch": { + "bool": { + "must": [ + { + "term": { + "namespaced_tags.name": "tag1" + } + }, + { + "bool": { + "should": [ + { + "term": { + "namespaced_tags.name": "tag2" + } + }, + { + "term": { + "namespaced_tags.name": "tag3" + } + } + ] + } + } + ] + } + } + } + ], + "authenticated_user": [ + { + "contexts": [ + { + "user": ["admin", "assistant", "moderator", "user"] + } + ], + "philomena": "my:faves", + "opensearch": { + "term": { + "favourited_by_user_ids": 1 + } + } + }, + { + "contexts": [ + { + "user": ["anon"] + } + ], + "philomena": "my:faves", + "opensearch": { + "term": { + "namespaced_tags.name": "my:faves" + } + } + }, + { + "contexts": [ + { + "user": ["admin", "assistant", "moderator", "user"], + "watch": [false] + } + ], + "philomena": "my:watched", + "opensearch": { + "bool": { + "should": [ + { + "terms": { + "tag_ids": [] + } + }, + { + "match_none": {} + } + ], + "must_not": [ + { + "match_none": {} + } + ] + } + } + }, + { + "contexts": [ + { + "user": ["anon"] + } + ], + "philomena": "my:watched", + "opensearch": { + "term": { + "namespaced_tags.name": "my:watched" + } + } + }, + { + "contexts": [ + { + "user": ["admin", "assistant", "moderator", "user"], + "watch": [true] + } + ], + "philomena": "my:watched", + "opensearch": "Error: Recursive watchlists are not allowed." + } + ], + "system_filter": [ + { + "contexts": [ + { + "filter": [false] + } + ], + "philomena": "filter_id:100", + "opensearch": { + "bool": { + "must_not": [ + { + "terms": { + "tag_ids": [12, 13] + } + }, + { + "bool": { + "must": [ + { + "term": { + "namespaced_tags.name": "truly" + } + }, + { + "term": { + "namespaced_tags.name": "complex" + } + } + ] + } + } + ] + } + } + }, + { + "contexts": [ + { + "filter": [true] + } + ], + "philomena": "filter_id:100", + "opensearch": "Error: Filter queries inside filters are not allowed." + } + ], + "invalid_filters": [ + { + "contexts": [ + { + "filter": [true] + } + ], + "philomena": "filter_id:invalid_id", + "opensearch": "Error: Filter queries inside filters are not allowed." + }, + { + "contexts": [ + { + "filter": [false] + } + ], + "philomena": "filter_id:invalid_id", + "opensearch": "Error: Invalid filter `invalid_id`." + }, + { + "contexts": [ + { + "filter": [true] + } + ], + "philomena": "filter_id:9999", + "opensearch": "Error: Filter queries inside filters are not allowed." + }, + { + "contexts": [ + { + "filter": [false] + } + ], + "philomena": "filter_id:9999", + "opensearch": "Error: Invalid filter `9999`." + } + ] +} diff --git a/test/support/fixtures/filters_fixtures.ex b/test/support/fixtures/filters_fixtures.ex new file mode 100644 index 000000000..810d0ec95 --- /dev/null +++ b/test/support/fixtures/filters_fixtures.ex @@ -0,0 +1,18 @@ +defmodule Philomena.FiltersFixtures do + @moduledoc """ + This module defines test helpers for creating entities via the + `Philomena.Filters` context. + """ + + alias Philomena.Filters.Filter + alias Philomena.Repo + + def system_filter_fixture(attrs \\ %{}) do + {:ok, filter} = + %Filter{id: attrs.id, system: true} + |> Filter.changeset(attrs) + |> Repo.insert() + + filter + end +end diff --git a/test/support/fixtures/tags_fixtures.ex b/test/support/fixtures/tags_fixtures.ex new file mode 100644 index 000000000..e25935a3c --- /dev/null +++ b/test/support/fixtures/tags_fixtures.ex @@ -0,0 +1,18 @@ +defmodule Philomena.TagsFixtures do + @moduledoc """ + This module defines test helpers for creating entities via the + `Philomena.Tags` context. + """ + + alias Philomena.Tags.Tag + alias Philomena.Repo + + def tag_fixture(attrs \\ %{}) do + {:ok, tag} = + %Tag{id: attrs.id} + |> Tag.creation_changeset(attrs) + |> Repo.insert() + + tag + end +end diff --git a/test/support/fixtures/users_fixtures.ex b/test/support/fixtures/users_fixtures.ex index 76a2ff917..5be0bc32f 100644 --- a/test/support/fixtures/users_fixtures.ex +++ b/test/support/fixtures/users_fixtures.ex @@ -5,6 +5,7 @@ defmodule Philomena.UsersFixtures do """ alias Philomena.Users + alias Philomena.Users.User alias Philomena.Repo def unique_user_email, do: "user#{System.unique_integer()}@example.com" @@ -13,6 +14,15 @@ defmodule Philomena.UsersFixtures do def user_fixture(attrs \\ %{}) do email = unique_user_email() + {role, attrs} = Map.pop(attrs, :role) + role = role || "user" + + {confirmed, attrs} = Map.pop(attrs, :confirmed) + confirmed = confirmed || role != "user" + + {locked, attrs} = Map.pop(attrs, :locked) + locked = locked || false + {:ok, user} = attrs |> Enum.into(%{ @@ -22,19 +32,41 @@ defmodule Philomena.UsersFixtures do }) |> Users.register_user() - user + updates = + [ + if role != "user" do + fn user -> + user + |> Repo.preload(:roles) + |> User.update_changeset(%{role: role}, []) + end + end, + if(confirmed, do: &User.confirm_changeset/1), + if(locked, do: &User.lock_changeset/1) + ] + |> Enum.reject(&is_nil/1) + + case updates do + [] -> + user + + _ -> + updates + |> Enum.reduce(user, fn update, user -> update.(user) end) + |> Repo.update!() + end end def confirmed_user_fixture(attrs \\ %{}) do - user_fixture(attrs) - |> Users.User.confirm_changeset() - |> Repo.update!() + attrs + |> Map.put(:confirmed, true) + |> user_fixture() end def locked_user_fixture(attrs \\ %{}) do - user_fixture(attrs) - |> Users.User.lock_changeset() - |> Repo.update!() + attrs + |> Map.put(:locked, true) + |> user_fixture() end def extract_user_token(fun) do diff --git a/test/support/labeled.ex b/test/support/labeled.ex new file mode 100644 index 000000000..d428c12a7 --- /dev/null +++ b/test/support/labeled.ex @@ -0,0 +1,20 @@ +defmodule Philomena.Labeled do + # Defines a `label, value` pair. The callers can use the `label` to render + # the value in a user-friendly way instead of using the `value` directly. + # The `prefer*` methods can be used to extract the label/value out of the + # value in case if it is a `Labeled` struct instance. + @enforce_keys [:label, :value] + defstruct [:label, :value] + @type t(v) :: %__MODULE__{label: String.t(), value: v} + + @spec new(String.t(), a) :: t(a) when a: any() + def new(label, value) do + %__MODULE__{label: label, value: value} + end + + def prefer_label(%__MODULE__{label: label}), do: label + def prefer_label(unlabeled), do: unlabeled + + def prefer_value(%__MODULE__{value: value}), do: value + def prefer_value(unlabeled), do: unlabeled +end diff --git a/test/support/search_syntax_case.ex b/test/support/search_syntax_case.ex new file mode 100644 index 000000000..796a61e79 --- /dev/null +++ b/test/support/search_syntax_case.ex @@ -0,0 +1,178 @@ +defmodule Philomena.SearchSyntaxCase do + @moduledoc """ + This module defines the setup for testing the Philomena Search Syntax parser. + """ + + alias PhilomenaQuery.Parse.Parser + alias Philomena.Labeled + import AssertValue + + use ExUnit.CaseTemplate + + using do + quote do + alias Philomena.Labeled + import Philomena.SearchSyntaxCase, only: [assert_search_syntax: 1] + end + end + + setup tags do + pid = Ecto.Adapters.SQL.Sandbox.start_owner!(Philomena.Repo, shared: not tags[:async]) + on_exit(fn -> Ecto.Adapters.SQL.Sandbox.stop_owner(pid) end) + :ok + end + + # Context combinations multimap. The number of keys in this map should be kept + # small, because the test calculates a multi-cartesian product of values + # between the keys of this map. + @type contexts_schema :: %{String.t() => [Labeled.t(any()) | any()]} + + @type search_syntax_test :: %{ + # The so-called "system under test". This function accepts a Philomena + # Search Syntax string, a context and returns the compiled OpenSearch + # query. + compile: (String.t(), keyword([any()]) -> Parser.result()), + + # Defines the combinations of contexts to test with. + contexts: contexts_schema(), + + # Path to the file where to store the snapshot of the test results. + snapshot: String.t(), + + # The test cases with input Philomena Search Syntax strings arbitrarily grouped for + # readability. + test_cases: keyword([String.t()]) + } + + @spec assert_search_syntax(search_syntax_test()) :: :ok + def assert_search_syntax(test) do + actual = + test.test_cases + |> map_values(fn inputs -> + inputs + |> Enum.map(&compile_input(test, &1)) + |> List.flatten() + end) + |> Jason.OrderedObject.new() + |> Jason.encode!(pretty: true) + + # Elixir's `System.cmd` API doesn't support passing custom payload via stdin + # for a command. As a simple workaround we use a bash wrapper that translates + # a CLI parameter into the stdin for `prettier`. An alternative way to do + # that could be with the Port API, but bash solution is a bit simpler: + # https://hexdocs.pm/elixir/1.18.3/Port.html#module-example + {actual, 0} = + System.cmd( + "bash", + [ + "-c", + "echo \"$1\" | npx prettier --stdin-filepath \"$2\" --parser json", + "--", + actual, + test.snapshot + ] + ) + + assert_value(actual == File.read!(test.snapshot)) + :ok + end + + @spec compile_input(search_syntax_test(), String.t()) :: [map()] + defp compile_input(test, input) do + test.contexts + |> multimap_cartesian_product() + |> Enum.group_by(fn ctx -> + ctx = map_values(ctx, &Labeled.prefer_value/1) + + case test.compile.(input, ctx) do + {:ok, output} -> output + {:error, error} -> "Error: #{error}" + end + end) + |> Enum.map(fn {output, contexts} -> + contexts = + contexts + |> Enum.map(fn ctx -> map_values(ctx, &Labeled.prefer_label/1) |> Map.new() end) + + contexts = + case normalize_contexts(test.contexts, contexts) do + [context] when map_size(context) == 0 -> [] + contexts -> [contexts: contexts] + end + + Jason.OrderedObject.new(contexts ++ [philomena: input, opensearch: output]) + end) + end + + @spec map_values([{k, v}], (v -> new_v)) :: [{k, new_v}] + when k: any(), v: any(), new_v: any() + defp map_values(key_values, map_value) do + Enum.map(key_values, fn {key, value} -> {key, map_value.(value)} end) + end + + defp multimap_cartesian_product(map) when map_size(map) == 0, do: [%{}] + + defp multimap_cartesian_product(map) do + {key, values} = map |> Enum.at(0) + + rest = map |> Map.delete(key) + + for value <- values, + rest <- multimap_cartesian_product(rest) do + Map.put_new(rest, key, value) + end + end + + # Reduces all the combinations of contexts that produce the same output. For + # example the sequence like this: + # ```ex + # [%{ a: "a1", b: "bar1" }, %{ a: "a2", b: "bar2" }] + # ``` + # will be reduced to: + # ```ex + # [%{ a: ["a1", "a2"] }] + # ``` + # only if `bar1` and `bar2` cover the set of all possible values for `b`, and + # `a1` and `a2` don't cover the set of all possible values for `a`. + # + # In other words the value of `b` doesn't influence the output at all, and + # thus can be omitted in the normalized contexts list, and the values of `a` + # are just collected into a list in a single map instead of being several maps + # with a single value in each of them. + @spec normalize_contexts(contexts_schema(), [map()]) :: [map()] + defp normalize_contexts(schema, contexts) + + defp normalize_contexts(schema, contexts) when map_size(schema) == 0 do + contexts + end + + defp normalize_contexts(schema, contexts) do + {key, possible_values} = schema |> Enum.at(0) + + schema = schema |> Map.delete(key) + + groups = + contexts + |> Enum.group_by( + fn ctx -> ctx[key] end, + fn ctx -> Map.delete(ctx, key) end + ) + + groups + |> Map.values() + |> Enum.uniq() + |> case do + [other] when map_size(groups) == length(possible_values) -> + normalize_contexts(schema, other) + + [other] -> + values = Map.keys(groups) |> Enum.sort() + + normalize_contexts(schema, other) + |> Enum.map(&Map.merge(&1, %{key => values})) + + _ -> + normalize_contexts(schema, contexts) + end + end +end