From 5919d30754b70aa434917fa5304fc2d812eccbff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20Caba=C3=A7o?= Date: Sat, 13 Jun 2026 10:09:53 +0100 Subject: [PATCH 1/3] fix: empty array on select yields error (#82) Fixes the behaviour where empty array would be handled as a NULL meaning the user could be mislead into thinking that they were properly filtering data when they weren't. The behaviour now is: * `NULL` returns all columns * `{}` errors out so we can inform the user https://linear.app/supabase/issue/REAL-843/treat-empty-pg-changes-select-as-an-error --- bin/installcheck | 2 +- ...s_migration_0015_like_ilike_is_not_ops.sql | 218 +++++++++++++++++ test/expected/issue_40_quoted_regtype.out | 2 +- test/expected/issue_50_delete_filters.out | 68 +++++- .../expected/issue_55_null_passes_filters.out | 2 +- test/expected/test_integration_filters.out | 12 +- test/expected/test_integration_in_filter.out | 14 +- .../test_integration_in_uuid_filter.out | 9 +- ..._integration_like_ilike_is_not_filters.out | 154 ++++++++++++ test/expected/test_select_columns_empty.out | 2 +- ...st_select_columns_filter_not_in_select.out | 2 +- test/expected/test_select_columns_invalid.out | 2 +- .../test_select_columns_null_element.out | 2 +- .../test_unit_like_ilike_is_not_filters.out | 230 ++++++++++++++++++ test/sql/issue_40_quoted_regtype.sql | 2 +- test/sql/issue_50_delete_filters.sql | 41 +++- test/sql/issue_55_null_passes_filters.sql | 2 +- test/sql/test_integration_filters.sql | 2 +- test/sql/test_integration_in_filter.sql | 4 +- test/sql/test_integration_in_uuid_filter.sql | 3 +- ..._integration_like_ilike_is_not_filters.sql | 80 ++++++ ...st_select_columns_filter_not_in_select.sql | 2 +- .../test_unit_like_ilike_is_not_filters.sql | 63 +++++ 23 files changed, 881 insertions(+), 37 deletions(-) create mode 100644 sql/walrus_migration_0015_like_ilike_is_not_ops.sql create mode 100644 test/expected/test_integration_like_ilike_is_not_filters.out create mode 100644 test/expected/test_unit_like_ilike_is_not_filters.out create mode 100644 test/sql/test_integration_like_ilike_is_not_filters.sql create mode 100644 test/sql/test_unit_like_ilike_is_not_filters.sql diff --git a/bin/installcheck b/bin/installcheck index 7496065..4855f17 100755 --- a/bin/installcheck +++ b/bin/installcheck @@ -41,7 +41,7 @@ REGRESS="${PGXS}/../test/regress/pg_regress" TESTS=$(ls ${TESTDIR}/sql | sed -e 's/\..*$//' | sort ) # Execute the test fixtures -psql -v ON_ERROR_STOP=1 -f sql/setup.sql -f sql/walrus--0.1.sql -f sql/walrus_migration_0001*.sql -f sql/walrus_migration_0002*.sql -f sql/walrus_migration_0003*.sql -f sql/walrus_migration_0004*.sql -f sql/walrus_migration_0005*.sql -f sql/walrus_migration_0006*.sql -f sql/walrus_migration_0007*.sql -f sql/walrus_migration_0008*.sql -f sql/walrus_migration_0009*.sql -f sql/walrus_migration_0010*.sql -f sql/walrus_migration_0011*.sql -f sql/walrus_migration_0012*.sql -f sql/walrus_migration_0013*.sql -f sql/walrus_migration_0014*.sql -f test/fixtures.sql -d contrib_regression +psql -v ON_ERROR_STOP=1 -f sql/setup.sql -f sql/walrus--0.1.sql -f sql/walrus_migration_0001*.sql -f sql/walrus_migration_0002*.sql -f sql/walrus_migration_0003*.sql -f sql/walrus_migration_0004*.sql -f sql/walrus_migration_0005*.sql -f sql/walrus_migration_0006*.sql -f sql/walrus_migration_0007*.sql -f sql/walrus_migration_0008*.sql -f sql/walrus_migration_0009*.sql -f sql/walrus_migration_0010*.sql -f sql/walrus_migration_0011*.sql -f sql/walrus_migration_0012*.sql -f sql/walrus_migration_0013*.sql -f sql/walrus_migration_0014*.sql -f sql/walrus_migration_0015*.sql -f test/fixtures.sql -d contrib_regression # Run tests ${REGRESS} --use-existing --dbname=contrib_regression --inputdir=${TESTDIR} ${TESTS} diff --git a/sql/walrus_migration_0015_like_ilike_is_not_ops.sql b/sql/walrus_migration_0015_like_ilike_is_not_ops.sql new file mode 100644 index 0000000..b976d11 --- /dev/null +++ b/sql/walrus_migration_0015_like_ilike_is_not_ops.sql @@ -0,0 +1,218 @@ +-- Fix 1: Drop old 4-arg overload and its dependent before creating the 5-arg version +drop function if exists realtime.is_visible_through_filters(realtime.wal_column[], realtime.user_defined_filter[]); +drop function if exists realtime.check_equality_op(realtime.equality_op, regtype, text, text); + +alter type realtime.equality_op add value 'like'; +alter type realtime.equality_op add value 'ilike'; +alter type realtime.equality_op add value 'is'; +alter type realtime.equality_op add value 'match'; +alter type realtime.equality_op add value 'imatch'; +alter type realtime.equality_op add value 'isdistinct'; + +alter type realtime.user_defined_filter add attribute negate boolean; + +drop function if exists realtime.check_equality_op(realtime.equality_op, regtype, text, text); + +create or replace function realtime.check_equality_op( + op realtime.equality_op, + type_ regtype, + val_1 text, + val_2 text, + negate boolean DEFAULT false +) + returns bool + stable -- Fix 2: was immutable, uses EXECUTE so must be stable + language plpgsql +as $$ +declare + op_symbol text; + res boolean; +begin + -- IS DISTINCT FROM / IS NOT DISTINCT FROM: infix, both sides typed literals + if op = 'isdistinct' then + execute format( + 'select %L::%s %s %L::%s', + val_1, + type_::text, + case when negate then 'IS NOT DISTINCT FROM' else 'IS DISTINCT FROM' end, + val_2, + type_::text + ) into res; + return res; + end if; + + -- IS requires a keyword RHS (NULL, TRUE, FALSE, UNKNOWN), not a typed literal + if op = 'is' then + if val_2 not in ('null', 'true', 'false', 'unknown') then + raise exception 'invalid value for is filter: must be null, true, false, or unknown'; + end if; + execute format( + 'select %L::%s %s %s', + val_1, + type_::text, + case when negate then 'IS NOT' else 'IS' end, + upper(val_2) + ) into res; + return res; + end if; + + op_symbol = case + when op = 'eq' then '=' + when op = 'neq' then '!=' + when op = 'lt' then '<' + when op = 'lte' then '<=' + when op = 'gt' then '>' + when op = 'gte' then '>=' + when op = 'in' then '= any' + when op = 'like' then 'LIKE' + when op = 'ilike' then 'ILIKE' + when op = 'match' then '~' + when op = 'imatch' then '~*' + else null + end; + + if op_symbol is null then + raise exception 'unsupported equality operator: %', op::text; + end if; + + execute format( + 'select %L::%s %s (%L::%s)', + val_1, + type_::text, + op_symbol, + val_2, + case when op = 'in' then type_::text || '[]' else type_::text end + ) into res; + + return case when negate then not res else res end; +end; +$$; + + +create or replace function realtime.is_visible_through_filters(columns realtime.wal_column[], filters realtime.user_defined_filter[]) + returns bool + language sql + stable -- Fix 2: was immutable, calls stable function so must be stable +as $$ + select + filters is null + or array_length(filters, 1) is null + or coalesce( + count(col.name) = count(1) + and sum( + realtime.check_equality_op( + op:=f.op, + type_:=coalesce(col.type_oid::regtype, col.type_name::regtype), + val_1:=col.value #>> '{}', + val_2:=f.value, + negate:=coalesce(f.negate, false) + )::int + ) filter (where col.name is not null) = count(col.name), + false + ) + from + unnest(filters) f + left join unnest(columns) col + on f.column_name = col.name; +$$; + + +create or replace function realtime.subscription_check_filters() + returns trigger + language plpgsql +as $$ +declare + col_names text[] = coalesce( + array_agg(c.column_name order by c.ordinal_position), + '{}'::text[] + ) + from + information_schema.columns c + where + format('%I.%I', c.table_schema, c.table_name)::regclass = new.entity + and pg_catalog.has_column_privilege( + (new.claims ->> 'role'), + format('%I.%I', c.table_schema, c.table_name)::regclass, + c.column_name, + 'SELECT' + ); + -- Fix 3: removed unused table_col_names declaration + filter realtime.user_defined_filter; + col_type regtype; + in_val jsonb; + selected_col text; +begin + for filter in select * from unnest(new.filters) loop + if not filter.column_name = any(col_names) then + raise exception 'invalid column for filter %', filter.column_name; + end if; + + col_type = ( + select atttypid::regtype + from pg_catalog.pg_attribute + where attrelid = new.entity + and attname = filter.column_name + ); + if col_type is null then + raise exception 'failed to lookup type for column %', filter.column_name; + end if; + + if filter.op = 'in'::realtime.equality_op then + in_val = realtime.cast(filter.value, (col_type::text || '[]')::regtype); + if coalesce(jsonb_array_length(in_val), 0) > 100 then + raise exception 'too many values for `in` filter. Maximum 100'; + end if; + elsif filter.op = 'is'::realtime.equality_op then + if filter.value not in ('null', 'true', 'false', 'unknown') then + raise exception 'invalid value for is filter: must be null, true, false, or unknown'; + end if; + -- Fix 4: validate like/ilike only applies to text-compatible columns + elsif filter.op in ('like'::realtime.equality_op, 'ilike'::realtime.equality_op) then + if not exists ( + select 1 from pg_catalog.pg_operator + where oprname = '~~' and oprleft = col_type + ) then + raise exception 'operator % requires a text-compatible column type, got %', filter.op::text, col_type::text; + end if; + -- Fix 5: validate match/imatch regex patterns eagerly + elsif filter.op in ('match'::realtime.equality_op, 'imatch'::realtime.equality_op) then + begin + perform '' ~ filter.value; + exception when others then + raise exception 'invalid regular expression for % filter: %', filter.op::text, sqlerrm; + end; + else + perform realtime.cast(filter.value, col_type); + end if; + end loop; + + if new.selected_columns = '{}' then + raise exception 'selected_columns cannot be empty. Remove the select parameter to capture all columns.'; + end if; + + if new.selected_columns is not null and array_position(new.selected_columns, null::text) is not null then + raise exception 'selected_columns cannot contain null values.'; + end if; + + if new.selected_columns is not null then + for selected_col in select * from unnest(new.selected_columns) loop + if not selected_col = any(col_names) then + raise exception 'invalid column for select %', selected_col; + end if; + end loop; + end if; + + -- Fix 6: include negate in the ORDER BY for deterministic normalization + new.filters = coalesce( + array_agg(f order by f.column_name, f.op, f.value, f.negate), + '{}' + ) from unnest(new.filters) f; + + new.selected_columns = ( + select array_agg(c order by c) + from unnest(new.selected_columns) c + ); + + return new; +end; +$$; diff --git a/test/expected/issue_40_quoted_regtype.out b/test/expected/issue_40_quoted_regtype.out index b6d5f4b..ed313de 100644 --- a/test/expected/issue_40_quoted_regtype.out +++ b/test/expected/issue_40_quoted_regtype.out @@ -25,7 +25,7 @@ select 'role', 'authenticated', 'sub', seed_uuid(2)::text ), - array[('primary_color', 'eq', 'RED')::realtime.user_defined_filter]; + array[('primary_color', 'eq', 'RED', null)::realtime.user_defined_filter]; insert into public.notes(id, primary_color) values (1, 'RED'), -- matches filter diff --git a/test/expected/issue_50_delete_filters.out b/test/expected/issue_50_delete_filters.out index 404e593..dfc2bd8 100644 --- a/test/expected/issue_50_delete_filters.out +++ b/test/expected/issue_50_delete_filters.out @@ -17,7 +17,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -25,10 +25,10 @@ from (2 , 'id', 'eq', '2') ) f(id, column_name, op, value); select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(id,eq,2)"} + subscription_id | filters +--------------------------------------+-------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb,)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(id,eq,2,)"} (2 rows) ---------------------------------------------------------------------------------------- @@ -160,6 +160,64 @@ from } | | | (2 rows) +---------------------------------------------------------------------------------------- +-- Multi-filter: missing column must not allow subscription to pass -- +---------------------------------------------------------------------------------------- +alter table public.notes replica identity default; +truncate table realtime.subscription; +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(3), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(3)::text + ), + array[ + ('id', 'eq', '1', null)::realtime.user_defined_filter, + ('body', 'eq', 'bbb', null)::realtime.user_defined_filter + ]; +insert into public.notes(id, body) values (1, 'bbb'); +select clear_wal(); + clear_wal +----------- + +(1 row) + +-- Non-full replica identity DELETE: only PK in WAL, body filter cannot be evaluated +-- Expect 0 subscriptions: body='bbb' filter is unverifiable, must fail closed +delete from public.notes; +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + rec | is_rls_enabled | subscription_ids | errors +----------------------------------------------------+----------------+------------------+-------- + { +| f | {} | {} + "type": "DELETE", +| | | + "table": "notes", +| | | + "schema": "public", +| | | + "columns": [ +| | | + { +| | | + "name": "id", +| | | + "type": "int4" +| | | + }, +| | | + { +| | | + "name": "body", +| | | + "type": "text" +| | | + } +| | | + ], +| | | + "old_record": { +| | | + "id": 1 +| | | + }, +| | | + "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | + } | | | +(1 row) + drop table public.notes; select pg_drop_replication_slot('realtime'); pg_drop_replication_slot diff --git a/test/expected/issue_55_null_passes_filters.out b/test/expected/issue_55_null_passes_filters.out index ce5c540..90288c1 100644 --- a/test/expected/issue_55_null_passes_filters.out +++ b/test/expected/issue_55_null_passes_filters.out @@ -17,7 +17,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(1)::text ), - array[('page_id', 'eq', '5')::realtime.user_defined_filter]; + array[('page_id', 'eq', '5', null)::realtime.user_defined_filter]; select clear_wal(); clear_wal ----------- diff --git a/test/expected/test_integration_filters.out b/test/expected/test_integration_filters.out index c3d74b7..a2bdf21 100644 --- a/test/expected/test_integration_filters.out +++ b/test/expected/test_integration_filters.out @@ -18,7 +18,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -35,11 +35,11 @@ select clear_wal(); insert into public.notes(id, body) values (1, 'bbb'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+-------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,eq,aaaa)"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,eq,cc)"} + subscription_id | filters +--------------------------------------+--------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb,)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,eq,aaaa,)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,eq,cc,)"} (3 rows) select diff --git a/test/expected/test_integration_in_filter.out b/test/expected/test_integration_in_filter.out index f2245d1..6edfd26 100644 --- a/test/expected/test_integration_in_filter.out +++ b/test/expected/test_integration_in_filter.out @@ -18,7 +18,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -35,11 +35,11 @@ select clear_wal(); insert into public.notes(id, body) values (1, 'bbb'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+--------------------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,in,\"{aaa,bbb,ccc}\")"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,in,\"{aaa,ccc}\")"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,in,{})"} + subscription_id | filters +--------------------------------------+---------------------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,in,\"{aaa,bbb,ccc}\",)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,in,\"{aaa,ccc}\",)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,in,{},)"} (3 rows) select @@ -103,7 +103,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(6)::text ), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])::realtime.user_defined_filter]; + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], null)::realtime.user_defined_filter]; ERROR: too many values for `in` filter. Maximum 100 CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 41 at RAISE drop table public.notes; diff --git a/test/expected/test_integration_in_uuid_filter.out b/test/expected/test_integration_in_uuid_filter.out index bed1550..c327065 100644 --- a/test/expected/test_integration_in_uuid_filter.out +++ b/test/expected/test_integration_in_uuid_filter.out @@ -21,7 +21,8 @@ select array[( 'identifier', 'in', - '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}' + '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}', + null )::realtime.user_defined_filter]; select clear_wal(); clear_wal @@ -32,9 +33,9 @@ select clear_wal(); insert into public.notes(id, identifier) values (1, 'ace23052-568e-4951-acc8-fd510ec667f9'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+----------------------------------------------------------------------------------------------------- - 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(identifier,in,\"{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}\")"} + subscription_id | filters +--------------------------------------+------------------------------------------------------------------------------------------------------ + 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(identifier,in,\"{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}\",)"} (1 row) select diff --git a/test/expected/test_integration_like_ilike_is_not_filters.out b/test/expected/test_integration_like_ilike_is_not_filters.out new file mode 100644 index 0000000..a1d9fcf --- /dev/null +++ b/test/expected/test_integration_like_ilike_is_not_filters.out @@ -0,0 +1,154 @@ +select 1 from pg_create_logical_replication_slot('realtime', 'wal2json', false); + ?column? +---------- + 1 +(1 row) + +create table public.notes( + id int primary key, + body text, + nullable_body text +); +alter table public.notes replica identity full; +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(id), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(id)::text + ), + array[(column_name, op, value, negate)::realtime.user_defined_filter] +from + ( + values + -- like: matches 'hello world' → visible + (1, 'body', 'like', '%world%', false), + -- like: does not match → not visible + (2, 'body', 'like', '%xyz%', false), + -- ilike: case-insensitive match → visible + (3, 'body', 'ilike', '%WORLD%', false), + -- NOT LIKE: row does not match pattern → visible + (4, 'body', 'like', '%xyz%', true), + -- NOT LIKE: row matches pattern → not visible + (5, 'body', 'like', '%world%', true), + -- NOT IN: 'hello world' outside the list → visible + (6, 'body', 'in', '{foo,bar}', true), + -- NOT IN: 'hello world' inside the list → not visible + (7, 'body', 'in', '{hello world,other}', true), + -- is null on nullable_body (null row) → visible + (8, 'nullable_body', 'is', 'null', false), + -- is not null on nullable_body (null row)→ not visible + (9, 'nullable_body', 'is', 'null', true) + ) f(id, column_name, op, value, negate); +select clear_wal(); + clear_wal +----------- + +(1 row) + +insert into public.notes(id, body, nullable_body) values (1, 'hello world', null); +delete from public.notes; +select subscription_id, filters from realtime.subscription order by subscription_id; + subscription_id | filters +--------------------------------------+----------------------------------------- + 0d2f6c5b-62a4-5cae-af96-780d1ff5441b | {"(nullable_body,is,null,t)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,ilike,%WORLD%,f)"} + 23407699-1647-5973-97ee-7848c18c8a4f | {"(nullable_body,is,null,f)"} + 2a5efacf-2c0a-5e59-861b-eff2079d1e2e | {"(body,in,\"{foo,bar}\",t)"} + 33b3e2e2-d91c-530c-9955-f89a94467a34 | {"(body,like,%xyz%,t)"} + 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(body,like,%world%,t)"} + 50737752-8d74-577e-9972-20ba646857af | {"(body,in,\"{hello world,other}\",t)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,like,%xyz%,f)"} + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,like,%world%,f)"} +(9 rows) + +-- Expected visible subscription_ids: 1, 3, 4, 6, 8 +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + rec | is_rls_enabled | subscription_ids | errors +----------------------------------------------------+----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------- + { +| f | {f4539ebe-c779-5788-bbc1-2421ffaa8954,11955172-4e1d-5836-925f-2bcb7a287b87,33b3e2e2-d91c-530c-9955-f89a94467a34,2a5efacf-2c0a-5e59-861b-eff2079d1e2e,23407699-1647-5973-97ee-7848c18c8a4f} | {} + "type": "INSERT", +| | | + "table": "notes", +| | | + "record": { +| | | + "id": 1, +| | | + "body": "hello world", +| | | + "nullable_body": null +| | | + }, +| | | + "schema": "public", +| | | + "columns": [ +| | | + { +| | | + "name": "id", +| | | + "type": "int4" +| | | + }, +| | | + { +| | | + "name": "body", +| | | + "type": "text" +| | | + }, +| | | + { +| | | + "name": "nullable_body", +| | | + "type": "text" +| | | + } +| | | + ], +| | | + "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | + } | | | + { +| f | {f4539ebe-c779-5788-bbc1-2421ffaa8954,11955172-4e1d-5836-925f-2bcb7a287b87,33b3e2e2-d91c-530c-9955-f89a94467a34,2a5efacf-2c0a-5e59-861b-eff2079d1e2e,23407699-1647-5973-97ee-7848c18c8a4f} | {} + "type": "DELETE", +| | | + "table": "notes", +| | | + "schema": "public", +| | | + "columns": [ +| | | + { +| | | + "name": "id", +| | | + "type": "int4" +| | | + }, +| | | + { +| | | + "name": "body", +| | | + "type": "text" +| | | + }, +| | | + { +| | | + "name": "nullable_body", +| | | + "type": "text" +| | | + } +| | | + ], +| | | + "old_record": { +| | | + "id": 1, +| | | + "body": "hello world", +| | | + "nullable_body": null +| | | + }, +| | | + "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | + } | | | +(2 rows) + +-- Confirm is with invalid value is rejected at subscription time +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(10), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(10)::text), + array[('body', 'is', 'invalid_value', false)::realtime.user_defined_filter]; +ERROR: invalid value for is filter: must be null, true, false, or unknown +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 45 at RAISE +-- Confirm in with more than 100 entries is rejected +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(11), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(11)::text), + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], true)::realtime.user_defined_filter]; +ERROR: too many values for `in` filter. Maximum 100 +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 41 at RAISE +drop table public.notes; +select pg_drop_replication_slot('realtime'); + pg_drop_replication_slot +-------------------------- + +(1 row) + +truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_empty.out b/test/expected/test_select_columns_empty.out index 20b8cf0..e5fd42d 100644 --- a/test/expected/test_select_columns_empty.out +++ b/test/expected/test_select_columns_empty.out @@ -16,6 +16,6 @@ select ), '{}'::text[]; ERROR: selected_columns cannot be empty. Remove the select parameter to capture all columns. -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 52 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 68 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_filter_not_in_select.out b/test/expected/test_select_columns_filter_not_in_select.out index 11ba40c..0aaa1a8 100644 --- a/test/expected/test_select_columns_filter_not_in_select.out +++ b/test/expected/test_select_columns_filter_not_in_select.out @@ -26,7 +26,7 @@ select 'sub', seed_uuid(1)::text ), array['body'], - array[('extra', 'eq', 'match')::realtime.user_defined_filter]; + array[('extra', 'eq', 'match', null)::realtime.user_defined_filter]; -- Matching row: filter on extra matches, but extra not in output select clear_wal(); clear_wal diff --git a/test/expected/test_select_columns_invalid.out b/test/expected/test_select_columns_invalid.out index 5a3901c..750a26c 100644 --- a/test/expected/test_select_columns_invalid.out +++ b/test/expected/test_select_columns_invalid.out @@ -16,6 +16,6 @@ select ), array['nonexistent_column']; ERROR: invalid column for select nonexistent_column -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 64 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 78 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_null_element.out b/test/expected/test_select_columns_null_element.out index 42b0357..fea7278 100644 --- a/test/expected/test_select_columns_null_element.out +++ b/test/expected/test_select_columns_null_element.out @@ -16,6 +16,6 @@ select ), array[null]::text[]; ERROR: selected_columns cannot contain null values. -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 57 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 72 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_unit_like_ilike_is_not_filters.out b/test/expected/test_unit_like_ilike_is_not_filters.out new file mode 100644 index 0000000..82acf01 --- /dev/null +++ b/test/expected/test_unit_like_ilike_is_not_filters.out @@ -0,0 +1,230 @@ +-- like (negate=false) +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', false); + check_equality_op +------------------- + f +(1 row) + +-- like negated (NOT LIKE) +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', true); + check_equality_op +------------------- + f +(1 row) + +-- ilike (negate=false) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', false); + check_equality_op +------------------- + f +(1 row) + +-- ilike negated (NOT ILIKE) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', true); + check_equality_op +------------------- + f +(1 row) + +-- in (negate=false) +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', false); + check_equality_op +------------------- + f +(1 row) + +-- in negated (NOT IN) +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', true); + check_equality_op +------------------- + f +(1 row) + +-- is null +select realtime.check_equality_op('is', 'text', null, 'null', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('is', 'text', 'value', 'null', false); + check_equality_op +------------------- + f +(1 row) + +-- is not null (negate=true) +select realtime.check_equality_op('is', 'text', 'value', 'null', true); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('is', 'text', null, 'null', true); + check_equality_op +------------------- + f +(1 row) + +-- is true / false on boolean +select realtime.check_equality_op('is', 'boolean', 'true', 'true', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('is', 'boolean', 'false', 'true', false); + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('is', 'boolean', 'false', 'true', true); + check_equality_op +------------------- + t +(1 row) + +-- match (regex) +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('match', 'text', 'hello world', '^world', false); + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', true); + check_equality_op +------------------- + f +(1 row) + +-- imatch (case-insensitive regex) +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', false); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('imatch', 'text', 'Hello World', '^world', false); + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', true); + check_equality_op +------------------- + f +(1 row) + +-- isdistinct (IS DISTINCT FROM — differs from neq in NULL handling) +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'bbb', false); -- true + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'aaa', false); -- false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', false); -- true (NULL IS DISTINCT FROM 'aaa') + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', true); -- false (IS NOT DISTINCT FROM) + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('isdistinct', 'text', null, null, false); -- false (NULL IS NOT DISTINCT FROM NULL) + check_equality_op +------------------- + f +(1 row) + +-- negate works on all existing operators +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa', true); -- NOT eq → false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('neq', 'text', 'aaa', 'bbb', true); -- NOT neq → false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('lt', 'bigint', '1', '2', true); -- NOT lt → false + check_equality_op +------------------- + f +(1 row) + +select realtime.check_equality_op('gte', 'bigint', '5', '3', true); -- NOT gte → false + check_equality_op +------------------- + f +(1 row) + +-- negate defaults to false (existing callers unaffected) +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa'); + check_equality_op +------------------- + t +(1 row) + +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}'); + check_equality_op +------------------- + t +(1 row) + diff --git a/test/sql/issue_40_quoted_regtype.sql b/test/sql/issue_40_quoted_regtype.sql index 02679fd..8a833b7 100644 --- a/test/sql/issue_40_quoted_regtype.sql +++ b/test/sql/issue_40_quoted_regtype.sql @@ -29,7 +29,7 @@ select 'role', 'authenticated', 'sub', seed_uuid(2)::text ), - array[('primary_color', 'eq', 'RED')::realtime.user_defined_filter]; + array[('primary_color', 'eq', 'RED', null)::realtime.user_defined_filter]; insert into public.notes(id, primary_color) values diff --git a/test/sql/issue_50_delete_filters.sql b/test/sql/issue_50_delete_filters.sql index 77d4253..daea76b 100644 --- a/test/sql/issue_50_delete_filters.sql +++ b/test/sql/issue_50_delete_filters.sql @@ -14,7 +14,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -64,6 +64,45 @@ select clear_wal(); delete from public.notes; +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + + +---------------------------------------------------------------------------------------- +-- Multi-filter: missing column must not allow subscription to pass -- +---------------------------------------------------------------------------------------- + +alter table public.notes replica identity default; + +truncate table realtime.subscription; + +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(3), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(3)::text + ), + array[ + ('id', 'eq', '1', null)::realtime.user_defined_filter, + ('body', 'eq', 'bbb', null)::realtime.user_defined_filter + ]; + +insert into public.notes(id, body) values (1, 'bbb'); + +select clear_wal(); + +-- Non-full replica identity DELETE: only PK in WAL, body filter cannot be evaluated +-- Expect 0 subscriptions: body='bbb' filter is unverifiable, must fail closed +delete from public.notes; + select rec, is_rls_enabled, diff --git a/test/sql/issue_55_null_passes_filters.sql b/test/sql/issue_55_null_passes_filters.sql index adf2802..e3bbfe2 100644 --- a/test/sql/issue_55_null_passes_filters.sql +++ b/test/sql/issue_55_null_passes_filters.sql @@ -14,7 +14,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(1)::text ), - array[('page_id', 'eq', '5')::realtime.user_defined_filter]; + array[('page_id', 'eq', '5', null)::realtime.user_defined_filter]; select clear_wal(); diff --git a/test/sql/test_integration_filters.sql b/test/sql/test_integration_filters.sql index 63cbc83..2e37725 100644 --- a/test/sql/test_integration_filters.sql +++ b/test/sql/test_integration_filters.sql @@ -16,7 +16,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values diff --git a/test/sql/test_integration_in_filter.sql b/test/sql/test_integration_in_filter.sql index 2608099..dab3eed 100644 --- a/test/sql/test_integration_in_filter.sql +++ b/test/sql/test_integration_in_filter.sql @@ -16,7 +16,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value)::realtime.user_defined_filter] + array[(column_name, op, value, null)::realtime.user_defined_filter] from ( values @@ -52,7 +52,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(6)::text ), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])::realtime.user_defined_filter]; + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], null)::realtime.user_defined_filter]; drop table public.notes; select pg_drop_replication_slot('realtime'); diff --git a/test/sql/test_integration_in_uuid_filter.sql b/test/sql/test_integration_in_uuid_filter.sql index 5ac3385..9fb538e 100644 --- a/test/sql/test_integration_in_uuid_filter.sql +++ b/test/sql/test_integration_in_uuid_filter.sql @@ -19,7 +19,8 @@ select array[( 'identifier', 'in', - '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}' + '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}', + null )::realtime.user_defined_filter]; diff --git a/test/sql/test_integration_like_ilike_is_not_filters.sql b/test/sql/test_integration_like_ilike_is_not_filters.sql new file mode 100644 index 0000000..249c04c --- /dev/null +++ b/test/sql/test_integration_like_ilike_is_not_filters.sql @@ -0,0 +1,80 @@ +select 1 from pg_create_logical_replication_slot('realtime', 'wal2json', false); + +create table public.notes( + id int primary key, + body text, + nullable_body text +); + +alter table public.notes replica identity full; + +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(id), + 'public.notes', + jsonb_build_object( + 'role', 'authenticated', + 'email', 'example@example.com', + 'sub', seed_uuid(id)::text + ), + array[(column_name, op, value, negate)::realtime.user_defined_filter] +from + ( + values + -- like: matches 'hello world' → visible + (1, 'body', 'like', '%world%', false), + -- like: does not match → not visible + (2, 'body', 'like', '%xyz%', false), + -- ilike: case-insensitive match → visible + (3, 'body', 'ilike', '%WORLD%', false), + -- NOT LIKE: row does not match pattern → visible + (4, 'body', 'like', '%xyz%', true), + -- NOT LIKE: row matches pattern → not visible + (5, 'body', 'like', '%world%', true), + -- NOT IN: 'hello world' outside the list → visible + (6, 'body', 'in', '{foo,bar}', true), + -- NOT IN: 'hello world' inside the list → not visible + (7, 'body', 'in', '{hello world,other}', true), + -- is null on nullable_body (null row) → visible + (8, 'nullable_body', 'is', 'null', false), + -- is not null on nullable_body (null row)→ not visible + (9, 'nullable_body', 'is', 'null', true) + ) f(id, column_name, op, value, negate); + + +select clear_wal(); +insert into public.notes(id, body, nullable_body) values (1, 'hello world', null); + +delete from public.notes; + +select subscription_id, filters from realtime.subscription order by subscription_id; + +-- Expected visible subscription_ids: 1, 3, 4, 6, 8 +select + rec, + is_rls_enabled, + subscription_ids, + errors +from + walrus; + + +-- Confirm is with invalid value is rejected at subscription time +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(10), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(10)::text), + array[('body', 'is', 'invalid_value', false)::realtime.user_defined_filter]; + +-- Confirm in with more than 100 entries is rejected +insert into realtime.subscription(subscription_id, entity, claims, filters) +select + seed_uuid(11), + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(11)::text), + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], true)::realtime.user_defined_filter]; + +drop table public.notes; +select pg_drop_replication_slot('realtime'); +truncate table realtime.subscription; diff --git a/test/sql/test_select_columns_filter_not_in_select.sql b/test/sql/test_select_columns_filter_not_in_select.sql index b927c78..77b8f28 100644 --- a/test/sql/test_select_columns_filter_not_in_select.sql +++ b/test/sql/test_select_columns_filter_not_in_select.sql @@ -24,7 +24,7 @@ select 'sub', seed_uuid(1)::text ), array['body'], - array[('extra', 'eq', 'match')::realtime.user_defined_filter]; + array[('extra', 'eq', 'match', null)::realtime.user_defined_filter]; -- Matching row: filter on extra matches, but extra not in output select clear_wal(); diff --git a/test/sql/test_unit_like_ilike_is_not_filters.sql b/test/sql/test_unit_like_ilike_is_not_filters.sql new file mode 100644 index 0000000..56ea901 --- /dev/null +++ b/test/sql/test_unit_like_ilike_is_not_filters.sql @@ -0,0 +1,63 @@ +-- like (negate=false) +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', false); +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', false); + +-- like negated (NOT LIKE) +select realtime.check_equality_op('like', 'text', 'hello world', '%xyz%', true); +select realtime.check_equality_op('like', 'text', 'hello world', '%world%', true); + +-- ilike (negate=false) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', false); +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', false); + +-- ilike negated (NOT ILIKE) +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%xyz%', true); +select realtime.check_equality_op('ilike', 'text', 'Hello World', '%world%', true); + +-- in (negate=false) +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', false); +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', false); + +-- in negated (NOT IN) +select realtime.check_equality_op('in', 'bigint', '4', '{1,2,3}', true); +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}', true); + +-- is null +select realtime.check_equality_op('is', 'text', null, 'null', false); +select realtime.check_equality_op('is', 'text', 'value', 'null', false); + +-- is not null (negate=true) +select realtime.check_equality_op('is', 'text', 'value', 'null', true); +select realtime.check_equality_op('is', 'text', null, 'null', true); + +-- is true / false on boolean +select realtime.check_equality_op('is', 'boolean', 'true', 'true', false); +select realtime.check_equality_op('is', 'boolean', 'false', 'true', false); +select realtime.check_equality_op('is', 'boolean', 'false', 'true', true); + +-- match (regex) +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', false); +select realtime.check_equality_op('match', 'text', 'hello world', '^world', false); +select realtime.check_equality_op('match', 'text', 'hello world', 'hel+o', true); + +-- imatch (case-insensitive regex) +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', false); +select realtime.check_equality_op('imatch', 'text', 'Hello World', '^world', false); +select realtime.check_equality_op('imatch', 'text', 'Hello World', 'hel+o', true); + +-- isdistinct (IS DISTINCT FROM — differs from neq in NULL handling) +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'bbb', false); -- true +select realtime.check_equality_op('isdistinct', 'text', 'aaa', 'aaa', false); -- false +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', false); -- true (NULL IS DISTINCT FROM 'aaa') +select realtime.check_equality_op('isdistinct', 'text', null, 'aaa', true); -- false (IS NOT DISTINCT FROM) +select realtime.check_equality_op('isdistinct', 'text', null, null, false); -- false (NULL IS NOT DISTINCT FROM NULL) + +-- negate works on all existing operators +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa', true); -- NOT eq → false +select realtime.check_equality_op('neq', 'text', 'aaa', 'bbb', true); -- NOT neq → false +select realtime.check_equality_op('lt', 'bigint', '1', '2', true); -- NOT lt → false +select realtime.check_equality_op('gte', 'bigint', '5', '3', true); -- NOT gte → false + +-- negate defaults to false (existing callers unaffected) +select realtime.check_equality_op('eq', 'text', 'aaa', 'aaa'); +select realtime.check_equality_op('in', 'bigint', '2', '{1,2,3}'); From 5ccc24b8176886f6cd73274061b58bfb5963ea8c Mon Sep 17 00:00:00 2001 From: Filipe Cabaco Date: Mon, 15 Jun 2026 15:07:42 +0100 Subject: [PATCH 2/3] update the readme --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d2583d5..5901ade 100644 --- a/README.md +++ b/README.md @@ -41,26 +41,28 @@ where `realtime.user_defined_filter` is create type realtime.user_defined_filter as ( column_name text, op realtime.equality_op, - value text + value text, + negate boolean ); ``` and `realtime.equality_op`s are a subset of [postgrest ops](https://postgrest.org/en/v4.1/api.html#horizontal-filtering-rows). Specifically: ```sql create type realtime.equality_op as enum( - 'eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'in' + 'eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'in', + 'like', 'ilike', 'is', 'match', 'imatch', 'isdistinct' ); ``` For example, to subscribe to a table named `public.notes` where the `id` is `6` as the `authenticated` role: ```sql insert into realtime.subscription(subscription_id, entity, filters, claims) -values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6')], '{"role", "authenticated"}'); +values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6', false)::realtime.user_defined_filter], '{"role", "authenticated"}'); ``` To subscribe to `INSERT`s only on a table named `public.notes` where the `id` is `6` as the `authenticated` role: ```sql insert into realtime.subscription(subscription_id, entity, filters, claims, action_filter) -values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6')], '{"role", "authenticated"}', 'INSERT'); +values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6', false)::realtime.user_defined_filter], '{"role", "authenticated"}', 'INSERT'); ``` To subscribe to `public.notes` and receive only the `id` and `title` columns (plus primary keys, which are always included): From 9dc7145e209f0a5062c4976d287ff0322c0cddd1 Mon Sep 17 00:00:00 2001 From: Filipe Cabaco Date: Mon, 15 Jun 2026 19:24:34 +0100 Subject: [PATCH 3/3] ensure compatibility, improve checks --- README.md | 43 +- ...s_migration_0015_like_ilike_is_not_ops.sql | 566 +++++++++++++++++- test/expected/issue_40_quoted_regtype.out | 2 +- test/expected/issue_50_delete_filters.out | 68 +-- .../expected/issue_55_null_passes_filters.out | 2 +- test/expected/test_integration_filters.out | 12 +- test/expected/test_integration_in_filter.out | 16 +- .../test_integration_in_uuid_filter.out | 9 +- ..._integration_like_ilike_is_not_filters.out | 313 +++++----- ...test_integration_v2_filter_composition.out | 144 +++++ test/expected/test_select_columns_empty.out | 2 +- ...st_select_columns_filter_not_in_select.out | 2 +- test/expected/test_select_columns_invalid.out | 2 +- .../test_select_columns_null_element.out | 2 +- test/sql/issue_40_quoted_regtype.sql | 2 +- test/sql/issue_50_delete_filters.sql | 41 +- test/sql/issue_55_null_passes_filters.sql | 2 +- test/sql/test_integration_filters.sql | 2 +- test/sql/test_integration_in_filter.sql | 4 +- test/sql/test_integration_in_uuid_filter.sql | 3 +- ..._integration_like_ilike_is_not_filters.sql | 199 ++++-- ...test_integration_v2_filter_composition.sql | 120 ++++ ...st_select_columns_filter_not_in_select.sql | 2 +- 23 files changed, 1186 insertions(+), 372 deletions(-) create mode 100644 test/expected/test_integration_v2_filter_composition.out create mode 100644 test/sql/test_integration_v2_filter_composition.sql diff --git a/README.md b/README.md index 5901ade..40f0faf 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,7 @@ where `realtime.user_defined_filter` is create type realtime.user_defined_filter as ( column_name text, op realtime.equality_op, - value text, - negate boolean + value text ); ``` and `realtime.equality_op`s are a subset of [postgrest ops](https://postgrest.org/en/v4.1/api.html#horizontal-filtering-rows). Specifically: @@ -56,13 +55,49 @@ create type realtime.equality_op as enum( For example, to subscribe to a table named `public.notes` where the `id` is `6` as the `authenticated` role: ```sql insert into realtime.subscription(subscription_id, entity, filters, claims) -values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6', false)::realtime.user_defined_filter], '{"role", "authenticated"}'); +values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6')], '{"role", "authenticated"}'); ``` +### Extended operators and negation (`filters_v2`) + +The operators `like`, `ilike`, `is`, `match` (`~`), `imatch` (`~*`) and `isdistinct`, +along with negation of any operator, are stored on a separate `filters_v2` +column whose composite type carries an extra `negate` boolean: +```sql +create type realtime.user_defined_filter_v2 as ( + column_name text, + op realtime.equality_op, + value text, + negate boolean +); +``` +This is an expand/contract rollout: the legacy 3-field `filters` column is left +untouched (so older Realtime server instances keep writing to it during a rolling +deploy), and the new operators are only accepted on `filters_v2`. `apply_rls` +evaluates both columns; for any given subscription one of them is always empty. + +When `negate` is `true` the operator's result is inverted (e.g. `like` → +`NOT LIKE`, `in` → `NOT IN`). To subscribe to `public.notes` where `body` does +**not** match the pattern `%draft%`: +```sql +insert into realtime.subscription(subscription_id, entity, filters_v2, claims) +values ( + '832bd278-dac7-4bef-96be-e21c8a0023c4', + 'public.notes', + array[('body', 'like', '%draft%', true)]::realtime.user_defined_filter_v2[], + '{"role": "authenticated"}' +); +``` + +Operator notes: +- `is` requires a keyword value: `null`, `true`, `false` or `unknown`. `is true`/`false`/`unknown` are only valid on boolean columns; `is null` works on any type. +- `like`/`ilike`/`match`/`imatch` require a text-compatible column type. +- `match`/`imatch` patterns are validated as regular expressions when the subscription is created. + To subscribe to `INSERT`s only on a table named `public.notes` where the `id` is `6` as the `authenticated` role: ```sql insert into realtime.subscription(subscription_id, entity, filters, claims, action_filter) -values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6', false)::realtime.user_defined_filter], '{"role", "authenticated"}', 'INSERT'); +values ('832bd278-dac7-4bef-96be-e21c8a0023c4', 'public.notes', array[('id', 'eq', '6')], '{"role", "authenticated"}', 'INSERT'); ``` To subscribe to `public.notes` and receive only the `id` and `title` columns (plus primary keys, which are always included): diff --git a/sql/walrus_migration_0015_like_ilike_is_not_ops.sql b/sql/walrus_migration_0015_like_ilike_is_not_ops.sql index b976d11..3157c24 100644 --- a/sql/walrus_migration_0015_like_ilike_is_not_ops.sql +++ b/sql/walrus_migration_0015_like_ilike_is_not_ops.sql @@ -1,27 +1,64 @@ --- Fix 1: Drop old 4-arg overload and its dependent before creating the 5-arg version -drop function if exists realtime.is_visible_through_filters(realtime.wal_column[], realtime.user_defined_filter[]); -drop function if exists realtime.check_equality_op(realtime.equality_op, regtype, text, text); +-- New equality operators (postgrest parity): like, ilike, is, match, imatch, +-- isdistinct, plus `negate` (postgrest `not.`). +-- +-- ROLLOUT — EXPAND/CONTRACT. The realtime.subscription.filters column is a +-- composite-typed array that the Realtime server writes positionally. Changing +-- the arity of realtime.user_defined_filter in place would break every old +-- server instance still writing 3-field literals during a rolling deploy (the +-- ALTER lands while old code is alive -> "cannot cast type record"). So instead +-- this migration EXPANDS: +-- * the existing 3-field `filters` column / type / functions are left intact, +-- so old instances keep inserting successfully; +-- * a new `filters_v2` column (4-field type carrying `negate`) is added with a +-- default of '{}', so it is invisible to old writers; +-- * apply_rls evaluates BOTH columns (one is always '{}' for a given row). +-- New server instances write filters_v2 once deployed. A later CONTRACT +-- migration drops the legacy column/type/functions after all old instances are +-- retired. +-- +-- Adding enum values is additive on its own, but it also makes the new ops legal +-- on the legacy 3-field column, where the legacy check_equality_op cannot +-- evaluate them ("UNKNOWN OP"). The trigger below therefore rejects the new ops +-- on the legacy `filters` column so they can only be stored on filters_v2. +-- All DDL below is written to be idempotent so the migration can be re-run or +-- recover from a partial application without erroring. +alter type realtime.equality_op add value if not exists 'like'; +alter type realtime.equality_op add value if not exists 'ilike'; +alter type realtime.equality_op add value if not exists 'is'; +alter type realtime.equality_op add value if not exists 'match'; +alter type realtime.equality_op add value if not exists 'imatch'; +alter type realtime.equality_op add value if not exists 'isdistinct'; -alter type realtime.equality_op add value 'like'; -alter type realtime.equality_op add value 'ilike'; -alter type realtime.equality_op add value 'is'; -alter type realtime.equality_op add value 'match'; -alter type realtime.equality_op add value 'imatch'; -alter type realtime.equality_op add value 'isdistinct'; - -alter type realtime.user_defined_filter add attribute negate boolean; +do $$ +begin + if not exists ( + select 1 + from pg_type t + join pg_namespace n on n.oid = t.typnamespace + where n.nspname = 'realtime' and t.typname = 'user_defined_filter_v2' + ) then + create type realtime.user_defined_filter_v2 as ( + column_name text, + op realtime.equality_op, + value text, + negate boolean + ); + end if; +end $$; -drop function if exists realtime.check_equality_op(realtime.equality_op, regtype, text, text); +-- v2 overload: same name, distinct 5-arg signature. It takes no default, so +-- there is no ambiguity with the existing 4-arg check_equality_op; a 4-arg call +-- resolves to the original function, a 5-arg call resolves to this one. create or replace function realtime.check_equality_op( op realtime.equality_op, type_ regtype, val_1 text, val_2 text, - negate boolean DEFAULT false + negate boolean ) returns bool - stable -- Fix 2: was immutable, uses EXECUTE so must be stable + stable -- uses EXECUTE, so cannot be immutable language plpgsql as $$ declare @@ -89,15 +126,20 @@ end; $$; -create or replace function realtime.is_visible_through_filters(columns realtime.wal_column[], filters realtime.user_defined_filter[]) +-- v2 overload: evaluates a record against the v2 filter type (with negate). +-- The original is_visible_through_filters(wal_column[], user_defined_filter[]) +-- overload is left in place for the legacy filters column. +create or replace function realtime.is_visible_through_filters(columns realtime.wal_column[], filters realtime.user_defined_filter_v2[]) returns bool language sql - stable -- Fix 2: was immutable, calls stable function so must be stable + stable -- calls a stable function, so cannot be immutable as $$ select filters is null or array_length(filters, 1) is null or coalesce( + -- Fail closed: every filter must match a column present in the WAL + -- payload, otherwise an unevaluable filter would default to visible. count(col.name) = count(1) and sum( realtime.check_equality_op( @@ -117,6 +159,32 @@ as $$ $$; +-- Additive new column. Default '{}' keeps it invisible to old writers that only +-- list the legacy `filters` column in their INSERTs. +alter table realtime.subscription + add column if not exists filters_v2 realtime.user_defined_filter_v2[] not null default '{}'; + +-- The unique constraint must cover both filter columns so new-style +-- subscriptions (filters = '{}') are still de-duplicated by their filters_v2. +-- Guarded so a re-run lands in the same state regardless of the starting point. +alter table realtime.subscription + drop constraint if exists subscription_subscription_id_entity_filters_key; +do $$ +begin + if not exists ( + select 1 + from pg_constraint + where conrelid = 'realtime.subscription'::regclass + and conname = 'subscription_subscription_id_entity_filters_key' + ) then + alter table realtime.subscription + add constraint subscription_subscription_id_entity_filters_key + unique (subscription_id, entity, filters, filters_v2); + end if; +end $$; + + +-- Trigger validates both filter columns and normalizes their order. create or replace function realtime.subscription_check_filters() returns trigger language plpgsql @@ -136,12 +204,15 @@ declare c.column_name, 'SELECT' ); - -- Fix 3: removed unused table_col_names declaration filter realtime.user_defined_filter; + filter_v2 realtime.user_defined_filter_v2; col_type regtype; in_val jsonb; selected_col text; begin + -- Legacy 3-field filters: only the original operators are evaluable by the + -- legacy check_equality_op path. Reject the new operators here so they + -- cannot be stored on this column and later crash apply_rls with UNKNOWN OP. for filter in select * from unnest(new.filters) loop if not filter.column_name = any(col_names) then raise exception 'invalid column for filter %', filter.column_name; @@ -157,43 +228,104 @@ begin raise exception 'failed to lookup type for column %', filter.column_name; end if; + if filter.op in ( + 'like'::realtime.equality_op, 'ilike'::realtime.equality_op, + 'is'::realtime.equality_op, 'match'::realtime.equality_op, + 'imatch'::realtime.equality_op, 'isdistinct'::realtime.equality_op + ) then + raise exception 'operator % is only supported on the filters_v2 column', filter.op::text; + end if; + if filter.op = 'in'::realtime.equality_op then in_val = realtime.cast(filter.value, (col_type::text || '[]')::regtype); if coalesce(jsonb_array_length(in_val), 0) > 100 then raise exception 'too many values for `in` filter. Maximum 100'; end if; - elsif filter.op = 'is'::realtime.equality_op then - if filter.value not in ('null', 'true', 'false', 'unknown') then + else + -- raises an exception if value is not coercable to type + perform realtime.cast(filter.value, col_type); + end if; + end loop; + + -- v2 filters: full validation including the new operators. + for filter_v2 in select * from unnest(new.filters_v2) loop + if not filter_v2.column_name = any(col_names) then + raise exception 'invalid column for filter %', filter_v2.column_name; + end if; + + col_type = ( + select atttypid::regtype + from pg_catalog.pg_attribute + where attrelid = new.entity + and attname = filter_v2.column_name + ); + if col_type is null then + raise exception 'failed to lookup type for column %', filter_v2.column_name; + end if; + + if filter_v2.op = 'in'::realtime.equality_op then + in_val = realtime.cast(filter_v2.value, (col_type::text || '[]')::regtype); + if coalesce(jsonb_array_length(in_val), 0) > 100 then + raise exception 'too many values for `in` filter. Maximum 100'; + end if; + elsif filter_v2.op = 'is'::realtime.equality_op then + -- `is` requires a keyword RHS rather than a typed literal + if filter_v2.value not in ('null', 'true', 'false', 'unknown') then raise exception 'invalid value for is filter: must be null, true, false, or unknown'; end if; - -- Fix 4: validate like/ilike only applies to text-compatible columns - elsif filter.op in ('like'::realtime.equality_op, 'ilike'::realtime.equality_op) then + -- IS NULL works for any type, but IS TRUE/FALSE/UNKNOWN require a + -- boolean operand. Reject the non-null keywords on non-boolean + -- columns here so they don't abort apply_rls at WAL time. + if filter_v2.value <> 'null' and col_type <> 'boolean'::regtype then + raise exception 'is % filter requires a boolean column, got %', filter_v2.value, col_type::text; + end if; + elsif filter_v2.op in ('like'::realtime.equality_op, 'ilike'::realtime.equality_op) then + -- like/ilike apply the text pattern operator (~~); reject column + -- types that have no such operator instead of failing at WAL time if not exists ( select 1 from pg_catalog.pg_operator where oprname = '~~' and oprleft = col_type ) then - raise exception 'operator % requires a text-compatible column type, got %', filter.op::text, col_type::text; + raise exception 'operator % requires a text-compatible column type, got %', filter_v2.op::text, col_type::text; + end if; + elsif filter_v2.op in ('match'::realtime.equality_op, 'imatch'::realtime.equality_op) then + -- match/imatch apply the regex operators ~ / ~*; reject column types + -- that have no such operator (e.g. integer) instead of failing at WAL + -- time, mirroring the like/ilike guard above. + if not exists ( + select 1 from pg_catalog.pg_operator + where oprname = case when filter_v2.op = 'imatch'::realtime.equality_op then '~*' else '~' end + and oprleft = col_type + and oprright = col_type + and oprresult = 'boolean'::regtype + ) then + raise exception 'operator % requires a text-compatible column type, got %', filter_v2.op::text, col_type::text; end if; - -- Fix 5: validate match/imatch regex patterns eagerly - elsif filter.op in ('match'::realtime.equality_op, 'imatch'::realtime.equality_op) then + -- validate the regex eagerly so a bad pattern is rejected here, not + -- inside apply_rls where it would abort the WAL stream for the entity begin - perform '' ~ filter.value; + perform '' ~ filter_v2.value; exception when others then - raise exception 'invalid regular expression for % filter: %', filter.op::text, sqlerrm; + raise exception 'invalid regular expression for % filter: %', filter_v2.op::text, sqlerrm; end; else - perform realtime.cast(filter.value, col_type); + -- eq/neq/lt/lte/gt/gte/isdistinct: value must be coercable to the type + perform realtime.cast(filter_v2.value, col_type); end if; end loop; + -- Reject empty selected_columns: array_agg would silently convert '{}' to NULL + -- during normalization, making it indistinguishable from "all columns" if new.selected_columns = '{}' then raise exception 'selected_columns cannot be empty. Remove the select parameter to capture all columns.'; end if; + -- Reject arrays with NULL elements which bypass column validation if new.selected_columns is not null and array_position(new.selected_columns, null::text) is not null then raise exception 'selected_columns cannot contain null values.'; end if; + -- Validate that selected_columns reference columns the role can SELECT if new.selected_columns is not null then for selected_col in select * from unnest(new.selected_columns) loop if not selected_col = any(col_names) then @@ -202,12 +334,20 @@ begin end loop; end if; - -- Fix 6: include negate in the ORDER BY for deterministic normalization + -- Apply consistent order to filters so the unique constraint can't be + -- tricked by a different filter order. negate is part of the v2 sort key. new.filters = coalesce( - array_agg(f order by f.column_name, f.op, f.value, f.negate), + array_agg(f order by f.column_name, f.op, f.value), '{}' ) from unnest(new.filters) f; + new.filters_v2 = coalesce( + array_agg(f order by f.column_name, f.op, f.value, f.negate), + '{}' + ) from unnest(new.filters_v2) f; + + -- Normalize selected_columns order so ARRAY['a','b'] and ARRAY['b','a'] are + -- treated as the same subscription group in apply_rls new.selected_columns = ( select array_agg(c order by c) from unnest(new.selected_columns) c @@ -216,3 +356,371 @@ begin return new; end; $$; + + +-- apply_rls re-defined so the visibility check evaluates BOTH filter columns. +-- For a given subscription one of the two is always '{}' (which evaluates to +-- visible), so this reduces to "enforce whichever column the writer populated". +create or replace function realtime.apply_rls(wal jsonb, max_record_bytes int = 1024 * 1024) + returns setof realtime.wal_rls + language plpgsql + volatile +as $$ +declare + -- Regclass of the table e.g. public.notes + entity_ regclass = (quote_ident(wal ->> 'schema') || '.' || quote_ident(wal ->> 'table'))::regclass; + + -- I, U, D, T: insert, update ... + action realtime.action = ( + case wal ->> 'action' + when 'I' then 'INSERT' + when 'U' then 'UPDATE' + when 'D' then 'DELETE' + else 'ERROR' + end + ); + + -- Is row level security enabled for the table + is_rls_enabled bool = relrowsecurity from pg_class where oid = entity_; + + subscriptions realtime.subscription[] = array_agg(subs) + from + realtime.subscription subs + where + subs.entity = entity_ + -- Filter by action early - only get subscriptions interested in this action + -- action_filter column can be: '*' (all), 'INSERT', 'UPDATE', or 'DELETE' + and (subs.action_filter = '*' or subs.action_filter = action::text); + + -- Subscription vars + working_role regrole; + working_selected_columns text[]; + claimed_role regrole; + claims jsonb; + + subscription_id uuid; + subscription_has_access bool; + visible_to_subscription_ids uuid[] = '{}'; + + -- structured info for wal's columns + columns realtime.wal_column[]; + -- previous identity values for update/delete + old_columns realtime.wal_column[]; + + error_record_exceeds_max_size boolean = octet_length(wal::text) > max_record_bytes; + + -- Primary jsonb output for record + output jsonb; + + -- Loop record for iterating unique roles (outer loop) + role_record record; + -- Loop record for iterating unique selected_columns within a role (inner loop) + cols_record record; + -- Subscription ids visible at the role level (before fanning out by selected_columns) + visible_role_sub_ids uuid[] = '{}'; + +begin + perform set_config('role', null, true); + + columns = + array_agg( + ( + x->>'name', + x->>'type', + x->>'typeoid', + realtime.cast( + (x->'value') #>> '{}', + coalesce( + (x->>'typeoid')::regtype, -- null when wal2json version <= 2.4 + (x->>'type')::regtype + ) + ), + (pks ->> 'name') is not null, + true + )::realtime.wal_column + ) + from + jsonb_array_elements(wal -> 'columns') x + left join jsonb_array_elements(wal -> 'pk') pks + on (x ->> 'name') = (pks ->> 'name'); + + old_columns = + array_agg( + ( + x->>'name', + x->>'type', + x->>'typeoid', + realtime.cast( + (x->'value') #>> '{}', + coalesce( + (x->>'typeoid')::regtype, -- null when wal2json version <= 2.4 + (x->>'type')::regtype + ) + ), + (pks ->> 'name') is not null, + true + )::realtime.wal_column + ) + from + jsonb_array_elements(wal -> 'identity') x + left join jsonb_array_elements(wal -> 'pk') pks + on (x ->> 'name') = (pks ->> 'name'); + + for role_record in + select claims_role + from (select distinct claims_role from unnest(subscriptions)) t + order by claims_role::text + loop + working_role := role_record.claims_role; + + -- Update `is_selectable` for columns and old_columns (once per role) + columns = + array_agg( + ( + c.name, + c.type_name, + c.type_oid, + c.value, + c.is_pkey, + pg_catalog.has_column_privilege(working_role, entity_, c.name, 'SELECT') + )::realtime.wal_column + ) + from + unnest(columns) c; + + old_columns = + array_agg( + ( + c.name, + c.type_name, + c.type_oid, + c.value, + c.is_pkey, + pg_catalog.has_column_privilege(working_role, entity_, c.name, 'SELECT') + )::realtime.wal_column + ) + from + unnest(old_columns) c; + + if action <> 'DELETE' and count(1) = 0 from unnest(columns) c where c.is_pkey then + -- Fan out 400 error per distinct selected_columns for this role + for cols_record in + select selected_columns + from (select distinct selected_columns from unnest(subscriptions) s where s.claims_role = working_role) t + order by coalesce(array_to_string(selected_columns, ','), '') + loop + working_selected_columns := cols_record.selected_columns; + return next ( + jsonb_build_object( + 'schema', wal ->> 'schema', + 'table', wal ->> 'table', + 'type', action + ), + is_rls_enabled, + (select array_agg(s.subscription_id) from unnest(subscriptions) as s where s.claims_role = working_role and (s.selected_columns is not distinct from working_selected_columns)), + array['Error 400: Bad Request, no primary key'] + )::realtime.wal_rls; + end loop; + + -- The claims role does not have SELECT permission to the primary key of entity + elsif action <> 'DELETE' and sum(c.is_selectable::int) <> count(1) from unnest(columns) c where c.is_pkey then + -- Fan out 401 error per distinct selected_columns for this role + for cols_record in + select selected_columns + from (select distinct selected_columns from unnest(subscriptions) s where s.claims_role = working_role) t + order by coalesce(array_to_string(selected_columns, ','), '') + loop + working_selected_columns := cols_record.selected_columns; + return next ( + jsonb_build_object( + 'schema', wal ->> 'schema', + 'table', wal ->> 'table', + 'type', action + ), + is_rls_enabled, + (select array_agg(s.subscription_id) from unnest(subscriptions) as s where s.claims_role = working_role and (s.selected_columns is not distinct from working_selected_columns)), + array['Error 401: Unauthorized'] + )::realtime.wal_rls; + end loop; + + else + -- Create the prepared statement (once per role) + if is_rls_enabled and action <> 'DELETE' then + if (select 1 from pg_prepared_statements where name = 'walrus_rls_stmt' limit 1) > 0 then + deallocate walrus_rls_stmt; + end if; + execute realtime.build_prepared_statement_sql('walrus_rls_stmt', entity_, columns); + end if; + + -- Collect all visible subscription IDs for this role (filter check + RLS check) + visible_role_sub_ids = '{}'; + + for subscription_id, claims in ( + select + subs.subscription_id, + subs.claims + from + unnest(subscriptions) subs + where + subs.entity = entity_ + and subs.claims_role = working_role + and ( + ( + realtime.is_visible_through_filters(columns, subs.filters) + and realtime.is_visible_through_filters(columns, subs.filters_v2) + ) + or ( + action = 'DELETE' + and realtime.is_visible_through_filters(old_columns, subs.filters) + and realtime.is_visible_through_filters(old_columns, subs.filters_v2) + ) + ) + ) loop + + if not is_rls_enabled or action = 'DELETE' then + visible_role_sub_ids = visible_role_sub_ids || subscription_id; + else + -- Check if RLS allows the role to see the record + perform + -- Trim leading and trailing quotes from working_role because set_config + -- doesn't recognize the role as valid if they are included + set_config('role', trim(both '"' from working_role::text), true), + set_config('request.jwt.claims', claims::text, true); + + execute 'execute walrus_rls_stmt' into subscription_has_access; + + if subscription_has_access then + visible_role_sub_ids = visible_role_sub_ids || subscription_id; + end if; + end if; + end loop; + + perform set_config('role', null, true); + + -- Inner loop: per distinct selected_columns for this role + for cols_record in + select selected_columns + from (select distinct selected_columns from unnest(subscriptions) s where s.claims_role = working_role) t + order by coalesce(array_to_string(selected_columns, ','), '') + loop + working_selected_columns := cols_record.selected_columns; + + output = jsonb_build_object( + 'schema', wal ->> 'schema', + 'table', wal ->> 'table', + 'type', action, + 'commit_timestamp', to_char( + ((wal ->> 'timestamp')::timestamptz at time zone 'utc'), + 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"' + ), + 'columns', ( + select + jsonb_agg( + jsonb_build_object( + 'name', pa.attname, + 'type', pt.typname + ) + order by pa.attnum asc + ) + from + pg_attribute pa + join pg_type pt + on pa.atttypid = pt.oid + left join ( + select unnest(conkey) as pkey_attnum + from pg_constraint + where conrelid = entity_ and contype = 'p' + ) pk on pk.pkey_attnum = pa.attnum + where + attrelid = entity_ + and attnum > 0 + and pg_catalog.has_column_privilege(working_role, entity_, pa.attname, 'SELECT') + and (working_selected_columns is null or pa.attname = any(working_selected_columns) or pk.pkey_attnum is not null) + ) + ) + -- Add "record" key for insert and update + || case + when action in ('INSERT', 'UPDATE') then + jsonb_build_object( + 'record', + ( + select + jsonb_object_agg( + -- if unchanged toast, get column name and value from old record + coalesce((c).name, (oc).name), + case + when (c).name is null then (oc).value + else (c).value + end + ) + from + unnest(columns) c + full outer join unnest(old_columns) oc + on (c).name = (oc).name + where + coalesce((c).is_selectable, (oc).is_selectable) + and (working_selected_columns is null or coalesce((c).name, (oc).name) = any(working_selected_columns) or coalesce((c).is_pkey, (oc).is_pkey)) + and ( not error_record_exceeds_max_size or (octet_length((c).value::text) <= 64)) + ) + ) + else '{}'::jsonb + end + -- Add "old_record" key for update and delete + || case + when action = 'UPDATE' then + jsonb_build_object( + 'old_record', + ( + select jsonb_object_agg((c).name, (c).value) + from unnest(old_columns) c + where + (c).is_selectable + and (working_selected_columns is null or (c).name = any(working_selected_columns) or (c).is_pkey) + and ( not error_record_exceeds_max_size or (octet_length((c).value::text) <= 64)) + ) + ) + when action = 'DELETE' then + jsonb_build_object( + 'old_record', + ( + select jsonb_object_agg((c).name, (c).value) + from unnest(old_columns) c + where + (c).is_selectable + and (working_selected_columns is null or (c).name = any(working_selected_columns) or (c).is_pkey) + and ( not error_record_exceeds_max_size or (octet_length((c).value::text) <= 64)) + and ( not is_rls_enabled or (c).is_pkey ) -- if RLS enabled, we can't secure deletes so filter to pkey + ) + ) + else '{}'::jsonb + end; + + -- Filter visible_role_sub_ids to those matching the current selected_columns group + visible_to_subscription_ids = coalesce( + ( + select array_agg(s.subscription_id) + from unnest(subscriptions) s + where s.claims_role = working_role + and (s.selected_columns is not distinct from working_selected_columns) + and s.subscription_id = any(visible_role_sub_ids) + ), + '{}'::uuid[] + ); + + return next ( + output, + is_rls_enabled, + visible_to_subscription_ids, + case + when error_record_exceeds_max_size then array['Error 413: Payload Too Large'] + else '{}' + end + )::realtime.wal_rls; + end loop; + + end if; + end loop; + + perform set_config('role', null, true); +end; +$$; diff --git a/test/expected/issue_40_quoted_regtype.out b/test/expected/issue_40_quoted_regtype.out index ed313de..b6d5f4b 100644 --- a/test/expected/issue_40_quoted_regtype.out +++ b/test/expected/issue_40_quoted_regtype.out @@ -25,7 +25,7 @@ select 'role', 'authenticated', 'sub', seed_uuid(2)::text ), - array[('primary_color', 'eq', 'RED', null)::realtime.user_defined_filter]; + array[('primary_color', 'eq', 'RED')::realtime.user_defined_filter]; insert into public.notes(id, primary_color) values (1, 'RED'), -- matches filter diff --git a/test/expected/issue_50_delete_filters.out b/test/expected/issue_50_delete_filters.out index dfc2bd8..404e593 100644 --- a/test/expected/issue_50_delete_filters.out +++ b/test/expected/issue_50_delete_filters.out @@ -17,7 +17,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value, null)::realtime.user_defined_filter] + array[(column_name, op, value)::realtime.user_defined_filter] from ( values @@ -25,10 +25,10 @@ from (2 , 'id', 'eq', '2') ) f(id, column_name, op, value); select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+-------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb,)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(id,eq,2,)"} + subscription_id | filters +--------------------------------------+------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(id,eq,2)"} (2 rows) ---------------------------------------------------------------------------------------- @@ -160,64 +160,6 @@ from } | | | (2 rows) ----------------------------------------------------------------------------------------- --- Multi-filter: missing column must not allow subscription to pass -- ----------------------------------------------------------------------------------------- -alter table public.notes replica identity default; -truncate table realtime.subscription; -insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(3), - 'public.notes', - jsonb_build_object( - 'role', 'authenticated', - 'email', 'example@example.com', - 'sub', seed_uuid(3)::text - ), - array[ - ('id', 'eq', '1', null)::realtime.user_defined_filter, - ('body', 'eq', 'bbb', null)::realtime.user_defined_filter - ]; -insert into public.notes(id, body) values (1, 'bbb'); -select clear_wal(); - clear_wal ------------ - -(1 row) - --- Non-full replica identity DELETE: only PK in WAL, body filter cannot be evaluated --- Expect 0 subscriptions: body='bbb' filter is unverifiable, must fail closed -delete from public.notes; -select - rec, - is_rls_enabled, - subscription_ids, - errors -from - walrus; - rec | is_rls_enabled | subscription_ids | errors -----------------------------------------------------+----------------+------------------+-------- - { +| f | {} | {} - "type": "DELETE", +| | | - "table": "notes", +| | | - "schema": "public", +| | | - "columns": [ +| | | - { +| | | - "name": "id", +| | | - "type": "int4" +| | | - }, +| | | - { +| | | - "name": "body", +| | | - "type": "text" +| | | - } +| | | - ], +| | | - "old_record": { +| | | - "id": 1 +| | | - }, +| | | - "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | - } | | | -(1 row) - drop table public.notes; select pg_drop_replication_slot('realtime'); pg_drop_replication_slot diff --git a/test/expected/issue_55_null_passes_filters.out b/test/expected/issue_55_null_passes_filters.out index 90288c1..ce5c540 100644 --- a/test/expected/issue_55_null_passes_filters.out +++ b/test/expected/issue_55_null_passes_filters.out @@ -17,7 +17,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(1)::text ), - array[('page_id', 'eq', '5', null)::realtime.user_defined_filter]; + array[('page_id', 'eq', '5')::realtime.user_defined_filter]; select clear_wal(); clear_wal ----------- diff --git a/test/expected/test_integration_filters.out b/test/expected/test_integration_filters.out index a2bdf21..c3d74b7 100644 --- a/test/expected/test_integration_filters.out +++ b/test/expected/test_integration_filters.out @@ -18,7 +18,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value, null)::realtime.user_defined_filter] + array[(column_name, op, value)::realtime.user_defined_filter] from ( values @@ -35,11 +35,11 @@ select clear_wal(); insert into public.notes(id, body) values (1, 'bbb'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+--------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb,)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,eq,aaaa,)"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,eq,cc,)"} + subscription_id | filters +--------------------------------------+-------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,eq,bbb)"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,eq,aaaa)"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,eq,cc)"} (3 rows) select diff --git a/test/expected/test_integration_in_filter.out b/test/expected/test_integration_in_filter.out index 6edfd26..ca44f0c 100644 --- a/test/expected/test_integration_in_filter.out +++ b/test/expected/test_integration_in_filter.out @@ -18,7 +18,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value, null)::realtime.user_defined_filter] + array[(column_name, op, value)::realtime.user_defined_filter] from ( values @@ -35,11 +35,11 @@ select clear_wal(); insert into public.notes(id, body) values (1, 'bbb'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+---------------------------------- - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,in,\"{aaa,bbb,ccc}\",)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,in,\"{aaa,ccc}\",)"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,in,{},)"} + subscription_id | filters +--------------------------------------+--------------------------------- + f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,in,\"{aaa,bbb,ccc}\")"} + 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,in,\"{aaa,ccc}\")"} + 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,in,{})"} (3 rows) select @@ -103,9 +103,9 @@ select 'email', 'example@example.com', 'sub', seed_uuid(6)::text ), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], null)::realtime.user_defined_filter]; + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])::realtime.user_defined_filter]; ERROR: too many values for `in` filter. Maximum 100 -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 41 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 52 at RAISE drop table public.notes; select pg_drop_replication_slot('realtime'); pg_drop_replication_slot diff --git a/test/expected/test_integration_in_uuid_filter.out b/test/expected/test_integration_in_uuid_filter.out index c327065..bed1550 100644 --- a/test/expected/test_integration_in_uuid_filter.out +++ b/test/expected/test_integration_in_uuid_filter.out @@ -21,8 +21,7 @@ select array[( 'identifier', 'in', - '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}', - null + '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}' )::realtime.user_defined_filter]; select clear_wal(); clear_wal @@ -33,9 +32,9 @@ select clear_wal(); insert into public.notes(id, identifier) values (1, 'ace23052-568e-4951-acc8-fd510ec667f9'); delete from public.notes; select subscription_id, filters from realtime.subscription; - subscription_id | filters ---------------------------------------+------------------------------------------------------------------------------------------------------ - 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(identifier,in,\"{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}\",)"} + subscription_id | filters +--------------------------------------+----------------------------------------------------------------------------------------------------- + 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(identifier,in,\"{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}\")"} (1 row) select diff --git a/test/expected/test_integration_like_ilike_is_not_filters.out b/test/expected/test_integration_like_ilike_is_not_filters.out index a1d9fcf..fb91e4e 100644 --- a/test/expected/test_integration_like_ilike_is_not_filters.out +++ b/test/expected/test_integration_like_ilike_is_not_filters.out @@ -1,154 +1,191 @@ -select 1 from pg_create_logical_replication_slot('realtime', 'wal2json', false); - ?column? ----------- - 1 -(1 row) - create table public.notes( id int primary key, body text, nullable_body text ); -alter table public.notes replica identity full; +-- ── Expand/contract: legacy `filters` and new `filters_v2` coexist ── +-- Old writers keep inserting 3-field legacy filters successfully (filters_v2 +-- defaults to '{}'). insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(id), +values ( + '00000000-0000-0000-0000-000000000001', 'public.notes', - jsonb_build_object( - 'role', 'authenticated', - 'email', 'example@example.com', - 'sub', seed_uuid(id)::text - ), - array[(column_name, op, value, negate)::realtime.user_defined_filter] -from - ( - values - -- like: matches 'hello world' → visible - (1, 'body', 'like', '%world%', false), - -- like: does not match → not visible - (2, 'body', 'like', '%xyz%', false), - -- ilike: case-insensitive match → visible - (3, 'body', 'ilike', '%WORLD%', false), - -- NOT LIKE: row does not match pattern → visible - (4, 'body', 'like', '%xyz%', true), - -- NOT LIKE: row matches pattern → not visible - (5, 'body', 'like', '%world%', true), - -- NOT IN: 'hello world' outside the list → visible - (6, 'body', 'in', '{foo,bar}', true), - -- NOT IN: 'hello world' inside the list → not visible - (7, 'body', 'in', '{hello world,other}', true), - -- is null on nullable_body (null row) → visible - (8, 'nullable_body', 'is', 'null', false), - -- is not null on nullable_body (null row)→ not visible - (9, 'nullable_body', 'is', 'null', true) - ) f(id, column_name, op, value, negate); -select clear_wal(); - clear_wal ------------ - -(1 row) - -insert into public.notes(id, body, nullable_body) values (1, 'hello world', null); -delete from public.notes; -select subscription_id, filters from realtime.subscription order by subscription_id; - subscription_id | filters ---------------------------------------+----------------------------------------- - 0d2f6c5b-62a4-5cae-af96-780d1ff5441b | {"(nullable_body,is,null,t)"} - 11955172-4e1d-5836-925f-2bcb7a287b87 | {"(body,ilike,%WORLD%,f)"} - 23407699-1647-5973-97ee-7848c18c8a4f | {"(nullable_body,is,null,f)"} - 2a5efacf-2c0a-5e59-861b-eff2079d1e2e | {"(body,in,\"{foo,bar}\",t)"} - 33b3e2e2-d91c-530c-9955-f89a94467a34 | {"(body,like,%xyz%,t)"} - 3fa85983-bc94-5c16-8bc7-157b8152c678 | {"(body,like,%world%,t)"} - 50737752-8d74-577e-9972-20ba646857af | {"(body,in,\"{hello world,other}\",t)"} - 5211e8ec-8c25-5c7f-9b03-6ff1eac0159e | {"(body,like,%xyz%,f)"} - f4539ebe-c779-5788-bbc1-2421ffaa8954 | {"(body,like,%world%,f)"} -(9 rows) - --- Expected visible subscription_ids: 1, 3, 4, 6, 8 -select - rec, - is_rls_enabled, - subscription_ids, - errors -from - walrus; - rec | is_rls_enabled | subscription_ids | errors -----------------------------------------------------+----------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------- - { +| f | {f4539ebe-c779-5788-bbc1-2421ffaa8954,11955172-4e1d-5836-925f-2bcb7a287b87,33b3e2e2-d91c-530c-9955-f89a94467a34,2a5efacf-2c0a-5e59-861b-eff2079d1e2e,23407699-1647-5973-97ee-7848c18c8a4f} | {} - "type": "INSERT", +| | | - "table": "notes", +| | | - "record": { +| | | - "id": 1, +| | | - "body": "hello world", +| | | - "nullable_body": null +| | | - }, +| | | - "schema": "public", +| | | - "columns": [ +| | | - { +| | | - "name": "id", +| | | - "type": "int4" +| | | - }, +| | | - { +| | | - "name": "body", +| | | - "type": "text" +| | | - }, +| | | - { +| | | - "name": "nullable_body", +| | | - "type": "text" +| | | - } +| | | - ], +| | | - "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | - } | | | - { +| f | {f4539ebe-c779-5788-bbc1-2421ffaa8954,11955172-4e1d-5836-925f-2bcb7a287b87,33b3e2e2-d91c-530c-9955-f89a94467a34,2a5efacf-2c0a-5e59-861b-eff2079d1e2e,23407699-1647-5973-97ee-7848c18c8a4f} | {} - "type": "DELETE", +| | | - "table": "notes", +| | | - "schema": "public", +| | | - "columns": [ +| | | - { +| | | - "name": "id", +| | | - "type": "int4" +| | | - }, +| | | - { +| | | - "name": "body", +| | | - "type": "text" +| | | - }, +| | | - { +| | | - "name": "nullable_body", +| | | - "type": "text" +| | | - } +| | | - ], +| | | - "old_record": { +| | | - "id": 1, +| | | - "body": "hello world", +| | | - "nullable_body": null +| | | - }, +| | | - "commit_timestamp": "2000-01-01T08:01:01.000Z"+| | | - } | | | + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000001'), + array[('id', 'eq', '6')]::realtime.user_defined_filter[] +); +-- New writers insert v2 filters (with negate) into filters_v2. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000002', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000002'), + array[ + ('body', 'ilike', '%world%', false), + ('body', 'like', '%x%', true) + ]::realtime.user_defined_filter_v2[] +); +-- filters_v2 is normalized (sorted by column_name, op, value, negate). +select filters, filters_v2 from realtime.subscription order by subscription_id; + filters | filters_v2 +---------------+------------------------------------------------ + {"(id,eq,6)"} | {} + {} | {"(body,like,%x%,t)","(body,ilike,%world%,f)"} (2 rows) --- Confirm is with invalid value is rejected at subscription time +-- The new operators are rejected on the LEGACY filters column (they can only be +-- evaluated on filters_v2; otherwise apply_rls would hit the UNKNOWN OP path). insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(10), +values ( + '00000000-0000-0000-0000-000000000003', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000003'), + array[('body', 'like', '%x%')]::realtime.user_defined_filter[] +); +ERROR: operator like is only supported on the filters_v2 column +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 46 at RAISE +-- `is` with an invalid keyword value is rejected at subscription time. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000004', 'public.notes', - jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(10)::text), - array[('body', 'is', 'invalid_value', false)::realtime.user_defined_filter]; + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000004'), + array[('body', 'is', 'maybe', false)]::realtime.user_defined_filter_v2[] +); ERROR: invalid value for is filter: must be null, true, false, or unknown -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 45 at RAISE --- Confirm in with more than 100 entries is rejected -insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(11), +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 84 at RAISE +-- like/ilike on a non-text column is rejected at subscription time. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000005', 'public.notes', - jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(11)::text), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], true)::realtime.user_defined_filter]; -ERROR: too many values for `in` filter. Maximum 100 -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 41 at RAISE -drop table public.notes; -select pg_drop_replication_slot('realtime'); - pg_drop_replication_slot --------------------------- - + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000005'), + array[('id', 'like', '%5%', false)]::realtime.user_defined_filter_v2[] +); +ERROR: operator like requires a text-compatible column type, got integer +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 99 at RAISE +-- An invalid regex is rejected at subscription time. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000006', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000006'), + array[('body', 'match', '(unclosed', false)]::realtime.user_defined_filter_v2[] +); +ERROR: invalid regular expression for match filter: invalid regular expression: parentheses () not balanced +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 119 at RAISE +-- match/imatch on a non-text column is rejected at subscription time (the regex +-- operator has no overload for the column type, which would abort apply_rls). +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000007', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000007'), + array[('id', 'match', 'foo.*', false)]::realtime.user_defined_filter_v2[] +); +ERROR: operator match requires a text-compatible column type, got integer +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 112 at RAISE +-- `is true` on a non-boolean column is rejected at subscription time (only +-- `is null` is type-agnostic). +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000008', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000008'), + array[('body', 'is', 'true', false)]::realtime.user_defined_filter_v2[] +); +ERROR: is true filter requires a boolean column, got text +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 90 at RAISE +-- ── Evaluation path (realtime.is_visible_through_filters over v2 filters) ── +-- A row equivalent to public.notes(id=1, body='hello world', nullable_body=null). +-- like: matches → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'like', '%world%', false)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- ilike: case-insensitive match → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'ilike', '%WORLD%', false)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- NOT LIKE: row matches pattern → not visible (false) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'like', '%world%', true)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +-- NOT IN: value inside the list → not visible (false) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'in', '{hello world,other}', true)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +-- is null on a null column → visible (true) +select realtime.is_visible_through_filters( + array[('nullable_body', 'text', null, 'null'::jsonb, false, true)]::realtime.wal_column[], + array[('nullable_body', 'is', 'null', false)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- is not null on a null column → not visible (false) +select realtime.is_visible_through_filters( + array[('nullable_body', 'text', null, 'null'::jsonb, false, true)]::realtime.wal_column[], + array[('nullable_body', 'is', 'null', true)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +-- isdistinct vs a different value → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'isdistinct', 'other', false)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- Fail closed: filter references a column absent from the WAL payload → not visible (false) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('missing', 'eq', 'x', false)]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +-- No filters → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + '{}'::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t (1 row) truncate table realtime.subscription; +drop table public.notes; diff --git a/test/expected/test_integration_v2_filter_composition.out b/test/expected/test_integration_v2_filter_composition.out new file mode 100644 index 0000000..a7c10af --- /dev/null +++ b/test/expected/test_integration_v2_filter_composition.out @@ -0,0 +1,144 @@ +-- Composite use cases: multiple v2 filters combined with AND semantics, mixing +-- every new operator and negations. realtime.is_visible_through_filters requires +-- ALL filters in the array to hold, so these exercise complex real-world +-- subscriptions like "title matches X AND status not in Y AND score >= Z". +create table public.articles( + id int primary key, + title text, + status text, + score int, + nickname text +); +-- A row equivalent to: +-- articles(id=42, title='Hello World', status='active', score=10, nickname=null) +-- expressed as a realtime.wal_column[] payload reused across the checks below. +-- ── Subscription path: the trigger accepts and normalizes a complex composite ── +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-0000000000aa', + 'public.articles', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-0000000000aa'), + array[ + ('title', 'ilike', '%world%', false), -- matches + ('status', 'in', '{pending,closed}', true), -- NOT IN → matches (active not listed) + ('score', 'gte', '5', false), -- 10 >= 5 → matches + ('nickname','is', 'null', false), -- null → matches + ('id', 'isdistinct', '0', false) -- 42 distinct from 0 → matches + ]::realtime.user_defined_filter_v2[] +); +select filters_v2 from realtime.subscription where subscription_id = '00000000-0000-0000-0000-0000000000aa'; + filters_v2 +--------------------------------------------------------------------------------------------------------------------------------- + {"(id,isdistinct,0,f)","(nickname,is,null,f)","(score,gte,5,f)","(status,in,\"{pending,closed}\",t)","(title,ilike,%world%,f)"} +(1 row) + +-- ── Evaluation: all-AND composites over the example row ── +-- 1. Every operator satisfied (ilike + NOT IN + gte + is null + isdistinct) → visible +select realtime.is_visible_through_filters( + array[ + ('id', 'int4', null, '42'::jsonb, true, true), + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('status', 'text', null, '"active"'::jsonb, false, true), + ('score', 'int4', null, '10'::jsonb, false, true), + ('nickname','text', null, 'null'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'ilike', '%world%', false), + ('status', 'in', '{pending,closed}', true), + ('score', 'gte', '5', false), + ('nickname','is', 'null', false), + ('id', 'isdistinct', '0', false) + ]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- 2. Same composite but one negation flips a clause (NOT ilike on a matching +-- pattern) → the AND fails → not visible +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('status', 'text', null, '"active"'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'ilike', '%world%', true), -- NOT ilike, but it matches → clause false + ('status', 'eq', 'active', false) + ]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +-- 3. Regex composite with negation: imatch matches AND NOT match a different +-- pattern AND neq → visible +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('status', 'text', null, '"active"'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'imatch', '^hello', false), -- case-insensitive match + ('title', 'match', '^Goodbye', true), -- NOT match → true (no match) + ('status', 'neq', 'archived', false) + ]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- 4. Negated isdistinct used as "IS NOT DISTINCT FROM" inside a composite: +-- score IS NOT DISTINCT FROM 10 (true) AND id between via gt/lt → visible +select realtime.is_visible_through_filters( + array[ + ('id', 'int4', null, '42'::jsonb, true, true), + ('score', 'int4', null, '10'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('score', 'isdistinct', '10', true), -- IS NOT DISTINCT FROM 10 → true + ('id', 'gt', '40', false), + ('id', 'lt', '100', false) + ]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + t +(1 row) + +-- 5. `is not null` inside a composite where the column is NULL → clause false → not visible +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('nickname','text', null, 'null'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'like', 'Hello%', false), + ('nickname','is', 'null', true) -- IS NOT NULL, but nickname is null → false + ]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +-- 6. Fail-closed within a composite: one clause references a column missing from +-- the payload → whole composite not visible even though the others match +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'like', 'Hello%', false), -- matches + ('status', 'eq', 'active', false) -- status absent from payload → fail closed + ]::realtime.user_defined_filter_v2[] +); + is_visible_through_filters +---------------------------- + f +(1 row) + +truncate table realtime.subscription; +drop table public.articles; diff --git a/test/expected/test_select_columns_empty.out b/test/expected/test_select_columns_empty.out index e5fd42d..dfcb515 100644 --- a/test/expected/test_select_columns_empty.out +++ b/test/expected/test_select_columns_empty.out @@ -16,6 +16,6 @@ select ), '{}'::text[]; ERROR: selected_columns cannot be empty. Remove the select parameter to capture all columns. -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 68 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 130 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_filter_not_in_select.out b/test/expected/test_select_columns_filter_not_in_select.out index 0aaa1a8..11ba40c 100644 --- a/test/expected/test_select_columns_filter_not_in_select.out +++ b/test/expected/test_select_columns_filter_not_in_select.out @@ -26,7 +26,7 @@ select 'sub', seed_uuid(1)::text ), array['body'], - array[('extra', 'eq', 'match', null)::realtime.user_defined_filter]; + array[('extra', 'eq', 'match')::realtime.user_defined_filter]; -- Matching row: filter on extra matches, but extra not in output select clear_wal(); clear_wal diff --git a/test/expected/test_select_columns_invalid.out b/test/expected/test_select_columns_invalid.out index 750a26c..faf83dc 100644 --- a/test/expected/test_select_columns_invalid.out +++ b/test/expected/test_select_columns_invalid.out @@ -16,6 +16,6 @@ select ), array['nonexistent_column']; ERROR: invalid column for select nonexistent_column -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 78 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 142 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/expected/test_select_columns_null_element.out b/test/expected/test_select_columns_null_element.out index fea7278..7690ccf 100644 --- a/test/expected/test_select_columns_null_element.out +++ b/test/expected/test_select_columns_null_element.out @@ -16,6 +16,6 @@ select ), array[null]::text[]; ERROR: selected_columns cannot contain null values. -CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 72 at RAISE +CONTEXT: PL/pgSQL function realtime.subscription_check_filters() line 135 at RAISE drop table public.notes; truncate table realtime.subscription; diff --git a/test/sql/issue_40_quoted_regtype.sql b/test/sql/issue_40_quoted_regtype.sql index 8a833b7..02679fd 100644 --- a/test/sql/issue_40_quoted_regtype.sql +++ b/test/sql/issue_40_quoted_regtype.sql @@ -29,7 +29,7 @@ select 'role', 'authenticated', 'sub', seed_uuid(2)::text ), - array[('primary_color', 'eq', 'RED', null)::realtime.user_defined_filter]; + array[('primary_color', 'eq', 'RED')::realtime.user_defined_filter]; insert into public.notes(id, primary_color) values diff --git a/test/sql/issue_50_delete_filters.sql b/test/sql/issue_50_delete_filters.sql index daea76b..77d4253 100644 --- a/test/sql/issue_50_delete_filters.sql +++ b/test/sql/issue_50_delete_filters.sql @@ -14,7 +14,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value, null)::realtime.user_defined_filter] + array[(column_name, op, value)::realtime.user_defined_filter] from ( values @@ -64,45 +64,6 @@ select clear_wal(); delete from public.notes; -select - rec, - is_rls_enabled, - subscription_ids, - errors -from - walrus; - - ----------------------------------------------------------------------------------------- --- Multi-filter: missing column must not allow subscription to pass -- ----------------------------------------------------------------------------------------- - -alter table public.notes replica identity default; - -truncate table realtime.subscription; - -insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(3), - 'public.notes', - jsonb_build_object( - 'role', 'authenticated', - 'email', 'example@example.com', - 'sub', seed_uuid(3)::text - ), - array[ - ('id', 'eq', '1', null)::realtime.user_defined_filter, - ('body', 'eq', 'bbb', null)::realtime.user_defined_filter - ]; - -insert into public.notes(id, body) values (1, 'bbb'); - -select clear_wal(); - --- Non-full replica identity DELETE: only PK in WAL, body filter cannot be evaluated --- Expect 0 subscriptions: body='bbb' filter is unverifiable, must fail closed -delete from public.notes; - select rec, is_rls_enabled, diff --git a/test/sql/issue_55_null_passes_filters.sql b/test/sql/issue_55_null_passes_filters.sql index e3bbfe2..adf2802 100644 --- a/test/sql/issue_55_null_passes_filters.sql +++ b/test/sql/issue_55_null_passes_filters.sql @@ -14,7 +14,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(1)::text ), - array[('page_id', 'eq', '5', null)::realtime.user_defined_filter]; + array[('page_id', 'eq', '5')::realtime.user_defined_filter]; select clear_wal(); diff --git a/test/sql/test_integration_filters.sql b/test/sql/test_integration_filters.sql index 2e37725..63cbc83 100644 --- a/test/sql/test_integration_filters.sql +++ b/test/sql/test_integration_filters.sql @@ -16,7 +16,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value, null)::realtime.user_defined_filter] + array[(column_name, op, value)::realtime.user_defined_filter] from ( values diff --git a/test/sql/test_integration_in_filter.sql b/test/sql/test_integration_in_filter.sql index dab3eed..2608099 100644 --- a/test/sql/test_integration_in_filter.sql +++ b/test/sql/test_integration_in_filter.sql @@ -16,7 +16,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(id)::text ), - array[(column_name, op, value, null)::realtime.user_defined_filter] + array[(column_name, op, value)::realtime.user_defined_filter] from ( values @@ -52,7 +52,7 @@ select 'email', 'example@example.com', 'sub', seed_uuid(6)::text ), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], null)::realtime.user_defined_filter]; + array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])::realtime.user_defined_filter]; drop table public.notes; select pg_drop_replication_slot('realtime'); diff --git a/test/sql/test_integration_in_uuid_filter.sql b/test/sql/test_integration_in_uuid_filter.sql index 9fb538e..5ac3385 100644 --- a/test/sql/test_integration_in_uuid_filter.sql +++ b/test/sql/test_integration_in_uuid_filter.sql @@ -19,8 +19,7 @@ select array[( 'identifier', 'in', - '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}', - null + '{ace23052-568e-4951-acc8-fd510ec667f9,7057e8c3-c05f-4944-b9d9-05f8c45393d1}' )::realtime.user_defined_filter]; diff --git a/test/sql/test_integration_like_ilike_is_not_filters.sql b/test/sql/test_integration_like_ilike_is_not_filters.sql index 249c04c..07560d0 100644 --- a/test/sql/test_integration_like_ilike_is_not_filters.sql +++ b/test/sql/test_integration_like_ilike_is_not_filters.sql @@ -1,80 +1,149 @@ -select 1 from pg_create_logical_replication_slot('realtime', 'wal2json', false); - create table public.notes( id int primary key, body text, nullable_body text ); -alter table public.notes replica identity full; +-- ── Expand/contract: legacy `filters` and new `filters_v2` coexist ── +-- Old writers keep inserting 3-field legacy filters successfully (filters_v2 +-- defaults to '{}'). insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(id), +values ( + '00000000-0000-0000-0000-000000000001', 'public.notes', - jsonb_build_object( - 'role', 'authenticated', - 'email', 'example@example.com', - 'sub', seed_uuid(id)::text - ), - array[(column_name, op, value, negate)::realtime.user_defined_filter] -from - ( - values - -- like: matches 'hello world' → visible - (1, 'body', 'like', '%world%', false), - -- like: does not match → not visible - (2, 'body', 'like', '%xyz%', false), - -- ilike: case-insensitive match → visible - (3, 'body', 'ilike', '%WORLD%', false), - -- NOT LIKE: row does not match pattern → visible - (4, 'body', 'like', '%xyz%', true), - -- NOT LIKE: row matches pattern → not visible - (5, 'body', 'like', '%world%', true), - -- NOT IN: 'hello world' outside the list → visible - (6, 'body', 'in', '{foo,bar}', true), - -- NOT IN: 'hello world' inside the list → not visible - (7, 'body', 'in', '{hello world,other}', true), - -- is null on nullable_body (null row) → visible - (8, 'nullable_body', 'is', 'null', false), - -- is not null on nullable_body (null row)→ not visible - (9, 'nullable_body', 'is', 'null', true) - ) f(id, column_name, op, value, negate); - - -select clear_wal(); -insert into public.notes(id, body, nullable_body) values (1, 'hello world', null); - -delete from public.notes; - -select subscription_id, filters from realtime.subscription order by subscription_id; - --- Expected visible subscription_ids: 1, 3, 4, 6, 8 -select - rec, - is_rls_enabled, - subscription_ids, - errors -from - walrus; - - --- Confirm is with invalid value is rejected at subscription time -insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(10), + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000001'), + array[('id', 'eq', '6')]::realtime.user_defined_filter[] +); + +-- New writers insert v2 filters (with negate) into filters_v2. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000002', 'public.notes', - jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(10)::text), - array[('body', 'is', 'invalid_value', false)::realtime.user_defined_filter]; + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000002'), + array[ + ('body', 'ilike', '%world%', false), + ('body', 'like', '%x%', true) + ]::realtime.user_defined_filter_v2[] +); + +-- filters_v2 is normalized (sorted by column_name, op, value, negate). +select filters, filters_v2 from realtime.subscription order by subscription_id; --- Confirm in with more than 100 entries is rejected +-- The new operators are rejected on the LEGACY filters column (they can only be +-- evaluated on filters_v2; otherwise apply_rls would hit the UNKNOWN OP path). insert into realtime.subscription(subscription_id, entity, claims, filters) -select - seed_uuid(11), +values ( + '00000000-0000-0000-0000-000000000003', 'public.notes', - jsonb_build_object('role', 'authenticated', 'email', 'example@example.com', 'sub', seed_uuid(11)::text), - array[('body', 'in', array[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1], true)::realtime.user_defined_filter]; + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000003'), + array[('body', 'like', '%x%')]::realtime.user_defined_filter[] +); + +-- `is` with an invalid keyword value is rejected at subscription time. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000004', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000004'), + array[('body', 'is', 'maybe', false)]::realtime.user_defined_filter_v2[] +); + +-- like/ilike on a non-text column is rejected at subscription time. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000005', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000005'), + array[('id', 'like', '%5%', false)]::realtime.user_defined_filter_v2[] +); + +-- An invalid regex is rejected at subscription time. +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000006', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000006'), + array[('body', 'match', '(unclosed', false)]::realtime.user_defined_filter_v2[] +); + +-- match/imatch on a non-text column is rejected at subscription time (the regex +-- operator has no overload for the column type, which would abort apply_rls). +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000007', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000007'), + array[('id', 'match', 'foo.*', false)]::realtime.user_defined_filter_v2[] +); + +-- `is true` on a non-boolean column is rejected at subscription time (only +-- `is null` is type-agnostic). +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-000000000008', + 'public.notes', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-000000000008'), + array[('body', 'is', 'true', false)]::realtime.user_defined_filter_v2[] +); + +-- ── Evaluation path (realtime.is_visible_through_filters over v2 filters) ── +-- A row equivalent to public.notes(id=1, body='hello world', nullable_body=null). + +-- like: matches → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'like', '%world%', false)]::realtime.user_defined_filter_v2[] +); + +-- ilike: case-insensitive match → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'ilike', '%WORLD%', false)]::realtime.user_defined_filter_v2[] +); + +-- NOT LIKE: row matches pattern → not visible (false) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'like', '%world%', true)]::realtime.user_defined_filter_v2[] +); + +-- NOT IN: value inside the list → not visible (false) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'in', '{hello world,other}', true)]::realtime.user_defined_filter_v2[] +); + +-- is null on a null column → visible (true) +select realtime.is_visible_through_filters( + array[('nullable_body', 'text', null, 'null'::jsonb, false, true)]::realtime.wal_column[], + array[('nullable_body', 'is', 'null', false)]::realtime.user_defined_filter_v2[] +); + +-- is not null on a null column → not visible (false) +select realtime.is_visible_through_filters( + array[('nullable_body', 'text', null, 'null'::jsonb, false, true)]::realtime.wal_column[], + array[('nullable_body', 'is', 'null', true)]::realtime.user_defined_filter_v2[] +); + +-- isdistinct vs a different value → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('body', 'isdistinct', 'other', false)]::realtime.user_defined_filter_v2[] +); + +-- Fail closed: filter references a column absent from the WAL payload → not visible (false) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + array[('missing', 'eq', 'x', false)]::realtime.user_defined_filter_v2[] +); + +-- No filters → visible (true) +select realtime.is_visible_through_filters( + array[('body', 'text', null, '"hello world"'::jsonb, false, true)]::realtime.wal_column[], + '{}'::realtime.user_defined_filter_v2[] +); -drop table public.notes; -select pg_drop_replication_slot('realtime'); truncate table realtime.subscription; +drop table public.notes; diff --git a/test/sql/test_integration_v2_filter_composition.sql b/test/sql/test_integration_v2_filter_composition.sql new file mode 100644 index 0000000..96a3176 --- /dev/null +++ b/test/sql/test_integration_v2_filter_composition.sql @@ -0,0 +1,120 @@ +-- Composite use cases: multiple v2 filters combined with AND semantics, mixing +-- every new operator and negations. realtime.is_visible_through_filters requires +-- ALL filters in the array to hold, so these exercise complex real-world +-- subscriptions like "title matches X AND status not in Y AND score >= Z". + +create table public.articles( + id int primary key, + title text, + status text, + score int, + nickname text +); + +-- A row equivalent to: +-- articles(id=42, title='Hello World', status='active', score=10, nickname=null) +-- expressed as a realtime.wal_column[] payload reused across the checks below. + +-- ── Subscription path: the trigger accepts and normalizes a complex composite ── +insert into realtime.subscription(subscription_id, entity, claims, filters_v2) +values ( + '00000000-0000-0000-0000-0000000000aa', + 'public.articles', + jsonb_build_object('role', 'authenticated', 'email', 'a@example.com', 'sub', '00000000-0000-0000-0000-0000000000aa'), + array[ + ('title', 'ilike', '%world%', false), -- matches + ('status', 'in', '{pending,closed}', true), -- NOT IN → matches (active not listed) + ('score', 'gte', '5', false), -- 10 >= 5 → matches + ('nickname','is', 'null', false), -- null → matches + ('id', 'isdistinct', '0', false) -- 42 distinct from 0 → matches + ]::realtime.user_defined_filter_v2[] +); +select filters_v2 from realtime.subscription where subscription_id = '00000000-0000-0000-0000-0000000000aa'; + +-- ── Evaluation: all-AND composites over the example row ── + +-- 1. Every operator satisfied (ilike + NOT IN + gte + is null + isdistinct) → visible +select realtime.is_visible_through_filters( + array[ + ('id', 'int4', null, '42'::jsonb, true, true), + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('status', 'text', null, '"active"'::jsonb, false, true), + ('score', 'int4', null, '10'::jsonb, false, true), + ('nickname','text', null, 'null'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'ilike', '%world%', false), + ('status', 'in', '{pending,closed}', true), + ('score', 'gte', '5', false), + ('nickname','is', 'null', false), + ('id', 'isdistinct', '0', false) + ]::realtime.user_defined_filter_v2[] +); + +-- 2. Same composite but one negation flips a clause (NOT ilike on a matching +-- pattern) → the AND fails → not visible +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('status', 'text', null, '"active"'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'ilike', '%world%', true), -- NOT ilike, but it matches → clause false + ('status', 'eq', 'active', false) + ]::realtime.user_defined_filter_v2[] +); + +-- 3. Regex composite with negation: imatch matches AND NOT match a different +-- pattern AND neq → visible +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('status', 'text', null, '"active"'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'imatch', '^hello', false), -- case-insensitive match + ('title', 'match', '^Goodbye', true), -- NOT match → true (no match) + ('status', 'neq', 'archived', false) + ]::realtime.user_defined_filter_v2[] +); + +-- 4. Negated isdistinct used as "IS NOT DISTINCT FROM" inside a composite: +-- score IS NOT DISTINCT FROM 10 (true) AND id between via gt/lt → visible +select realtime.is_visible_through_filters( + array[ + ('id', 'int4', null, '42'::jsonb, true, true), + ('score', 'int4', null, '10'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('score', 'isdistinct', '10', true), -- IS NOT DISTINCT FROM 10 → true + ('id', 'gt', '40', false), + ('id', 'lt', '100', false) + ]::realtime.user_defined_filter_v2[] +); + +-- 5. `is not null` inside a composite where the column is NULL → clause false → not visible +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true), + ('nickname','text', null, 'null'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'like', 'Hello%', false), + ('nickname','is', 'null', true) -- IS NOT NULL, but nickname is null → false + ]::realtime.user_defined_filter_v2[] +); + +-- 6. Fail-closed within a composite: one clause references a column missing from +-- the payload → whole composite not visible even though the others match +select realtime.is_visible_through_filters( + array[ + ('title', 'text', null, '"Hello World"'::jsonb, false, true) + ]::realtime.wal_column[], + array[ + ('title', 'like', 'Hello%', false), -- matches + ('status', 'eq', 'active', false) -- status absent from payload → fail closed + ]::realtime.user_defined_filter_v2[] +); + +truncate table realtime.subscription; +drop table public.articles; diff --git a/test/sql/test_select_columns_filter_not_in_select.sql b/test/sql/test_select_columns_filter_not_in_select.sql index 77b8f28..b927c78 100644 --- a/test/sql/test_select_columns_filter_not_in_select.sql +++ b/test/sql/test_select_columns_filter_not_in_select.sql @@ -24,7 +24,7 @@ select 'sub', seed_uuid(1)::text ), array['body'], - array[('extra', 'eq', 'match', null)::realtime.user_defined_filter]; + array[('extra', 'eq', 'match')::realtime.user_defined_filter]; -- Matching row: filter on extra matches, but extra not in output select clear_wal();