From e2f184dd32485ad60849bb4812e5624d8aed7f26 Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 11:16:03 -0800 Subject: [PATCH 1/8] Add function to improve search performance. --- sql/2025-12-15_faster-project-permissions.sql | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 sql/2025-12-15_faster-project-permissions.sql diff --git a/sql/2025-12-15_faster-project-permissions.sql b/sql/2025-12-15_faster-project-permissions.sql new file mode 100644 index 00000000..77c5c0e0 --- /dev/null +++ b/sql/2025-12-15_faster-project-permissions.sql @@ -0,0 +1,49 @@ +-- The previous user_has_project_permission function is called on _every_ project when doing global omnisearch, +-- which is too slow. + +-- Create a view which serves as join table for finding projects we should consider in search, it's much faster +-- than running a permission check for every private project. +CREATE FUNCTION projects_searchable_by_user(arg_user_id UUID) +-- Returns a subset of the projects table +RETURNS SETOF projects AS $$ + -- Get all public projects and projects owned by the user, + -- as well as all public projects. + SELECT p.* + FROM projects p + WHERE p.owner_user_id = arg_user_id + OR NOT p.private +UNION + SELECT + p.* + FROM org_members om + JOIN projects p + ON om.organization_user_id = p.owner_user_id + JOIN roles r ON om.role_id = r.id + WHERE om.member_user_id = arg_user_id + AND 'project:view' = ANY(r.permissions) + -- All public projects are already included above + AND p.private + UNION + -- Include projects the user is a direct maintainer of + SELECT + p.* + FROM users u + JOIN role_memberships rm ON u.subject_id = rm.subject_id + JOIN roles r ON rm.role_id = r.id + JOIN projects p ON rm.resource_id = p.resource_id + WHERE u.id = arg_user_id + AND 'project:view' = ANY(r.permissions) + AND p.private; +$$ LANGUAGE sql STABLE PARALLEL SAFE; + +-- A better index for this query. +-- CREATE INDEX idx_projects_by_owner_and_privacy +-- ON projects (private, owner_user_id); + +CREATE INDEX idx_public_projects_by_owner + ON projects (owner_user_id) + WHERE NOT private; + +CREATE INDEX idx_private_projects_by_owner + ON projects (owner_user_id) + WHERE private; From a01ef79dae56282231fa165fd2660ac805c34ff6 Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 12:07:23 -0800 Subject: [PATCH 2/8] Update project queries to faster search function --- share-api/src/Share/Postgres/Queries.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/share-api/src/Share/Postgres/Queries.hs b/share-api/src/Share/Postgres/Queries.hs index d7fee185..2955fa3d 100644 --- a/share-api/src/Share/Postgres/Queries.hs +++ b/share-api/src/Share/Postgres/Queries.hs @@ -156,10 +156,9 @@ searchProjects caller userIdFilter (Query query) psk limit = do PG.queryListRows @(Project PG.:. PG.Only UserHandle) [PG.sql| SELECT p.id, p.owner_user_id, p.slug, p.summary, p.tags, p.private, p.created_at, p.updated_at, owner.handle - FROM projects p + FROM projects_searchable_by_user(#{caller}) p JOIN users owner ON p.owner_user_id = owner.id WHERE p.owner_user_id = #{userId} - AND user_has_project_permission(#{caller}, p.id, #{ProjectView}) ORDER BY p.created_at DESC LIMIT #{limit} |] @@ -167,11 +166,10 @@ searchProjects caller userIdFilter (Query query) psk limit = do PG.queryListRows [PG.sql| SELECT p.id, p.owner_user_id, p.slug, p.summary, p.tags, p.private, p.created_at, p.updated_at, owner.handle - FROM websearch_to_tsquery('english', #{query}) AS tokenquery, projects AS p + FROM websearch_to_tsquery('english', #{query}) AS tokenquery, projects_searchable_by_user(#{caller}) AS p JOIN users AS owner ON p.owner_user_id = owner.id WHERE (tokenquery @@ p.project_text_document OR p.slug ILIKE ('%' || like_escape(#{query}) || '%')) AND (#{userIdFilter} IS NULL OR p.owner_user_id = #{userIdFilter}) - AND user_has_project_permission(#{caller}, p.id, #{ProjectView}) ^{pskFilter} ORDER BY p.slug = #{query} DESC, From cb504a876a91f7964006b47bbebd800b67b2d2d4 Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 12:12:41 -0800 Subject: [PATCH 3/8] Generalize to work with any permission --- ...5-12-15_faster-project-by-permissions.sql} | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) rename sql/{2025-12-15_faster-project-permissions.sql => 2025-12-15_faster-project-by-permissions.sql} (51%) diff --git a/sql/2025-12-15_faster-project-permissions.sql b/sql/2025-12-15_faster-project-by-permissions.sql similarity index 51% rename from sql/2025-12-15_faster-project-permissions.sql rename to sql/2025-12-15_faster-project-by-permissions.sql index 77c5c0e0..7110df16 100644 --- a/sql/2025-12-15_faster-project-permissions.sql +++ b/sql/2025-12-15_faster-project-by-permissions.sql @@ -1,9 +1,12 @@ -- The previous user_has_project_permission function is called on _every_ project when doing global omnisearch, -- which is too slow. --- Create a view which serves as join table for finding projects we should consider in search, it's much faster --- than running a permission check for every private project. -CREATE FUNCTION projects_searchable_by_user(arg_user_id UUID) +-- Create a view which serves as join table for finding projects for which the user has a given permission, it's much faster +-- than running a permission check for every private project when we need to discover the list of all projects a user +-- has access to. +-- +-- This special-cases the 'project:view' permission to be even faster, it's the most common case. +CREATE FUNCTION projects_by_user_permission(arg_user_id UUID, arg_permission permission) -- Returns a subset of the projects table RETURNS SETOF projects AS $$ -- Get all public projects and projects owned by the user, @@ -11,7 +14,7 @@ RETURNS SETOF projects AS $$ SELECT p.* FROM projects p WHERE p.owner_user_id = arg_user_id - OR NOT p.private + OR (arg_permission = 'project:view' AND NOT p.private) UNION SELECT p.* @@ -20,9 +23,9 @@ UNION ON om.organization_user_id = p.owner_user_id JOIN roles r ON om.role_id = r.id WHERE om.member_user_id = arg_user_id - AND 'project:view' = ANY(r.permissions) - -- All public projects are already included above - AND p.private + AND arg_permission = ANY(r.permissions) + -- All public projects are already included above if the permission is 'project:view' + AND (p.private OR arg_permission <> 'project:view') UNION -- Include projects the user is a direct maintainer of SELECT @@ -32,18 +35,11 @@ UNION JOIN roles r ON rm.role_id = r.id JOIN projects p ON rm.resource_id = p.resource_id WHERE u.id = arg_user_id - AND 'project:view' = ANY(r.permissions) - AND p.private; + AND arg_permission = ANY(r.permissions) + -- All public projects are already included above if the permission is 'project:view' + AND (p.private OR arg_permission <> 'project:view') $$ LANGUAGE sql STABLE PARALLEL SAFE; -- A better index for this query. --- CREATE INDEX idx_projects_by_owner_and_privacy --- ON projects (private, owner_user_id); - -CREATE INDEX idx_public_projects_by_owner - ON projects (owner_user_id) - WHERE NOT private; - -CREATE INDEX idx_private_projects_by_owner - ON projects (owner_user_id) - WHERE private; +CREATE INDEX idx_projects_by_owner_and_privacy + ON projects (private, owner_user_id); From adc285434bebda5f138547c04288847d50fa64b8 Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 12:12:41 -0800 Subject: [PATCH 4/8] Use new permissions project join where possible --- .../src/Share/Postgres/Contributions/Queries.hs | 4 ++-- share-api/src/Share/Postgres/Queries.hs | 6 ++---- .../Postgres/Search/DefinitionSearch/Queries.hs | 13 ++++++------- share-api/src/Share/Postgres/Tickets/Queries.hs | 3 +-- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/share-api/src/Share/Postgres/Contributions/Queries.hs b/share-api/src/Share/Postgres/Contributions/Queries.hs index 7d233aa7..91753c82 100644 --- a/share-api/src/Share/Postgres/Contributions/Queries.hs +++ b/share-api/src/Share/Postgres/Contributions/Queries.hs @@ -260,10 +260,10 @@ listContributionsByUserId callerUserId userId limit mayCursor mayStatusFilter ma contribution.author_id, (SELECT COUNT(*) FROM comments comment WHERE comment.contribution_id = contribution.id AND comment.deleted_at IS NULL) as num_comments FROM contributions AS contribution - JOIN projects AS project ON project.id = contribution.project_id + JOIN projects_by_user_permission(#{callerUserId}, #{ProjectView}) AS project + ON project.id = contribution.project_id WHERE contribution.author_id = #{userId} - AND user_has_project_permission(#{callerUserId}, project.id, #{ProjectView}) AND (#{mayStatusFilter} IS NULL OR contribution.status = #{mayStatusFilter}) AND ^{cursorFilter} AND ^{kindFilter} diff --git a/share-api/src/Share/Postgres/Queries.hs b/share-api/src/Share/Postgres/Queries.hs index 2955fa3d..0ef444f2 100644 --- a/share-api/src/Share/Postgres/Queries.hs +++ b/share-api/src/Share/Postgres/Queries.hs @@ -273,10 +273,9 @@ listProjectsByUserWithMetadata callerUserId projectOwnerUserId = do owner.handle, owner.name, EXISTS (SELECT FROM org_members WHERE org_members.organization_user_id = owner.id) AS is_org - FROM projects p + FROM projects_by_user_permission(#{callerUserId}, #{ProjectView}) AS project ON project.id = b.project_id JOIN users owner ON owner.id = p.owner_user_id WHERE p.owner_user_id = #{projectOwnerUserId} - AND user_has_project_permission(#{callerUserId}, p.id, #{ProjectView}) ORDER BY p.created_at DESC |] where @@ -876,12 +875,11 @@ listContributorBranchesOfUserAccessibleToCaller contributorUserId mayCallerUserI project_owner.name, EXISTS (SELECT FROM org_members WHERE org_members.organization_user_id = project.owner_user_id) FROM project_branches b - JOIN projects project ON project.id = b.project_id + JOIN projects_by_user_permission(#{mayCallerUserId}, #{ProjectView}) AS project ON project.id = b.project_id JOIN users AS project_owner ON project_owner.id = project.owner_user_id WHERE b.deleted_at IS NULL AND b.contributor_id = #{contributorUserId} - AND user_has_project_permission(#{mayCallerUserId}, b.project_id, #{ProjectView}) |], branchNameFilter, cursorFilter, diff --git a/share-api/src/Share/Postgres/Search/DefinitionSearch/Queries.hs b/share-api/src/Share/Postgres/Search/DefinitionSearch/Queries.hs index 682a445a..18ef87ab 100644 --- a/share-api/src/Share/Postgres/Search/DefinitionSearch/Queries.hs +++ b/share-api/src/Share/Postgres/Search/DefinitionSearch/Queries.hs @@ -401,13 +401,13 @@ globalDefNameCompletionSearch mayCaller mayUserFilter (Query query) limit = do [sql| WITH results(name, tag) AS ( SELECT DISTINCT doc.name, doc.tag FROM global_definition_search_docs doc - JOIN projects p ON p.id = doc.project_id + JOIN projects_by_user_permission(#{mayCaller}, #{ProjectView}) p + ON p.id = doc.project_id WHERE -- Find names which contain the query doc.name ILIKE ('%.' || like_escape(#{query}) || '%') - AND user_has_project_permission(#{mayCaller}, p.id, #{ProjectView}) - ^{filters} + ^{filters} ) SELECT r.name, r.tag FROM results r -- Docs and tests to the bottom, then -- prefer matches where the original query appears (case-matched), @@ -501,11 +501,11 @@ globalDefinitionTokenSearch mayCaller mayUserFilter limit searchTokens preferred queryListRows @(ProjectId, ReleaseId, Name, Hasql.Jsonb) [sql| SELECT doc.project_id, doc.release_id, doc.name, doc.metadata FROM global_definition_search_docs doc - JOIN projects p ON p.id = doc.project_id + JOIN projects_by_user_permission(#{mayCaller}, #{ProjectView}) p + ON p.id = doc.project_id WHERE -- match on search tokens using GIN index. tsquery(#{tsQueryText}) @@ doc.search_tokens - AND user_has_project_permission(#{mayCaller}, p.id, #{ProjectView}) AND (#{preferredArity} IS NULL OR doc.arity >= #{preferredArity}) ^{filters} ^{namesFilter} @@ -619,11 +619,10 @@ globalDefinitionNameSearch mayCaller mayUserFilter limit (Query query) = do queryListRows @(ProjectId, ReleaseId, Name, Hasql.Jsonb) [sql| SELECT doc.project_id, doc.release_id, doc.name, doc.metadata FROM global_definition_search_docs doc - JOIN projects p ON p.id = doc.project_id + JOIN projects_by_user_permission(#{mayCaller}, #{ProjectView}) p ON p.id = doc.project_id WHERE -- We may wish to adjust the similarity threshold before the query. #{query} <% doc.name - AND user_has_project_permission(#{mayCaller}, p.id, #{ProjectView}) ^{filters} -- Score matches by: -- - projects in the catalog diff --git a/share-api/src/Share/Postgres/Tickets/Queries.hs b/share-api/src/Share/Postgres/Tickets/Queries.hs index 76e26e3e..fa9b5e76 100644 --- a/share-api/src/Share/Postgres/Tickets/Queries.hs +++ b/share-api/src/Share/Postgres/Tickets/Queries.hs @@ -251,10 +251,9 @@ listTicketsByUserId callerUserId userId limit mayCursor mayStatusFilter = do ticket.author_id, (SELECT COUNT(*) FROM comments comment WHERE comment.ticket_id = ticket.id AND comment.deleted_at IS NULL) as num_comments FROM tickets AS ticket - JOIN projects AS project ON project.id = ticket.project_id + JOIN projects_by_user_permission(#{callerUserId}, #{ProjectView}) AS project ON project.id = ticket.project_id WHERE ticket.author_id = #{userId} - AND user_has_project_permission(#{callerUserId}, project.id, #{ProjectView}) AND (#{mayStatusFilter} IS NULL OR ticket.status = #{mayStatusFilter}::ticket_status) AND ^{cursorFilter} ORDER BY ticket.updated_at DESC, ticket.id DESC From 8546cd3db661813e0ba22690f22e86b025fa3590 Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 12:27:29 -0800 Subject: [PATCH 5/8] Fix bad function name --- share-api/src/Share/Postgres/Queries.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share-api/src/Share/Postgres/Queries.hs b/share-api/src/Share/Postgres/Queries.hs index 0ef444f2..1e53616e 100644 --- a/share-api/src/Share/Postgres/Queries.hs +++ b/share-api/src/Share/Postgres/Queries.hs @@ -156,7 +156,7 @@ searchProjects caller userIdFilter (Query query) psk limit = do PG.queryListRows @(Project PG.:. PG.Only UserHandle) [PG.sql| SELECT p.id, p.owner_user_id, p.slug, p.summary, p.tags, p.private, p.created_at, p.updated_at, owner.handle - FROM projects_searchable_by_user(#{caller}) p + FROM projects_by_user_permission(#{caller}, #{ProjectView}) p JOIN users owner ON p.owner_user_id = owner.id WHERE p.owner_user_id = #{userId} ORDER BY p.created_at DESC @@ -166,7 +166,7 @@ searchProjects caller userIdFilter (Query query) psk limit = do PG.queryListRows [PG.sql| SELECT p.id, p.owner_user_id, p.slug, p.summary, p.tags, p.private, p.created_at, p.updated_at, owner.handle - FROM websearch_to_tsquery('english', #{query}) AS tokenquery, projects_searchable_by_user(#{caller}) AS p + FROM websearch_to_tsquery('english', #{query}) AS tokenquery, projects_by_user_permission(#{caller}, #{ProjectView}) AS p JOIN users AS owner ON p.owner_user_id = owner.id WHERE (tokenquery @@ p.project_text_document OR p.slug ILIKE ('%' || like_escape(#{query}) || '%')) AND (#{userIdFilter} IS NULL OR p.owner_user_id = #{userIdFilter}) From 4e8fa57a816dfdaa1f089f09a033150c971db2a1 Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 12:27:29 -0800 Subject: [PATCH 6/8] Fix bad copy-pasta --- share-api/src/Share/Postgres/Queries.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share-api/src/Share/Postgres/Queries.hs b/share-api/src/Share/Postgres/Queries.hs index 1e53616e..ad9742c8 100644 --- a/share-api/src/Share/Postgres/Queries.hs +++ b/share-api/src/Share/Postgres/Queries.hs @@ -273,10 +273,10 @@ listProjectsByUserWithMetadata callerUserId projectOwnerUserId = do owner.handle, owner.name, EXISTS (SELECT FROM org_members WHERE org_members.organization_user_id = owner.id) AS is_org - FROM projects_by_user_permission(#{callerUserId}, #{ProjectView}) AS project ON project.id = b.project_id + FROM projects_by_user_permission(#{callerUserId}, #{ProjectView}) AS p JOIN users owner ON owner.id = p.owner_user_id WHERE p.owner_user_id = #{projectOwnerUserId} - ORDER BY p.created_at DESC + ORDER BY p.created_at, p.slug DESC |] where unpackRows :: [Project PG.:. FavData PG.:. ProjectOwner] -> [(Project, FavData, ProjectOwner)] From e3405adeb98b6c91ad8db99af68340954403002d Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 12:27:29 -0800 Subject: [PATCH 7/8] Make ordering more deterministic in transcripts --- share-api/src/Share/Postgres/Queries.hs | 2 +- .../share-apis/projects-flow/out/project-list.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/share-api/src/Share/Postgres/Queries.hs b/share-api/src/Share/Postgres/Queries.hs index ad9742c8..795554b2 100644 --- a/share-api/src/Share/Postgres/Queries.hs +++ b/share-api/src/Share/Postgres/Queries.hs @@ -159,7 +159,7 @@ searchProjects caller userIdFilter (Query query) psk limit = do FROM projects_by_user_permission(#{caller}, #{ProjectView}) p JOIN users owner ON p.owner_user_id = owner.id WHERE p.owner_user_id = #{userId} - ORDER BY p.created_at DESC + ORDER BY p.created_at, p.slug DESC LIMIT #{limit} |] _ -> do diff --git a/transcripts/share-apis/projects-flow/out/project-list.json b/transcripts/share-apis/projects-flow/out/project-list.json index 8e47fc2e..f7dfd133 100644 --- a/transcripts/share-apis/projects-flow/out/project-list.json +++ b/transcripts/share-apis/projects-flow/out/project-list.json @@ -9,8 +9,8 @@ "name": "The Transcript User", "type": "user" }, - "slug": "containers", - "summary": "This is my project", + "slug": "transcriptproject", + "summary": null, "tags": [], "updatedAt": "", "visibility": "private" @@ -24,8 +24,8 @@ "name": "The Transcript User", "type": "user" }, - "slug": "transcriptproject", - "summary": null, + "slug": "containers", + "summary": "This is my project", "tags": [], "updatedAt": "", "visibility": "private" From 6a099bf5d366f6e5ece34d35a238a3eb85097f4f Mon Sep 17 00:00:00 2001 From: Chris Penner Date: Mon, 15 Dec 2025 13:23:50 -0800 Subject: [PATCH 8/8] Fix project ordering --- share-api/src/Share/Postgres/Queries.hs | 4 ++-- .../out/read-maintainer-project-list-after.json | 16 ++++++++-------- .../projects-flow/out/project-list.json | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/share-api/src/Share/Postgres/Queries.hs b/share-api/src/Share/Postgres/Queries.hs index 795554b2..eb461d66 100644 --- a/share-api/src/Share/Postgres/Queries.hs +++ b/share-api/src/Share/Postgres/Queries.hs @@ -159,7 +159,7 @@ searchProjects caller userIdFilter (Query query) psk limit = do FROM projects_by_user_permission(#{caller}, #{ProjectView}) p JOIN users owner ON p.owner_user_id = owner.id WHERE p.owner_user_id = #{userId} - ORDER BY p.created_at, p.slug DESC + ORDER BY p.created_at DESC, p.slug ASC LIMIT #{limit} |] _ -> do @@ -276,7 +276,7 @@ listProjectsByUserWithMetadata callerUserId projectOwnerUserId = do FROM projects_by_user_permission(#{callerUserId}, #{ProjectView}) AS p JOIN users owner ON owner.id = p.owner_user_id WHERE p.owner_user_id = #{projectOwnerUserId} - ORDER BY p.created_at, p.slug DESC + ORDER BY p.created_at DESC, p.slug ASC |] where unpackRows :: [Project PG.:. FavData PG.:. ProjectOwner] -> [(Project, FavData, ProjectOwner)] diff --git a/transcripts/share-apis/project-maintainers/out/read-maintainer-project-list-after.json b/transcripts/share-apis/project-maintainers/out/read-maintainer-project-list-after.json index d64ee06c..0a9d2604 100644 --- a/transcripts/share-apis/project-maintainers/out/read-maintainer-project-list-after.json +++ b/transcripts/share-apis/project-maintainers/out/read-maintainer-project-list-after.json @@ -3,32 +3,32 @@ { "createdAt": "", "isFaved": false, - "numFavs": 1, + "numFavs": 0, "owner": { "handle": "@test", "name": null, "type": "user" }, - "slug": "publictestproject", - "summary": "test project summary", + "slug": "privatetestproject", + "summary": "private summary", "tags": [], "updatedAt": "", - "visibility": "public" + "visibility": "private" }, { "createdAt": "", "isFaved": false, - "numFavs": 0, + "numFavs": 1, "owner": { "handle": "@test", "name": null, "type": "user" }, - "slug": "privatetestproject", - "summary": "private summary", + "slug": "publictestproject", + "summary": "test project summary", "tags": [], "updatedAt": "", - "visibility": "private" + "visibility": "public" } ], "status": [ diff --git a/transcripts/share-apis/projects-flow/out/project-list.json b/transcripts/share-apis/projects-flow/out/project-list.json index f7dfd133..8e47fc2e 100644 --- a/transcripts/share-apis/projects-flow/out/project-list.json +++ b/transcripts/share-apis/projects-flow/out/project-list.json @@ -9,8 +9,8 @@ "name": "The Transcript User", "type": "user" }, - "slug": "transcriptproject", - "summary": null, + "slug": "containers", + "summary": "This is my project", "tags": [], "updatedAt": "", "visibility": "private" @@ -24,8 +24,8 @@ "name": "The Transcript User", "type": "user" }, - "slug": "containers", - "summary": "This is my project", + "slug": "transcriptproject", + "summary": null, "tags": [], "updatedAt": "", "visibility": "private"