From 1f941e9aade91688d2c240d321cb606a9876c780 Mon Sep 17 00:00:00 2001 From: blakeaowens Date: Fri, 10 Jul 2026 17:00:42 -0500 Subject: [PATCH 1/3] feat(search): add FTS + trigram GIN indexes for global search Adds a weighted tsvector GIN index and a gin_trgm_ops index on the short display column of the ten models the Pro global search queries (finding, product, product_type, engagement, test, endpoint, location, finding_template, app_analysis, vulnerability_id), plus the pg_trgm extension. The indexes are built from SearchVector/OpClass ORM objects so the index expression tracks the query's compiled SQL across Django upgrades, and are database-only (SeparateDatabaseAndState with no state operations) since they are search-performance indexes, not part of any model's public schema. atomic=False because CONCURRENTLY / CREATE EXTENSION cannot run in a transaction. Co-Authored-By: Claude Opus 4.8 --- .../0278_global_search_fts_trigram_indexes.py | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 dojo/db_migrations/0278_global_search_fts_trigram_indexes.py diff --git a/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py new file mode 100644 index 00000000000..b17c0ed6558 --- /dev/null +++ b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py @@ -0,0 +1,101 @@ +""" +Full-text-search and fuzzy-match GIN indexes for global search. + +Adds, on the ten models the Pro global search (``pro/search/``) queries: + +* one weighted ``tsvector`` GIN index per model, built from the same + ``SearchVector`` objects the query annotates with, so the index expression + can never drift from the compiled SQL across Django upgrades; and +* one ``gin_trgm_ops`` index on each model's short display column for the + fuzzy ``%>`` (``__trigram_word_similar``) lookup. + +The indexes are database-only (wrapped in ``SeparateDatabaseAndState`` with no +state operations, so they stay out of every model's ``Meta.indexes``): they +exist purely for global-search performance and are not part of any model's +public schema. Building them from ``SearchVector``/``OpClass`` ORM objects +(rather than raw SQL) is what keeps the index expression from drifting away +from the query's compiled SQL across Django upgrades. + +``AddIndexConcurrently`` / ``CREATE EXTENSION`` cannot run inside a +transaction, hence ``atomic = False``. ``TrigramExtension`` is the sole +creator of ``pg_trgm`` here, so its symmetric reverse (dropping the extension +after the trigram indexes are already gone) is correct. +""" + +from django.contrib.postgres.indexes import GinIndex, OpClass +from django.contrib.postgres.operations import AddIndexConcurrently, TrigramExtension +from django.contrib.postgres.search import SearchVector +from django.db import migrations +from django.db.models import F + +# (model_name, ((column, weight), ...), index_name) -- weighted tsvector GIN. +_FTS_SPECS = ( + ("finding", (("title", "A"), ("description", "B")), "dojo_finding_fts_gin"), + ("product", (("name", "A"), ("description", "B")), "dojo_product_fts_gin"), + ("product_type", (("name", "A"), ("description", "B")), "dojo_product_type_fts_gin"), + ("engagement", (("name", "A"), ("description", "B")), "dojo_engagement_fts_gin"), + ("test", (("title", "A"), ("description", "B")), "dojo_test_fts_gin"), + ("endpoint", (("host", "A"), ("path", "B")), "dojo_endpoint_fts_gin"), + ("location", (("location_value", "A"), ("location_type", "B")), "dojo_location_fts_gin"), + ("finding_template", (("title", "A"), ("description", "B")), "dojo_finding_template_fts_gin"), + ("app_analysis", (("name", "A"),), "dojo_app_analysis_fts_gin"), + ("vulnerability_id", (("vulnerability_id", "A"),), "dojo_vulnerability_id_fts_gin"), +) + +# (model_name, column, index_name) -- gin_trgm_ops fuzzy-match index. Three +# names are abbreviated to stay within the 30-char index-name limit (E033): +# location_value -> locval, finding_template -> findtmpl, vulnerability_id -> vuln_id. +_TRGM_SPECS = ( + ("finding", "title", "dojo_finding_title_trgm"), + ("product", "name", "dojo_product_name_trgm"), + ("product_type", "name", "dojo_product_type_name_trgm"), + ("engagement", "name", "dojo_engagement_name_trgm"), + ("test", "title", "dojo_test_title_trgm"), + ("endpoint", "host", "dojo_endpoint_host_trgm"), + ("location", "location_value", "dojo_location_locval_trgm"), + ("finding_template", "title", "dojo_findtmpl_title_trgm"), + ("app_analysis", "name", "dojo_app_analysis_name_trgm"), + ("vulnerability_id", "vulnerability_id", "dojo_vuln_id_trgm"), +) + + +def _fts_vector(fields): + vector = None + for column, weight in fields: + component = SearchVector(column, weight=weight, config="english") + vector = component if vector is None else vector + component + return vector + + +def _index_operations(): + add_index = [] + for model_name, fields, name in _FTS_SPECS: + add_index.append( + AddIndexConcurrently( + model_name=model_name, + index=GinIndex(_fts_vector(fields), name=name), + ), + ) + for model_name, column, name in _TRGM_SPECS: + add_index.append( + AddIndexConcurrently( + model_name=model_name, + index=GinIndex(OpClass(F(column), name="gin_trgm_ops"), name=name), + ), + ) + # database_operations only: create the indexes without recording them in + # model state, so no model's Meta.indexes has to carry a search-only index. + return [ + TrigramExtension(), + migrations.SeparateDatabaseAndState(database_operations=add_index, state_operations=[]), + ] + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("dojo", "0277_seed_deduplication_execution_mode"), + ] + + operations = _index_operations() From 1ca3789a29c27eb1d44e42ec18d219bd3063e538 Mon Sep 17 00:00:00 2001 From: blakeaowens Date: Fri, 10 Jul 2026 17:46:20 -0500 Subject: [PATCH 2/3] fix(search): build trigram indexes with opclasses, not OpClass The 0278 trigram GIN indexes used a django.contrib.postgres OpClass expression, which renders to invalid SQL ("syntax error at or near gin_trgm_ops") unless django.contrib.postgres is in INSTALLED_APPS, and it is not in the OSS standalone settings. Switch to the base Index opclasses option, which renders the identical `USING gin (col gin_trgm_ops)` SQL without requiring that app. The tsvector indexes are unaffected (SearchVector renders without the app). Co-Authored-By: Claude Opus 4.8 --- .../0278_global_search_fts_trigram_indexes.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py index b17c0ed6558..603a8c86b0f 100644 --- a/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py +++ b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py @@ -12,9 +12,14 @@ The indexes are database-only (wrapped in ``SeparateDatabaseAndState`` with no state operations, so they stay out of every model's ``Meta.indexes``): they exist purely for global-search performance and are not part of any model's -public schema. Building them from ``SearchVector``/``OpClass`` ORM objects -(rather than raw SQL) is what keeps the index expression from drifting away -from the query's compiled SQL across Django upgrades. +public schema. The tsvector indexes are built from the same ``SearchVector`` +objects the query annotates with (rather than raw SQL), so the index +expression cannot drift from the query's compiled SQL across Django upgrades. + +The trigram indexes use ``opclasses`` (a base ``Index`` option) rather than a +``django.contrib.postgres`` ``OpClass`` expression, so they render without +requiring that app in ``INSTALLED_APPS`` (only the Pro query side needs it, +for the ``__trigram_word_similar`` lookup). ``AddIndexConcurrently`` / ``CREATE EXTENSION`` cannot run inside a transaction, hence ``atomic = False``. ``TrigramExtension`` is the sole @@ -22,11 +27,10 @@ after the trigram indexes are already gone) is correct. """ -from django.contrib.postgres.indexes import GinIndex, OpClass +from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.operations import AddIndexConcurrently, TrigramExtension from django.contrib.postgres.search import SearchVector from django.db import migrations -from django.db.models import F # (model_name, ((column, weight), ...), index_name) -- weighted tsvector GIN. _FTS_SPECS = ( @@ -80,7 +84,7 @@ def _index_operations(): add_index.append( AddIndexConcurrently( model_name=model_name, - index=GinIndex(OpClass(F(column), name="gin_trgm_ops"), name=name), + index=GinIndex(fields=[column], opclasses=["gin_trgm_ops"], name=name), ), ) # database_operations only: create the indexes without recording them in From f9b971805f0de9c1364f5a3cdc5f2cba6ce82be6 Mon Sep 17 00:00:00 2001 From: blakeaowens Date: Fri, 10 Jul 2026 18:25:33 -0500 Subject: [PATCH 3/3] refactor(search): declare FTS/trigram indexes on model Meta Move the global-search GIN indexes from a database-only migration into each model's Meta.indexes (house style, matching the other functional-index migrations), and register django.contrib.postgres in INSTALLED_APPS so the SearchVector index expressions and trigram lookups are supported. 0278 becomes a plain AddIndexConcurrently migration matching model state. Indexes: weighted tsvector FTS + gin_trgm_ops trigram on finding, product, product_type, engagement, test, endpoint, location, finding_template, app_analysis, vulnerability_id. Co-Authored-By: Claude Opus 4.8 --- .../0278_global_search_fts_trigram_indexes.py | 25 +++++----------- dojo/endpoint/models.py | 9 ++++++ dojo/engagement/models.py | 9 ++++++ dojo/finding/models.py | 29 +++++++++++++++++++ dojo/location/models.py | 9 ++++++ dojo/models.py | 12 ++++++++ dojo/product/models.py | 9 ++++++ dojo/product_type/models.py | 11 +++++++ dojo/settings/settings.dist.py | 3 ++ dojo/test/models.py | 9 ++++++ 10 files changed, 108 insertions(+), 17 deletions(-) diff --git a/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py index 603a8c86b0f..dbd826a6d85 100644 --- a/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py +++ b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py @@ -9,17 +9,13 @@ * one ``gin_trgm_ops`` index on each model's short display column for the fuzzy ``%>`` (``__trigram_word_similar``) lookup. -The indexes are database-only (wrapped in ``SeparateDatabaseAndState`` with no -state operations, so they stay out of every model's ``Meta.indexes``): they -exist purely for global-search performance and are not part of any model's -public schema. The tsvector indexes are built from the same ``SearchVector`` -objects the query annotates with (rather than raw SQL), so the index -expression cannot drift from the query's compiled SQL across Django upgrades. - -The trigram indexes use ``opclasses`` (a base ``Index`` option) rather than a -``django.contrib.postgres`` ``OpClass`` expression, so they render without -requiring that app in ``INSTALLED_APPS`` (only the Pro query side needs it, -for the ``__trigram_word_similar`` lookup). +Each index is declared on its model's ``Meta.indexes``; this migration is the +paired ``AddIndexConcurrently`` that actually builds them (the same house +style as the other functional-index migrations, e.g. 0273). The tsvector +indexes are built from the same ``SearchVector`` objects the query annotates +with, so the index expression cannot drift from the compiled SQL across Django +upgrades. The trigram indexes use ``opclasses`` (a base ``Index`` option), so +they need no ``OpClass`` expression. ``AddIndexConcurrently`` / ``CREATE EXTENSION`` cannot run inside a transaction, hence ``atomic = False``. ``TrigramExtension`` is the sole @@ -87,12 +83,7 @@ def _index_operations(): index=GinIndex(fields=[column], opclasses=["gin_trgm_ops"], name=name), ), ) - # database_operations only: create the indexes without recording them in - # model state, so no model's Meta.indexes has to carry a search-only index. - return [ - TrigramExtension(), - migrations.SeparateDatabaseAndState(database_operations=add_index, state_operations=[]), - ] + return [TrigramExtension(), *add_index] class Migration(migrations.Migration): diff --git a/dojo/endpoint/models.py b/dojo/endpoint/models.py index f55e3f5f8c0..f1c2f36628f 100644 --- a/dojo/endpoint/models.py +++ b/dojo/endpoint/models.py @@ -5,6 +5,8 @@ import hyperlink from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.exceptions import ValidationError from django.core.validators import validate_ipv46_address from django.db import connection, models @@ -123,6 +125,13 @@ class Meta: Lower("host"), name="idx_ep_product_lower_host", ), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("host", weight="A", config="english") + + SearchVector("path", weight="B", config="english"), + name="dojo_endpoint_fts_gin", + ), + GinIndex(fields=["host"], opclasses=["gin_trgm_ops"], name="dojo_endpoint_host_trgm"), ] def __init__(self, *args, **kwargs): diff --git a/dojo/engagement/models.py b/dojo/engagement/models.py index ec658e4f6f7..5a39deda276 100644 --- a/dojo/engagement/models.py +++ b/dojo/engagement/models.py @@ -2,6 +2,8 @@ from contextlib import suppress from dateutil.relativedelta import relativedelta +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.db import models from django.urls import reverse from django.utils import timezone @@ -96,6 +98,13 @@ class Meta: ordering = ["-target_start"] indexes = [ models.Index(fields=["product", "active"]), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_engagement_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_engagement_name_trgm"), ] def __str__(self): diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 8a55d3df9e6..7442eb1155e 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -10,6 +10,8 @@ from dateutil.parser import parse as datetutilsparse from dateutil.relativedelta import relativedelta from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.urls import reverse @@ -441,6 +443,14 @@ class Finding(BaseModel): class Meta: ordering = ("numerical_severity", "-date", "title", "epss_score", "epss_percentile") indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("title", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_finding_fts_gin", + ), + GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_finding_title_trgm"), + models.Index(fields=["test", "active", "verified"]), models.Index(fields=["test", "is_mitigated"]), @@ -1367,6 +1377,16 @@ class Vulnerability_Id(models.Model): finding = models.ForeignKey("dojo.Finding", editable=False, on_delete=models.CASCADE) vulnerability_id = models.TextField(max_length=50, blank=False, null=False) + class Meta: + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("vulnerability_id", weight="A", config="english"), + name="dojo_vulnerability_id_fts_gin", + ), + GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vuln_id_trgm"), + ] + def __str__(self): return self.vulnerability_id @@ -1506,6 +1526,15 @@ class Finding_Template(models.Model): class Meta: ordering = ["-cwe"] + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("title", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_finding_template_fts_gin", + ), + GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_findtmpl_title_trgm"), + ] def __str__(self): return self.title diff --git a/dojo/location/models.py b/dojo/location/models.py index 6c6b738655d..f341f934eba 100644 --- a/dojo/location/models.py +++ b/dojo/location/models.py @@ -3,6 +3,8 @@ import hashlib from typing import TYPE_CHECKING, Self +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.validators import MinLengthValidator from django.db import transaction from django.db.models import ( @@ -280,6 +282,13 @@ class Meta: indexes = [ Index(fields=["location_type"]), Index(fields=["location_value"]), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("location_value", weight="A", config="english") + + SearchVector("location_type", weight="B", config="english"), + name="dojo_location_fts_gin", + ), + GinIndex(fields=["location_value"], opclasses=["gin_trgm_ops"], name="dojo_location_locval_trgm"), ] diff --git a/dojo/models.py b/dojo/models.py index a58fa3c5a74..9629aa6ed1c 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -7,6 +7,8 @@ import tagulous.admin from django.contrib import admin from django.contrib.auth import get_user_model +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.exceptions import ValidationError from django.db import models from django.db.models import Count @@ -531,6 +533,16 @@ class App_Analysis(models.Model): tags = TagField(blank=True, force_lowercase=True) + class Meta: + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english"), + name="dojo_app_analysis_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_app_analysis_name_trgm"), + ] + def __str__(self): return self.name + " | " + self.product.name diff --git a/dojo/product/models.py b/dojo/product/models.py index d6d872090d2..0811fa80faf 100644 --- a/dojo/product/models.py +++ b/dojo/product/models.py @@ -1,5 +1,7 @@ from decimal import Decimal +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.validators import MinValueValidator from django.db import models from django.db.models.functions import Upper @@ -130,6 +132,13 @@ class Meta: # (WHERE UPPER(name) = UPPER(%s)) used when filtering findings by # product name; the plain unique btree on name can't (Sentry DJANGO-D2M). models.Index(Upper("name"), name="dojo_product_upper_name_idx"), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_product_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_product_name_trgm"), ] def __str__(self): diff --git a/dojo/product_type/models.py b/dojo/product_type/models.py index 50bfcc722b4..e9394d7e45a 100644 --- a/dojo/product_type/models.py +++ b/dojo/product_type/models.py @@ -1,3 +1,5 @@ +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.db import models from django.urls import reverse from django.utils.functional import cached_property @@ -25,6 +27,15 @@ class Product_Type(BaseModel): class Meta: ordering = ("name",) + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_product_type_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_product_type_name_trgm"), + ] def __str__(self): return self.name diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index d1b661e095e..59d06cf30f3 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -773,6 +773,9 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param INSTALLED_APPS = ( "django.contrib.auth", "django.contrib.contenttypes", + # Registers the postgres-specific lookups and index expressions + # (trigram word similarity, tsvector search) used by global search. + "django.contrib.postgres", "django.contrib.sessions", "django.contrib.sites", "django.contrib.messages", diff --git a/dojo/test/models.py b/dojo/test/models.py index 31cefaf52ac..e3676e58686 100644 --- a/dojo/test/models.py +++ b/dojo/test/models.py @@ -2,6 +2,8 @@ from contextlib import suppress from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.db import models from django.db.models import Count, Q from django.urls import reverse @@ -81,6 +83,13 @@ class Test(models.Model): class Meta: indexes = [ models.Index(fields=["engagement", "test_type"]), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("title", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_test_fts_gin", + ), + GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_test_title_trgm"), ] def __init__(self, *args, **kwargs):