feat(search): FTS + trigram GIN indexes for global search

dojo/db_migrations/0278_global_search_fts_trigram_indexes.py

@@ -0,0 +1,96 @@
+"""
+Full-text-search and fuzzy-match GIN indexes for global search.
+
+Adds, on the ten models the Pro global search (``pro/search/``) queries:
+
+* one weighted ``tsvector`` GIN index per model, built from the same
+  ``SearchVector`` objects the query annotates with, so the index expression
+  can never drift from the compiled SQL across Django upgrades; and
+* one ``gin_trgm_ops`` index on each model's short display column for the
+  fuzzy ``%>`` (``__trigram_word_similar``) lookup.
+
+Each index is declared on its model's ``Meta.indexes``; this migration is the
+paired ``AddIndexConcurrently`` that actually builds them (the same house
+style as the other functional-index migrations, e.g. 0273). The tsvector
+indexes are built from the same ``SearchVector`` objects the query annotates
+with, so the index expression cannot drift from the compiled SQL across Django
+upgrades. The trigram indexes use ``opclasses`` (a base ``Index`` option), so
+they need no ``OpClass`` expression.
+
+``AddIndexConcurrently`` / ``CREATE EXTENSION`` cannot run inside a
+transaction, hence ``atomic = False``. ``TrigramExtension`` is the sole
+creator of ``pg_trgm`` here, so its symmetric reverse (dropping the extension
+after the trigram indexes are already gone) is correct.
+"""
+
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.operations import AddIndexConcurrently, TrigramExtension
+from django.contrib.postgres.search import SearchVector
+from django.db import migrations
+
+# (model_name, ((column, weight), ...), index_name) -- weighted tsvector GIN.
+_FTS_SPECS = (
+    ("finding", (("title", "A"), ("description", "B")), "dojo_finding_fts_gin"),
+    ("product", (("name", "A"), ("description", "B")), "dojo_product_fts_gin"),
+    ("product_type", (("name", "A"), ("description", "B")), "dojo_product_type_fts_gin"),
+    ("engagement", (("name", "A"), ("description", "B")), "dojo_engagement_fts_gin"),
+    ("test", (("title", "A"), ("description", "B")), "dojo_test_fts_gin"),
+    ("endpoint", (("host", "A"), ("path", "B")), "dojo_endpoint_fts_gin"),
+    ("location", (("location_value", "A"), ("location_type", "B")), "dojo_location_fts_gin"),
+    ("finding_template", (("title", "A"), ("description", "B")), "dojo_finding_template_fts_gin"),
+    ("app_analysis", (("name", "A"),), "dojo_app_analysis_fts_gin"),
+    ("vulnerability_id", (("vulnerability_id", "A"),), "dojo_vulnerability_id_fts_gin"),
+)
+
+# (model_name, column, index_name) -- gin_trgm_ops fuzzy-match index. Three
+# names are abbreviated to stay within the 30-char index-name limit (E033):
+# location_value -> locval, finding_template -> findtmpl, vulnerability_id -> vuln_id.
+_TRGM_SPECS = (
+    ("finding", "title", "dojo_finding_title_trgm"),
+    ("product", "name", "dojo_product_name_trgm"),
+    ("product_type", "name", "dojo_product_type_name_trgm"),
+    ("engagement", "name", "dojo_engagement_name_trgm"),
+    ("test", "title", "dojo_test_title_trgm"),
+    ("endpoint", "host", "dojo_endpoint_host_trgm"),
+    ("location", "location_value", "dojo_location_locval_trgm"),
+    ("finding_template", "title", "dojo_findtmpl_title_trgm"),
+    ("app_analysis", "name", "dojo_app_analysis_name_trgm"),
+    ("vulnerability_id", "vulnerability_id", "dojo_vuln_id_trgm"),
+)
+
+
+def _fts_vector(fields):
+    vector = None
+    for column, weight in fields:
+        component = SearchVector(column, weight=weight, config="english")
+        vector = component if vector is None else vector + component
+    return vector
+
+
+def _index_operations():
+    add_index = []
+    for model_name, fields, name in _FTS_SPECS:
+        add_index.append(
+            AddIndexConcurrently(
+                model_name=model_name,
+                index=GinIndex(_fts_vector(fields), name=name),
+            ),
+        )
+    for model_name, column, name in _TRGM_SPECS:
+        add_index.append(
+            AddIndexConcurrently(
+                model_name=model_name,
+                index=GinIndex(fields=[column], opclasses=["gin_trgm_ops"], name=name),
+            ),
+        )
+    return [TrigramExtension(), *add_index]
+
+
+class Migration(migrations.Migration):
+    atomic = False
+
+    dependencies = [
+        ("dojo", "0277_seed_deduplication_execution_mode"),
+    ]
+
+    operations = _index_operations()

dojo/endpoint/models.py

@@ -5,6 +5,8 @@
 
 import hyperlink
 from django.conf import settings
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.core.exceptions import ValidationError
 from django.core.validators import validate_ipv46_address
 from django.db import connection, models
@@ -123,6 +125,13 @@ class Meta:
                 Lower("host"),
                 name="idx_ep_product_lower_host",
             ),
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("host", weight="A", config="english")
+                + SearchVector("path", weight="B", config="english"),
+                name="dojo_endpoint_fts_gin",
+            ),
+            GinIndex(fields=["host"], opclasses=["gin_trgm_ops"], name="dojo_endpoint_host_trgm"),
         ]
 
     def __init__(self, *args, **kwargs):

dojo/engagement/models.py

@@ -2,6 +2,8 @@
 from contextlib import suppress
 
 from dateutil.relativedelta import relativedelta
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
@@ -96,6 +98,13 @@ class Meta:
         ordering = ["-target_start"]
         indexes = [
             models.Index(fields=["product", "active"]),
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("name", weight="A", config="english")
+                + SearchVector("description", weight="B", config="english"),
+                name="dojo_engagement_fts_gin",
+            ),
+            GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_engagement_name_trgm"),
         ]
 
     def __str__(self):

dojo/finding/models.py

@@ -10,6 +10,8 @@
 from dateutil.parser import parse as datetutilsparse
 from dateutil.relativedelta import relativedelta
 from django.conf import settings
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.urls import reverse
@@ -441,6 +443,14 @@ class Finding(BaseModel):
     class Meta:
         ordering = ("numerical_severity", "-date", "title", "epss_score", "epss_percentile")
         indexes = [
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("title", weight="A", config="english")
+                + SearchVector("description", weight="B", config="english"),
+                name="dojo_finding_fts_gin",
+            ),
+            GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_finding_title_trgm"),
+
             models.Index(fields=["test", "active", "verified"]),
 
             models.Index(fields=["test", "is_mitigated"]),
@@ -1367,6 +1377,16 @@ class Vulnerability_Id(models.Model):
     finding = models.ForeignKey("dojo.Finding", editable=False, on_delete=models.CASCADE)
     vulnerability_id = models.TextField(max_length=50, blank=False, null=False)
 
+    class Meta:
+        indexes = [
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("vulnerability_id", weight="A", config="english"),
+                name="dojo_vulnerability_id_fts_gin",
+            ),
+            GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vuln_id_trgm"),
+        ]
+
     def __str__(self):
         return self.vulnerability_id
 
@@ -1506,6 +1526,15 @@ class Finding_Template(models.Model):
 
     class Meta:
         ordering = ["-cwe"]
+        indexes = [
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("title", weight="A", config="english")
+                + SearchVector("description", weight="B", config="english"),
+                name="dojo_finding_template_fts_gin",
+            ),
+            GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_findtmpl_title_trgm"),
+        ]
 
     def __str__(self):
         return self.title

dojo/location/models.py

@@ -3,6 +3,8 @@
 import hashlib
 from typing import TYPE_CHECKING, Self
 
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.core.validators import MinLengthValidator
 from django.db import transaction
 from django.db.models import (
@@ -280,6 +282,13 @@ class Meta:
         indexes = [
             Index(fields=["location_type"]),
             Index(fields=["location_value"]),
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("location_value", weight="A", config="english")
+                + SearchVector("location_type", weight="B", config="english"),
+                name="dojo_location_fts_gin",
+            ),
+            GinIndex(fields=["location_value"], opclasses=["gin_trgm_ops"], name="dojo_location_locval_trgm"),
         ]
 
 

dojo/models.py

@@ -7,6 +7,8 @@
 import tagulous.admin
 from django.contrib import admin
 from django.contrib.auth import get_user_model
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.db.models import Count
@@ -531,6 +533,16 @@ class App_Analysis(models.Model):
 
     tags = TagField(blank=True, force_lowercase=True)
 
+    class Meta:
+        indexes = [
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("name", weight="A", config="english"),
+                name="dojo_app_analysis_fts_gin",
+            ),
+            GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_app_analysis_name_trgm"),
+        ]
+
     def __str__(self):
         return self.name + " | " + self.product.name
 

dojo/product/models.py

@@ -1,5 +1,7 @@
 from decimal import Decimal
 
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.core.validators import MinValueValidator
 from django.db import models
 from django.db.models.functions import Upper
@@ -130,6 +132,13 @@ class Meta:
             # (WHERE UPPER(name) = UPPER(%s)) used when filtering findings by
             # product name; the plain unique btree on name can't (Sentry DJANGO-D2M).
             models.Index(Upper("name"), name="dojo_product_upper_name_idx"),
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("name", weight="A", config="english")
+                + SearchVector("description", weight="B", config="english"),
+                name="dojo_product_fts_gin",
+            ),
+            GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_product_name_trgm"),
         ]
 
     def __str__(self):

dojo/product_type/models.py

@@ -1,3 +1,5 @@
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.db import models
 from django.urls import reverse
 from django.utils.functional import cached_property
@@ -25,6 +27,15 @@ class Product_Type(BaseModel):
 
     class Meta:
         ordering = ("name",)
+        indexes = [
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("name", weight="A", config="english")
+                + SearchVector("description", weight="B", config="english"),
+                name="dojo_product_type_fts_gin",
+            ),
+            GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_product_type_name_trgm"),
+        ]
 
     def __str__(self):
         return self.name

dojo/settings/settings.dist.py

@@ -773,6 +773,9 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param
 INSTALLED_APPS = (
     "django.contrib.auth",
     "django.contrib.contenttypes",
+    # Registers the postgres-specific lookups and index expressions
+    # (trigram word similarity, tsvector search) used by global search.
+    "django.contrib.postgres",
     "django.contrib.sessions",
     "django.contrib.sites",
     "django.contrib.messages",

dojo/test/models.py

@@ -2,6 +2,8 @@
 from contextlib import suppress
 
 from django.conf import settings
+from django.contrib.postgres.indexes import GinIndex
+from django.contrib.postgres.search import SearchVector
 from django.db import models
 from django.db.models import Count, Q
 from django.urls import reverse
@@ -81,6 +83,13 @@ class Test(models.Model):
     class Meta:
         indexes = [
             models.Index(fields=["engagement", "test_type"]),
+            # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match.
+            GinIndex(
+                SearchVector("title", weight="A", config="english")
+                + SearchVector("description", weight="B", config="english"),
+                name="dojo_test_fts_gin",
+            ),
+            GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_test_title_trgm"),
         ]
 
     def __init__(self, *args, **kwargs):