diff --git a/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py new file mode 100644 index 00000000000..dbd826a6d85 --- /dev/null +++ b/dojo/db_migrations/0278_global_search_fts_trigram_indexes.py @@ -0,0 +1,96 @@ +""" +Full-text-search and fuzzy-match GIN indexes for global search. + +Adds, on the ten models the Pro global search (``pro/search/``) queries: + +* one weighted ``tsvector`` GIN index per model, built from the same + ``SearchVector`` objects the query annotates with, so the index expression + can never drift from the compiled SQL across Django upgrades; and +* one ``gin_trgm_ops`` index on each model's short display column for the + fuzzy ``%>`` (``__trigram_word_similar``) lookup. + +Each index is declared on its model's ``Meta.indexes``; this migration is the +paired ``AddIndexConcurrently`` that actually builds them (the same house +style as the other functional-index migrations, e.g. 0273). The tsvector +indexes are built from the same ``SearchVector`` objects the query annotates +with, so the index expression cannot drift from the compiled SQL across Django +upgrades. The trigram indexes use ``opclasses`` (a base ``Index`` option), so +they need no ``OpClass`` expression. + +``AddIndexConcurrently`` / ``CREATE EXTENSION`` cannot run inside a +transaction, hence ``atomic = False``. ``TrigramExtension`` is the sole +creator of ``pg_trgm`` here, so its symmetric reverse (dropping the extension +after the trigram indexes are already gone) is correct. +""" + +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.operations import AddIndexConcurrently, TrigramExtension +from django.contrib.postgres.search import SearchVector +from django.db import migrations + +# (model_name, ((column, weight), ...), index_name) -- weighted tsvector GIN. +_FTS_SPECS = ( + ("finding", (("title", "A"), ("description", "B")), "dojo_finding_fts_gin"), + ("product", (("name", "A"), ("description", "B")), "dojo_product_fts_gin"), + ("product_type", (("name", "A"), ("description", "B")), "dojo_product_type_fts_gin"), + ("engagement", (("name", "A"), ("description", "B")), "dojo_engagement_fts_gin"), + ("test", (("title", "A"), ("description", "B")), "dojo_test_fts_gin"), + ("endpoint", (("host", "A"), ("path", "B")), "dojo_endpoint_fts_gin"), + ("location", (("location_value", "A"), ("location_type", "B")), "dojo_location_fts_gin"), + ("finding_template", (("title", "A"), ("description", "B")), "dojo_finding_template_fts_gin"), + ("app_analysis", (("name", "A"),), "dojo_app_analysis_fts_gin"), + ("vulnerability_id", (("vulnerability_id", "A"),), "dojo_vulnerability_id_fts_gin"), +) + +# (model_name, column, index_name) -- gin_trgm_ops fuzzy-match index. Three +# names are abbreviated to stay within the 30-char index-name limit (E033): +# location_value -> locval, finding_template -> findtmpl, vulnerability_id -> vuln_id. +_TRGM_SPECS = ( + ("finding", "title", "dojo_finding_title_trgm"), + ("product", "name", "dojo_product_name_trgm"), + ("product_type", "name", "dojo_product_type_name_trgm"), + ("engagement", "name", "dojo_engagement_name_trgm"), + ("test", "title", "dojo_test_title_trgm"), + ("endpoint", "host", "dojo_endpoint_host_trgm"), + ("location", "location_value", "dojo_location_locval_trgm"), + ("finding_template", "title", "dojo_findtmpl_title_trgm"), + ("app_analysis", "name", "dojo_app_analysis_name_trgm"), + ("vulnerability_id", "vulnerability_id", "dojo_vuln_id_trgm"), +) + + +def _fts_vector(fields): + vector = None + for column, weight in fields: + component = SearchVector(column, weight=weight, config="english") + vector = component if vector is None else vector + component + return vector + + +def _index_operations(): + add_index = [] + for model_name, fields, name in _FTS_SPECS: + add_index.append( + AddIndexConcurrently( + model_name=model_name, + index=GinIndex(_fts_vector(fields), name=name), + ), + ) + for model_name, column, name in _TRGM_SPECS: + add_index.append( + AddIndexConcurrently( + model_name=model_name, + index=GinIndex(fields=[column], opclasses=["gin_trgm_ops"], name=name), + ), + ) + return [TrigramExtension(), *add_index] + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("dojo", "0277_seed_deduplication_execution_mode"), + ] + + operations = _index_operations() diff --git a/dojo/endpoint/models.py b/dojo/endpoint/models.py index f55e3f5f8c0..f1c2f36628f 100644 --- a/dojo/endpoint/models.py +++ b/dojo/endpoint/models.py @@ -5,6 +5,8 @@ import hyperlink from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.exceptions import ValidationError from django.core.validators import validate_ipv46_address from django.db import connection, models @@ -123,6 +125,13 @@ class Meta: Lower("host"), name="idx_ep_product_lower_host", ), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("host", weight="A", config="english") + + SearchVector("path", weight="B", config="english"), + name="dojo_endpoint_fts_gin", + ), + GinIndex(fields=["host"], opclasses=["gin_trgm_ops"], name="dojo_endpoint_host_trgm"), ] def __init__(self, *args, **kwargs): diff --git a/dojo/engagement/models.py b/dojo/engagement/models.py index ec658e4f6f7..5a39deda276 100644 --- a/dojo/engagement/models.py +++ b/dojo/engagement/models.py @@ -2,6 +2,8 @@ from contextlib import suppress from dateutil.relativedelta import relativedelta +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.db import models from django.urls import reverse from django.utils import timezone @@ -96,6 +98,13 @@ class Meta: ordering = ["-target_start"] indexes = [ models.Index(fields=["product", "active"]), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_engagement_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_engagement_name_trgm"), ] def __str__(self): diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 8a55d3df9e6..7442eb1155e 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -10,6 +10,8 @@ from dateutil.parser import parse as datetutilsparse from dateutil.relativedelta import relativedelta from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.urls import reverse @@ -441,6 +443,14 @@ class Finding(BaseModel): class Meta: ordering = ("numerical_severity", "-date", "title", "epss_score", "epss_percentile") indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("title", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_finding_fts_gin", + ), + GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_finding_title_trgm"), + models.Index(fields=["test", "active", "verified"]), models.Index(fields=["test", "is_mitigated"]), @@ -1367,6 +1377,16 @@ class Vulnerability_Id(models.Model): finding = models.ForeignKey("dojo.Finding", editable=False, on_delete=models.CASCADE) vulnerability_id = models.TextField(max_length=50, blank=False, null=False) + class Meta: + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("vulnerability_id", weight="A", config="english"), + name="dojo_vulnerability_id_fts_gin", + ), + GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vuln_id_trgm"), + ] + def __str__(self): return self.vulnerability_id @@ -1506,6 +1526,15 @@ class Finding_Template(models.Model): class Meta: ordering = ["-cwe"] + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("title", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_finding_template_fts_gin", + ), + GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_findtmpl_title_trgm"), + ] def __str__(self): return self.title diff --git a/dojo/location/models.py b/dojo/location/models.py index 6c6b738655d..f341f934eba 100644 --- a/dojo/location/models.py +++ b/dojo/location/models.py @@ -3,6 +3,8 @@ import hashlib from typing import TYPE_CHECKING, Self +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.validators import MinLengthValidator from django.db import transaction from django.db.models import ( @@ -280,6 +282,13 @@ class Meta: indexes = [ Index(fields=["location_type"]), Index(fields=["location_value"]), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("location_value", weight="A", config="english") + + SearchVector("location_type", weight="B", config="english"), + name="dojo_location_fts_gin", + ), + GinIndex(fields=["location_value"], opclasses=["gin_trgm_ops"], name="dojo_location_locval_trgm"), ] diff --git a/dojo/models.py b/dojo/models.py index a58fa3c5a74..9629aa6ed1c 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -7,6 +7,8 @@ import tagulous.admin from django.contrib import admin from django.contrib.auth import get_user_model +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.exceptions import ValidationError from django.db import models from django.db.models import Count @@ -531,6 +533,16 @@ class App_Analysis(models.Model): tags = TagField(blank=True, force_lowercase=True) + class Meta: + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english"), + name="dojo_app_analysis_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_app_analysis_name_trgm"), + ] + def __str__(self): return self.name + " | " + self.product.name diff --git a/dojo/product/models.py b/dojo/product/models.py index d6d872090d2..0811fa80faf 100644 --- a/dojo/product/models.py +++ b/dojo/product/models.py @@ -1,5 +1,7 @@ from decimal import Decimal +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.core.validators import MinValueValidator from django.db import models from django.db.models.functions import Upper @@ -130,6 +132,13 @@ class Meta: # (WHERE UPPER(name) = UPPER(%s)) used when filtering findings by # product name; the plain unique btree on name can't (Sentry DJANGO-D2M). models.Index(Upper("name"), name="dojo_product_upper_name_idx"), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_product_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_product_name_trgm"), ] def __str__(self): diff --git a/dojo/product_type/models.py b/dojo/product_type/models.py index 50bfcc722b4..e9394d7e45a 100644 --- a/dojo/product_type/models.py +++ b/dojo/product_type/models.py @@ -1,3 +1,5 @@ +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.db import models from django.urls import reverse from django.utils.functional import cached_property @@ -25,6 +27,15 @@ class Product_Type(BaseModel): class Meta: ordering = ("name",) + indexes = [ + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("name", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_product_type_fts_gin", + ), + GinIndex(fields=["name"], opclasses=["gin_trgm_ops"], name="dojo_product_type_name_trgm"), + ] def __str__(self): return self.name diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index d1b661e095e..59d06cf30f3 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -773,6 +773,9 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param INSTALLED_APPS = ( "django.contrib.auth", "django.contrib.contenttypes", + # Registers the postgres-specific lookups and index expressions + # (trigram word similarity, tsvector search) used by global search. + "django.contrib.postgres", "django.contrib.sessions", "django.contrib.sites", "django.contrib.messages", diff --git a/dojo/test/models.py b/dojo/test/models.py index 31cefaf52ac..e3676e58686 100644 --- a/dojo/test/models.py +++ b/dojo/test/models.py @@ -2,6 +2,8 @@ from contextlib import suppress from django.conf import settings +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector from django.db import models from django.db.models import Count, Q from django.urls import reverse @@ -81,6 +83,13 @@ class Test(models.Model): class Meta: indexes = [ models.Index(fields=["engagement", "test_type"]), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("title", weight="A", config="english") + + SearchVector("description", weight="B", config="english"), + name="dojo_test_fts_gin", + ), + GinIndex(fields=["title"], opclasses=["gin_trgm_ops"], name="dojo_test_title_trgm"), ] def __init__(self, *args, **kwargs):