Cross Reference Feature and basic mappings

.gitignore

@@ -36,3 +36,5 @@ server/vector_index.pkl
 
 # Temporary files for magic items update
 temp_magic_items/
+references_report.json
+sources_report.json

api_v2/crossreference_utils.py

@@ -0,0 +1,36 @@
+"""
+Re-export crossreference logic from scripts.crossreference.core for backwards compatibility.
+
+Management commands find_crossreference_candidates and delete_crossreferences delegate to
+scripts/crossreference/find_candidates.py and delete_crossreferences.py.
+"""
+
+from scripts.crossreference.core import (
+    REFERENCE_MODEL_NAMES,
+    build_crossreference_reports,
+    build_object_url,
+    get_all_crossreferences_for_document,
+    get_crossreferences_by_source_document,
+    get_document,
+    get_reference_candidates_for_document,
+    get_reference_models_and_filters_for_document,
+    get_source_models_and_filters_for_document,
+    identify_crossreferences_from_text,
+    load_blacklist,
+    write_crossreference_report_files,
+)
+
+__all__ = [
+    "REFERENCE_MODEL_NAMES",
+    "build_crossreference_reports",
+    "build_object_url",
+    "get_all_crossreferences_for_document",
+    "get_crossreferences_by_source_document",
+    "get_document",
+    "get_reference_candidates_for_document",
+    "get_reference_models_and_filters_for_document",
+    "get_source_models_and_filters_for_document",
+    "identify_crossreferences_from_text",
+    "load_blacklist",
+    "write_crossreference_report_files",
+]

api_v2/management/commands/apply_crossreferences.py

@@ -0,0 +1,66 @@
+"""
+Apply suggested cross-references to the database from text-matching.
+
+Delegates to scripts/crossreference/apply_crossreferences.py.
+"""
+
+from django.core.management.base import BaseCommand, CommandError
+
+from scripts.crossreference.apply_crossreferences import run as run_apply_crossreferences
+
+
+class Command(BaseCommand):
+    help = (
+        "Create CrossReference rows from text-matching for the given document. "
+        "Use --dry-run to preview; use --replace to delete existing crossreferences for "
+        "the document before creating."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--document",
+            type=str,
+            required=True,
+            help="Document key (e.g. srd-2024).",
+        )
+        parser.add_argument(
+            "--source-blacklist",
+            type=str,
+            default=None,
+            help="Path to file with source keys to exclude (one per line).",
+        )
+        parser.add_argument(
+            "--reference-blacklist",
+            type=str,
+            default=None,
+            help="Path to file with reference keys to exclude (one per line).",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Only print what would be created; do not create.",
+        )
+        parser.add_argument(
+            "--replace",
+            action="store_true",
+            help="Delete existing crossreferences whose source is in this document before creating.",
+        )
+
+    def handle(self, *args, **options):
+        doc_key = options["document"]
+        try:
+            run_apply_crossreferences(
+                doc_key,
+                source_blacklist_path=options["source_blacklist"],
+                reference_blacklist_path=options["reference_blacklist"],
+                dry_run=options["dry_run"],
+                replace_existing=options["replace"],
+                stdout=self.stdout,
+                style_success=self.style.SUCCESS,
+            )
+        except Exception as e:
+            if type(e).__name__ == "DoesNotExist" or "not found" in str(e).lower():
+                raise CommandError(f"Document not found: {doc_key} ({e})") from e
+            if isinstance(e, ValueError):
+                raise CommandError(str(e)) from e
+            raise

api_v2/management/commands/delete_crossreferences.py

@@ -0,0 +1,65 @@
+"""
+Delete groups of crossreferences by source document.
+
+Delegates to scripts/crossreference/delete_crossreferences.py.
+"""
+
+from django.core.management.base import BaseCommand, CommandError
+
+from scripts.crossreference.delete_crossreferences import run as run_delete_crossreferences
+
+
+class Command(BaseCommand):
+    help = (
+        "Delete CrossReference rows whose source object belongs to the given "
+        "document. Optional: restrict by source model; protect sources/references "
+        "with blacklists; use --dry-run to preview."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--document",
+            type=str,
+            required=True,
+            help="Document key; delete crossreferences whose source is in this document.",
+        )
+        parser.add_argument(
+            "--model",
+            type=str,
+            default=None,
+            help="If set, only delete crossreferences whose source is this model (e.g. Spell, Item).",
+        )
+        parser.add_argument(
+            "--source-blacklist",
+            type=str,
+            default=None,
+            help="Path to file; do not delete crossreferences whose source key is in this set.",
+        )
+        parser.add_argument(
+            "--reference-blacklist",
+            type=str,
+            default=None,
+            help="Path to file; do not delete crossreferences whose reference key is in this set.",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Only print what would be deleted; do not delete.",
+        )
+
+    def handle(self, *args, **options):
+        doc_key = options["document"]
+        try:
+            run_delete_crossreferences(
+                doc_key,
+                model_name=options["model"],
+                source_blacklist_path=options["source_blacklist"],
+                reference_blacklist_path=options["reference_blacklist"],
+                dry_run=options["dry_run"],
+                stdout=self.stdout,
+                style_success=self.style.SUCCESS,
+            )
+        except Exception as e:
+            if type(e).__name__ == "DoesNotExist" or "not found" in str(e).lower():
+                raise CommandError(f"Document not found: {doc_key} ({e})") from e
+            raise

api_v2/management/commands/export.py

@@ -127,7 +127,7 @@ def handle(self, *args, **options) -> None:
                     CHILD_MODEL_NAMES = ['SpeciesTrait', 'FeatBenefit', 'BackgroundBenefit', 'ClassFeatureItem', 'SpellCastingOption','CreatureAction', 'CreatureTrait']
                     CHILD_CHILD_MODEL_NAMES = ['CreatureActionAttack']
 
-                    if model._meta.app_label == 'api_v2' and model.__name__ not in SKIPPED_MODEL_NAMES and model.__name__ not in CONCEPT_MODEL_NAMES:
+                    if model._meta.app_label == 'api_v2' and model.__name__ not in SKIPPED_MODEL_NAMES:
                         modelq=None
                         if model.__name__ in CHILD_CHILD_MODEL_NAMES:
                             modelq = model.objects.filter(parent__parent__document=doc).order_by('pk')
@@ -142,7 +142,15 @@ def handle(self, *args, **options) -> None:
                             doc_key=doc.key,
                             base_path=options['dir'],
                             format=options['format'])
-                        write_queryset_data(model_path, modelq, format=options['format'])
+                        # CrossReference fixtures use natural keys for ContentType (loaddata supports them).
+                        # Always write CrossReference so stale rows (e.g. blacklisted refs) are removed from the fixture.
+                        write_queryset_data(
+                            model_path,
+                            modelq,
+                            format=options['format'],
+                            use_natural_foreign_keys=(model.__name__ == 'CrossReference'),
+                            always_write_if_empty=(model.__name__ == 'CrossReference'),
+                        )
 
                 self.stdout.write(self.style.SUCCESS(
                     'Wrote {} to {}'.format(doc.key, doc_path)))
@@ -182,18 +190,29 @@ def get_filepath_by_model(model_name, app_label, pub_key=None, doc_key=None, bas
             return "/".join((base_path, root_folder_name, doc_key, model_name+file_ext))
 
 
-def write_queryset_data(filepath, queryset, format='json'):
-    if queryset.count() > 0:
+def write_queryset_data(filepath, queryset, format='json', use_natural_foreign_keys=False, always_write_if_empty=False):
+    count = queryset.count()
+    if count > 0 or always_write_if_empty:
         dir = os.path.dirname(filepath)
         if not os.path.exists(dir):
             os.makedirs(dir)
 
         output_filepath = filepath
 
         with open(output_filepath, 'w', encoding='utf-8') as f:
-            if format=='json':
-                serializers.serialize("json", queryset, indent=2, stream=f, sort_keys=True)
-            if format=='csv':
+            if format == 'json':
+                if count > 0:
+                    serializers.serialize(
+                        "json",
+                        queryset,
+                        indent=2,
+                        stream=f,
+                        sort_keys=True,
+                        use_natural_foreign_keys=use_natural_foreign_keys,
+                    )
+                else:
+                    f.write("[]")
+            if format == 'csv' and count > 0:
                 # Create headers:
                 fieldnames = []
                 for field in queryset.first().__dict__.keys():

api_v2/management/commands/find_crossreference_candidates.py

@@ -0,0 +1,67 @@
+"""
+Find objects in a document that are candidates for adding crossreferences.
+
+Delegates to scripts/crossreference/find_candidates.py.
+"""
+
+from django.core.management.base import BaseCommand, CommandError
+
+from scripts.crossreference.find_candidates import run as run_find_candidates
+
+
+class Command(BaseCommand):
+    help = (
+        "List objects with descriptions in a document that are candidates for "
+        "adding crossreferences. Output to console."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--document",
+            type=str,
+            required=True,
+            help="Document key (e.g. srd-2014).",
+        )
+        parser.add_argument(
+            "--source-blacklist",
+            type=str,
+            default=None,
+            help="Path to file with source keys to exclude (one per line).",
+        )
+        parser.add_argument(
+            "--reference-blacklist",
+            type=str,
+            default=None,
+            help="Path to file with reference keys to exclude (one per line).",
+        )
+        parser.add_argument(
+            "--sources-report",
+            type=str,
+            default=None,
+            help="Write JSON report of source URLs and their crossreference_to list (most first).",
+        )
+        parser.add_argument(
+            "--references-report",
+            type=str,
+            default=None,
+            help="Write JSON report of reference URLs and their crossreference_from list (most first).",
+        )
+
+    def handle(self, *args, **options):
+        doc_key = options["document"]
+        try:
+            run_find_candidates(
+                doc_key,
+                source_blacklist_path=options["source_blacklist"],
+                reference_blacklist_path=options["reference_blacklist"],
+                sources_report_path=options["sources_report"],
+                references_report_path=options["references_report"],
+                stdout=self.stdout,
+                style_success=self.style.SUCCESS,
+            )
+        except Exception as e:
+            if type(e).__name__ == "DoesNotExist" or "not found" in str(e).lower():
+                raise CommandError(f"Document not found: {doc_key} ({e})") from e
+            if isinstance(e, ValueError):
+                raise CommandError(str(e)) from e
+            raise

api_v2/migrations/0072_crossreference.py

@@ -0,0 +1,31 @@
+# Generated by Django 5.2.1 on 2026-02-14 22:58
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api_v2', '0071_convert_distance_fields_to_integer'),
+        ('contenttypes', '0002_remove_content_type_name'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CrossReference',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source_object_key', models.CharField(help_text='Primary key of the source object (e.g. item key, spell key).', max_length=100)),
+                ('reference_object_key', models.CharField(help_text='Primary key of the reference object.', max_length=100)),
+                ('anchor', models.CharField(help_text="The text in the source's description to highlight and link to the reference.", max_length=100)),
+                ('reference_content_type', models.ForeignKey(help_text='The model of the object this Crossreference points to.', on_delete=django.db.models.deletion.CASCADE, related_name='Crossreferences_as_reference', to='contenttypes.contenttype')),
+                ('source_content_type', models.ForeignKey(help_text='The model of the object that contains the description.', on_delete=django.db.models.deletion.CASCADE, related_name='Crossreferences_as_source', to='contenttypes.contenttype')),
+            ],
+            options={
+                'verbose_name_plural': 'crossreferences',
+                'ordering': ['source_content_type', 'source_object_key', 'id'],
+                'indexes': [models.Index(fields=['source_content_type', 'source_object_key'], name='api_v2_cros_source__44db64_idx'), models.Index(fields=['reference_content_type', 'reference_object_key'], name='api_v2_cros_referen_a96fe0_idx')],
+            },
+        ),
+    ]

api_v2/migrations/0073_crossreference_document.py

@@ -0,0 +1,59 @@
+# Generated by Django 5.2.1
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def backfill_crossreference_document(apps, schema_editor):
+    CrossReference = apps.get_model("api_v2", "CrossReference")
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    for cr in CrossReference.objects.select_related("source_content_type").iterator():
+        model = cr.source_content_type.model_class()
+        if model is None:
+            continue
+        try:
+            obj = model.objects.get(pk=cr.source_object_key)
+        except model.DoesNotExist:
+            continue
+        document = getattr(obj, "document", None)
+        if document is None and hasattr(obj, "parent") and obj.parent_id is not None:
+            document = getattr(obj.parent, "document", None)
+        if document is None and hasattr(obj, "parent") and getattr(obj.parent, "parent_id", None) is not None:
+            document = getattr(obj.parent.parent, "document", None)
+        if document is not None:
+            cr.document_id = document.pk
+            cr.save(update_fields=["document_id"])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api_v2", "0072_crossreference"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="crossreference",
+            name="document",
+            field=models.ForeignKey(
+                help_text="Document the source object belongs to (denormalized for filtering).",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="api_v2.document",
+            ),
+        ),
+        migrations.RunPython(backfill_crossreference_document, migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="crossreference",
+            name="document",
+            field=models.ForeignKey(
+                help_text="Document the source object belongs to (denormalized for filtering).",
+                on_delete=django.db.models.deletion.CASCADE,
+                to="api_v2.document",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="crossreference",
+            index=models.Index(fields=["document"], name="api_v2_cros_documen_idx"),
+        ),
+    ]