From df5861a427ba356c298c186554ac8075b395ba99 Mon Sep 17 00:00:00 2001 From: Ahmath-Gadji Date: Thu, 9 Jul 2026 09:12:29 +0000 Subject: [PATCH] feat(indexation): disable topic tagging by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Topic tagging ran on every indexed document, spending one LLM call per file to produce tags that nothing consumes yet — they are not exposed in the API and not used in retrieval. The admin UI already ships the toggle as disabled with a "Coming soon" hint. Make it opt-in instead: - IndexationPipelineConfig.enable_topic_tagging defaults to False - the seeded `default` indexation preset sets it False explicitly - _required_llm_names no longer resolves an LLM endpoint for an absent key - _replace_topic_tags_if_needed treats an absent key as disabled, so a re-index under a preset that never enabled tagging clears tags left behind by an earlier run - the admin UI toggle falls back to off, matching the backend Existing deployments are unaffected: seed_defaults only inserts when a preset type has no rows, so a stored preset with tagging on keeps it on. --- openrag/core/config/indexation_pipeline.py | 6 ++++-- openrag/services/orchestrators/preset_service.py | 2 +- openrag/services/workers/indexer_actor.py | 5 ++++- openrag/services/workers/indexer_pool.py | 2 +- tests/unit/core/config/test_pipeline_configs.py | 5 +++++ .../services/orchestrators/test_preset_service.py | 11 +++++++++++ tests/unit/services/workers/test_indexer_pool.py | 5 +++-- ui/src/pages/admin/presets.tsx | 2 +- 8 files changed, 30 insertions(+), 8 deletions(-) diff --git a/openrag/core/config/indexation_pipeline.py b/openrag/core/config/indexation_pipeline.py index e0ecb0f6b..5b53c3908 100644 --- a/openrag/core/config/indexation_pipeline.py +++ b/openrag/core/config/indexation_pipeline.py @@ -52,8 +52,10 @@ class IndexationPipelineConfig(BaseModel): default_factory=lambda: ["person", "organization", "location", "event"], ) - # Topic tagging - enable_topic_tagging: bool = True + # Topic tagging. Off by default: the generated tags are not yet surfaced in + # the API or used for retrieval, so enabling it only buys an extra LLM call + # per document. Partitions that want the tags opt in on their preset. + enable_topic_tagging: bool = False max_topic_tags: int = Field(default=7, ge=1, le=50) topic_tagging_llm: str | None = None diff --git a/openrag/services/orchestrators/preset_service.py b/openrag/services/orchestrators/preset_service.py index 62520808d..6df9fd9ff 100644 --- a/openrag/services/orchestrators/preset_service.py +++ b/openrag/services/orchestrators/preset_service.py @@ -51,7 +51,7 @@ # pymupdf-configured deployment. Named presets below opt in explicitly. "enable_image_captioning": True, "enable_entity_extraction": True, - "enable_topic_tagging": True, + "enable_topic_tagging": False, }, "legal": { "chunking": {"name": "recursive_splitter", "chunk_size": 1024, "chunk_overlap_rate": 0.25}, diff --git a/openrag/services/workers/indexer_actor.py b/openrag/services/workers/indexer_actor.py index 40fa9f2cb..3af5ffb1e 100644 --- a/openrag/services/workers/indexer_actor.py +++ b/openrag/services/workers/indexer_actor.py @@ -159,7 +159,10 @@ async def _replace_topic_tags_if_needed( return has_topic_tags = "topic_tags" in row - topic_tagging_disabled = indexation_config is not None and indexation_config.get("enable_topic_tagging") is False + # Mirrors IndexationPipelineConfig.enable_topic_tagging: an absent key means + # disabled, so a re-index under a preset that never enabled tagging still + # clears tags left behind by an earlier run. + topic_tagging_disabled = indexation_config is not None and not indexation_config.get("enable_topic_tagging", False) if not has_topic_tags and not topic_tagging_disabled: return diff --git a/openrag/services/workers/indexer_pool.py b/openrag/services/workers/indexer_pool.py index d5f8b983b..c1ae0270b 100644 --- a/openrag/services/workers/indexer_pool.py +++ b/openrag/services/workers/indexer_pool.py @@ -359,7 +359,7 @@ def _required_llm_names(indexation_config: dict[str, Any] | None) -> list[str]: names: list[str] = [] if indexation_config.get("enable_contextualization"): names.append(indexation_config.get("contextualization_llm") or "default") - if indexation_config.get("enable_topic_tagging", True): + if indexation_config.get("enable_topic_tagging", False): names.append(indexation_config.get("topic_tagging_llm") or "default") return names diff --git a/tests/unit/core/config/test_pipeline_configs.py b/tests/unit/core/config/test_pipeline_configs.py index 3461274cc..206b7c7ba 100644 --- a/tests/unit/core/config/test_pipeline_configs.py +++ b/tests/unit/core/config/test_pipeline_configs.py @@ -41,6 +41,11 @@ def test_indexation_pipeline_rejects_unknown_contextualization_mode(): IndexationPipelineConfig(contextualization_mode="verbose") +def test_indexation_pipeline_topic_tagging_defaults_off(): + """Topic tagging is opt-in: tags are not yet surfaced or used in retrieval.""" + assert IndexationPipelineConfig().enable_topic_tagging is False + + def test_retrieval_pipeline_rejects_unknown_type(): """Retrieval presets reject unsupported retrieval modes.""" with pytest.raises(ValidationError): diff --git a/tests/unit/services/orchestrators/test_preset_service.py b/tests/unit/services/orchestrators/test_preset_service.py index 24cb558fa..0c28f19a6 100644 --- a/tests/unit/services/orchestrators/test_preset_service.py +++ b/tests/unit/services/orchestrators/test_preset_service.py @@ -155,6 +155,17 @@ async def test_seed_defaults_inserts_six_presets_when_empty(): assert len(upserts) == 6 # 3 indexation + 3 retrieval +@pytest.mark.asyncio +async def test_seed_defaults_indexation_leaves_topic_tagging_off(): + repo = _FakePresetRepo() + svc = _make_service(repo) + await svc.seed_defaults() + + idx = [r for r in repo._store.values() if r["preset_type"] == "indexation"] + assert idx # sanity: indexation presets were seeded + assert all(r["config"].get("enable_topic_tagging", False) is False for r in idx) + + @pytest.mark.asyncio async def test_seed_defaults_retrieval_inherits_reranker_enabled(): from core.config.root import Settings diff --git a/tests/unit/services/workers/test_indexer_pool.py b/tests/unit/services/workers/test_indexer_pool.py index 84cc86a06..500f2727f 100644 --- a/tests/unit/services/workers/test_indexer_pool.py +++ b/tests/unit/services/workers/test_indexer_pool.py @@ -480,9 +480,10 @@ def test_required_llm_names_mirrors_pipeline_selection() -> None: from services.workers.indexer_pool import _required_llm_names assert _required_llm_names(None) == [] - # Topic tagging defaults on, contextualization defaults off. - assert _required_llm_names({}) == ["default"] + # Both topic tagging and contextualization default off. + assert _required_llm_names({}) == [] assert _required_llm_names({"enable_topic_tagging": False}) == [] + assert _required_llm_names({"enable_topic_tagging": True}) == ["default"] assert _required_llm_names( { "enable_contextualization": True, diff --git a/ui/src/pages/admin/presets.tsx b/ui/src/pages/admin/presets.tsx index 5c3c06178..e42f9915a 100644 --- a/ui/src/pages/admin/presets.tsx +++ b/ui/src/pages/admin/presets.tsx @@ -402,7 +402,7 @@ function IndexationPresetForm({ /> toggleFeature("enable_topic_tagging", "topic_tagging_llm", on)} // Postponed: tags are generated but not yet surfaced or used in // retrieval, so the control is disabled until the feature ships.