From b74bec59074b3daeb34aae70594ca5ad3fea4358 Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Wed, 28 Jan 2026 22:54:34 +0200 Subject: [PATCH 1/4] feat: added topic scoping to the SDK --- assets/images/coverage.svg | 4 ++-- pyproject.toml | 2 +- qualifire/client.py | 9 +++++++++ qualifire/types.py | 3 +++ uv.lock | 4 ++-- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/assets/images/coverage.svg b/assets/images/coverage.svg index a4262d3..012a849 100644 --- a/assets/images/coverage.svg +++ b/assets/images/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 69% - 69% + 70% + 70% diff --git a/pyproject.toml b/pyproject.toml index 5ceac0f..cd95f22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "qualifire" -version = "0.13.0" +version = "0.14.0" description = "Qualifire Python SDK" authors = [{ name = "qualifire-dev", email = "dror@qualifire.ai" }] requires-python = ">=3.8,<4" diff --git a/qualifire/client.py b/qualifire/client.py index 0cc6026..5b582e5 100644 --- a/qualifire/client.py +++ b/qualifire/client.py @@ -63,6 +63,9 @@ def evaluate( grounding_multi_turn_mode: bool = False, policy_multi_turn_mode: bool = False, policy_target: PolicyTarget = PolicyTarget.BOTH, + topic_scoping_mode: Optional[ModelMode] = None, + topic_scoping_multi_turn_mode: bool = False, + topic_scoping_target: Optional[str] = None, ) -> Union[EvaluationResponse, None]: """ Evaluates the given input and output pairs. @@ -105,6 +108,9 @@ def evaluate( :param grounding_multi_turn_mode: Enable multi-turn mode for grounding check. :param policy_multi_turn_mode: Enable multi-turn mode for policy check. :param policy_target: Target for policy checks (input/output/both). + :param topic_scoping_mode: Model mode for topic scoping check (speed/balanced/quality). + :param topic_scoping_multi_turn_mode: Enable multi-turn mode for topic scoping check. + :param topic_scoping_target: Target topic for topic scoping check. :return: An EvaluationResponse object containing the evaluation results. :raises Exception: If an error occurs during the evaluation. @@ -224,6 +230,9 @@ def evaluate( grounding_multi_turn_mode=grounding_multi_turn_mode, policy_multi_turn_mode=policy_multi_turn_mode, policy_target=policy_target, + topic_scoping_mode=topic_scoping_mode, + topic_scoping_multi_turn_mode=topic_scoping_multi_turn_mode, + topic_scoping_target=topic_scoping_target, ) response = requests.post( diff --git a/qualifire/types.py b/qualifire/types.py index a9c8d85..bd6786a 100644 --- a/qualifire/types.py +++ b/qualifire/types.py @@ -66,6 +66,9 @@ class EvaluationRequest(BaseModel): grounding_multi_turn_mode: bool = False policy_multi_turn_mode: bool = False policy_target: PolicyTarget = PolicyTarget.BOTH + topic_scoping_mode: Optional[ModelMode] = None + topic_scoping_multi_turn_mode: bool = False + topic_scoping_target: Optional[str] = None @model_validator(mode="after") def validate_model(self) -> "EvaluationRequest": diff --git a/uv.lock b/uv.lock index 73478f5..bac7727 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.8, <4" resolution-markers = [ "python_full_version >= '3.13'", @@ -2771,7 +2771,7 @@ wheels = [ [[package]] name = "qualifire" -version = "0.13.0" +version = "0.14.0" source = { editable = "." } dependencies = [ { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, From e935e339a7b886c3053f34073bb468a6d4fdc7dc Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Wed, 28 Jan 2026 23:19:24 +0200 Subject: [PATCH 2/4] added topics to the prompt --- pyproject.toml | 2 +- qualifire/client.py | 3 +++ qualifire/types.py | 1 + uv.lock | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd95f22..5ceac0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "qualifire" -version = "0.14.0" +version = "0.13.0" description = "Qualifire Python SDK" authors = [{ name = "qualifire-dev", email = "dror@qualifire.ai" }] requires-python = ">=3.8,<4" diff --git a/qualifire/client.py b/qualifire/client.py index 5b582e5..600b505 100644 --- a/qualifire/client.py +++ b/qualifire/client.py @@ -66,6 +66,7 @@ def evaluate( topic_scoping_mode: Optional[ModelMode] = None, topic_scoping_multi_turn_mode: bool = False, topic_scoping_target: Optional[str] = None, + allowed_topics: Optional[List[str]] = None, ) -> Union[EvaluationResponse, None]: """ Evaluates the given input and output pairs. @@ -111,6 +112,7 @@ def evaluate( :param topic_scoping_mode: Model mode for topic scoping check (speed/balanced/quality). :param topic_scoping_multi_turn_mode: Enable multi-turn mode for topic scoping check. :param topic_scoping_target: Target topic for topic scoping check. + :param allowed_topics: List of allowed topics for topic scoping check. :return: An EvaluationResponse object containing the evaluation results. :raises Exception: If an error occurs during the evaluation. @@ -233,6 +235,7 @@ def evaluate( topic_scoping_mode=topic_scoping_mode, topic_scoping_multi_turn_mode=topic_scoping_multi_turn_mode, topic_scoping_target=topic_scoping_target, + allowed_topics=allowed_topics, ) response = requests.post( diff --git a/qualifire/types.py b/qualifire/types.py index bd6786a..7fb184a 100644 --- a/qualifire/types.py +++ b/qualifire/types.py @@ -69,6 +69,7 @@ class EvaluationRequest(BaseModel): topic_scoping_mode: Optional[ModelMode] = None topic_scoping_multi_turn_mode: bool = False topic_scoping_target: Optional[str] = None + allowed_topics: Optional[List[str]] = None @model_validator(mode="after") def validate_model(self) -> "EvaluationRequest": diff --git a/uv.lock b/uv.lock index bac7727..a70ee33 100644 --- a/uv.lock +++ b/uv.lock @@ -2771,7 +2771,7 @@ wheels = [ [[package]] name = "qualifire" -version = "0.14.0" +version = "0.13.0" source = { editable = "." } dependencies = [ { name = "pydantic", version = "2.10.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, From ad769fb2adddcfd15aa0062155e112a138d11c1f Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Thu, 29 Jan 2026 10:34:17 +0200 Subject: [PATCH 3/4] wip --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ceac0f..2410c85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "qualifire" -version = "0.13.0" +version = "0.13.1" description = "Qualifire Python SDK" authors = [{ name = "qualifire-dev", email = "dror@qualifire.ai" }] requires-python = ">=3.8,<4" From cf02e9038ead157f714bd5a8e2fcc3f725ec1f03 Mon Sep 17 00:00:00 2001 From: Dror Ivry Date: Thu, 29 Jan 2026 12:19:27 +0200 Subject: [PATCH 4/4] docs --- CLAUDE.md | 4 +++- qualifire/client.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0a2e94e..96eb9ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,15 +42,17 @@ make check-safety The SDK has a minimal structure centered on the `Client` class: -- **`qualifire/client.py`**: Main `Client` class with two public methods: +- **`qualifire/client.py`**: Main `Client` class with three public methods: - `evaluate()` - Run ad-hoc evaluations with various checks (hallucinations, grounding, PII, content moderation, etc.) - `invoke_evaluation()` - Run pre-configured evaluations from the Qualifire dashboard + - `compile_prompt()` - Compile a prompt with topic scoping and policy targets - **`qualifire/types.py`**: All data classes and enums: - `EvaluationRequest`/`EvaluationResponse` - API request/response structures - `LLMMessage`, `LLMToolCall`, `LLMToolDefinition` - Message and tool types for conversation-based evaluation - `ModelMode` (SPEED/BALANCED/QUALITY) - Quality vs speed trade-off for checks - `SyntaxCheckArgs` - Configuration for syntax validation + - `PolicyTarget`, `CompilePromptResponse` - Types for prompt compilation with topic scoping - **`qualifire/utils.py`**: Helper functions for API key and base URL resolution from environment variables diff --git a/qualifire/client.py b/qualifire/client.py index 600b505..8cbac40 100644 --- a/qualifire/client.py +++ b/qualifire/client.py @@ -65,7 +65,7 @@ def evaluate( policy_target: PolicyTarget = PolicyTarget.BOTH, topic_scoping_mode: Optional[ModelMode] = None, topic_scoping_multi_turn_mode: bool = False, - topic_scoping_target: Optional[str] = None, + topic_scoping_target: PolicyTarget = PolicyTarget.BOTH, allowed_topics: Optional[List[str]] = None, ) -> Union[EvaluationResponse, None]: """