From 409f879a6cee07e3fc35e8045ce6913073b67648 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Tue, 9 Dec 2025 10:37:12 +0100 Subject: [PATCH 1/8] text analysis --- CHANGELOG.rst | 3 + toolium/utils/ai_utils/openai.py | 12 +- toolium/utils/ai_utils/text_analysis.py | 155 ++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 toolium/utils/ai_utils/text_analysis.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 34873cdc..d6a0cd2f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,9 @@ v3.7.1 *Release date: In development* +- Add text analysis tool to get an overall match of a text against a list of expected caractersitics + using AI libraries that come with the `ai` extra dependency + v3.7.0 ------ diff --git a/toolium/utils/ai_utils/openai.py b/toolium/utils/ai_utils/openai.py index 3a1d6072..82b58df9 100644 --- a/toolium/utils/ai_utils/openai.py +++ b/toolium/utils/ai_utils/openai.py @@ -49,12 +49,16 @@ def openai_request(system_message, user_message, model_name=None, azure=False, * model_name = model_name or config.get_optional('AI', 'openai_model', 'gpt-4o-mini') logger.info(f"Calling to OpenAI API with model {model_name}") client = AzureOpenAI(**kwargs) if azure else OpenAI(**kwargs) + msg = [] + if isinstance(system_message, list): + for prompt in system_message: + msg.append({"role": "system", "content": prompt}) + else: + msg.append({"role": "system", "content": system_message}) + msg.append({"role": "user", "content": user_message}) completion = client.chat.completions.create( model=model_name, - messages=[ - {"role": "system", "content": system_message}, - {"role": "user", "content": user_message}, - ], + messages=msg, ) response = completion.choices[0].message.content logger.debug(f"OpenAI response: {response}") diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py new file mode 100644 index 00000000..8ae4f1a5 --- /dev/null +++ b/toolium/utils/ai_utils/text_analysis.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2025 Telefónica Innovación Digital, S.L. +This file is part of Toolium. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import logging + +try: + from sentence_transformers import SentenceTransformer, util +except ImportError: + SentenceTransformer = None + +from toolium.utils.ai_utils.openai import openai_request +from toolium.driver_wrappers_pool import DriverWrappersPool + +# Configure logger +logger = logging.getLogger(__name__) + +def build_system_message(characteristics): + """ + Build system message for text criteria analysis prompt. + + :param characteristics: list of target characteristics to evaluate + """ + feature_list = "\n".join(f"- {c}" for c in characteristics) + base_prompt = f""" + You are an assistant that scores how well a given text matches a set of target characteristics and returns a JSON object. + + You will receive a user message that contains ONLY the text to analyze. + + Target characteristics: + {feature_list} + + Tasks: + 1) For EACH characteristic, decide how well the text satisfies it on a scale from 0.0 (does not satisfy it at all) to 1.0 (perfectly satisfies it). Consider style, tone and content when relevant. + 2) Only for each low scored characteristic (<=0.2), output: + - "name": the exact characteristic name as listed above. + - "score": a float between 0.0 and 0.2. + 3) Compute an overall score "overall_match" between 0.0 and 1.0 that summarizes how well the text matches the whole set. It does not have to be a simple arithmetic mean, but must still be in [0.0, 1.0]. + 4) Produce a "data" object that can contain extra structured analysis sections: + - "data" MUST always be present. + - "data" MUST be a JSON object. + - Each key in "data" is the title/name of a section (e.g. "genres", "entities", "style_breakdown"). + - Each value is a JSON array (the structure of its objects will be defined by additional system instructions). + + Output format (IMPORTANT): + Return ONLY a single valid JSON object with this exact top-level structure and property names: + + {{ + "overall_match": float, + "features": [ + {{ + "name": string, + "score": float + }} + ], + "data": {{ + "": [ + {{}} + ] + }} + }} + + Constraints: + - The "data" field must ALWAYS be present. If there are no extra sections, it MUST be: "data": {{}}. + - Use a dot as decimal separator (e.g. 0.75, not 0,75). + - Use at most 2 decimal places for all scores. + - Do NOT include any text outside the JSON (no Markdown, no comments, no explanations). + - If a characteristic is not applicable to the text, give it a low score (<= 0.2). + """ + return base_prompt.strip() + + +def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=None, model_name=None, azure=False, **kwargs): + """ + Get text criteria analysis using Azure OpenAI. To analyze how well a given text + matches a set of target characteristics. + The response is a structured JSON object with overall match score, individual feature scores, + and additional data sections. + + :param text_input: text to analyze + :param target_features: list of target characteristics to evaluate + :param extra_tasks: additional system messages for extra analysis sections (optional) + :param model_name: name of the Azure OpenAI model to use + :param azure: whether to use Azure OpenAI or standard OpenAI + :param kwargs: additional parameters to be used by Azure OpenAI client + :returns: response from Azure OpenAI + """ + # Build prompt using base prompt and target features + system_message = build_system_message(target_features) + msg = [system_message] + if extra_tasks: + if isinstance(extra_tasks, list): + for task in extra_tasks: + msg.append(task) + else: + msg.append(extra_tasks) + return openai_request(system_message, text_input, model_name, azure, **kwargs) + + +def get_text_criteria_analysis_sentence_transformers(text_input, target_features, extra_tasks=None, + model_name=None, azure=True, **kwargs): + """ + Get text criteria analysis using Sentence Transformers. To analyze how well a given text + matches a set of target characteristics. + + :param text_input: text to analyze + :param target_features: list of target characteristics to evaluate + :param extra_tasks: additional system messages for extra analysis sections (not used here, for compatibility) + :param model_name: name of the Sentence Transformers model to use + :param azure: whether to use Azure OpenAI or standard OpenAI (not used here, for compatibility) + :param kwargs: additional parameters to be used by Sentence Transformers client + """ + if SentenceTransformer is None: + raise ImportError("Sentence Transformers is not installed. Please run 'pip install toolium[ai]'" + " to use Sentence Transformers features") + config = DriverWrappersPool.get_default_wrapper().config + model_name = model_name or config.get_optional('AI', 'sentence_transformers_model', 'all-mpnet-base-v2') + model = SentenceTransformer(model_name, **kwargs) + # Pre-compute feature embeddings + feature_embs = model.encode([f for f in target_features], normalize_embeddings=True) + # text_input embedding + text_emb = model.encode(text_input, normalize_embeddings=True) + # Computes cosine-similarities between the text and features tensors (range [-1, 1]) + sims = util.cos_sim(text_emb, feature_embs)[0].tolist() + results = [] + # Generate contracted results + for f, sim in zip(target_features, sims): + # Normalize similarity from [-1, 1] to [0, 1] + score = (sim + 1.0) / 2.0 + results.append({ + "name": f, + "score": round(score, 2), + }) + + # overall score as average of feature scores + overall = sum(r["score"] for r in results) / len(results) + + return { + "overall_match": round(overall, 2), + "features": results, + "data": {} + } From 77eaef7b3d8205eee46b2127c1831d84e9cafc23 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Tue, 9 Dec 2025 10:45:18 +0100 Subject: [PATCH 2/8] lint --- toolium/utils/ai_utils/text_analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 8ae4f1a5..79d5f031 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -28,6 +28,7 @@ # Configure logger logger = logging.getLogger(__name__) +# flake8: noqa E501 def build_system_message(characteristics): """ Build system message for text criteria analysis prompt. @@ -83,7 +84,8 @@ def build_system_message(characteristics): return base_prompt.strip() -def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=None, model_name=None, azure=False, **kwargs): +def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=None, model_name=None, + azure=False, **kwargs): """ Get text criteria analysis using Azure OpenAI. To analyze how well a given text matches a set of target characteristics. From 1296021eeaf8ac99e4c0ee14e4b5657ea5b5d84a Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Thu, 11 Dec 2025 23:41:59 +0100 Subject: [PATCH 3/8] sentence transformers and tests --- .../test/utils/ai_utils/test_text_analysis.py | 109 ++++++++++++++++++ toolium/utils/ai_utils/text_analysis.py | 22 +++- 2 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 toolium/test/utils/ai_utils/test_text_analysis.py diff --git a/toolium/test/utils/ai_utils/test_text_analysis.py b/toolium/test/utils/ai_utils/test_text_analysis.py new file mode 100644 index 00000000..8ed61e23 --- /dev/null +++ b/toolium/test/utils/ai_utils/test_text_analysis.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" +Copyright 2025 Telefónica Innovación Digital, S.L. +This file is part of Toolium. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import json +import pytest + +from toolium.driver_wrappers_pool import DriverWrappersPool +from toolium.utils.ai_utils.text_analysis import (get_text_criteria_analysis_openai, + get_text_criteria_analysis_sentence_transformers) + + +def configure_default_openai_model(): + """ + Configure OpenAI model used in unit tests + """ + config = DriverWrappersPool.get_default_wrapper().config + try: + config.add_section('AI') + except Exception: + pass + config.set('AI', 'openai_model', 'gpt-4o-mini') + + +get_analysis_examples = ( + ('How are you today?', ["is a greeting phrase", "is a question"], 0.7, 1), + ('Today is sunny', ["is an affirmation", "talks about the weather"], 0.7, 1), + ('I love programming', ["expresses a positive sentiment"], 0.7, 1), + ('How are you today?', ["is an affirmation", "talks about the weather"], 0.0, 0.2), + ('Today is sunny', ["is a greeting phrase", "is a question"], 0.0, 0.2), + ('I love programming', ["is a greeting phrase", "is a question"], 0.0, 0.2), +) + + +@pytest.mark.skipif(os.getenv("AZURE_OPENAI_API_KEY") is None, + reason="AZURE_OPENAI_API_KEY environment variable not set") +@pytest.mark.parametrize('input_text, features_list, expected_low, expected_high', get_analysis_examples) +def test_get_text_analysis(input_text, features_list, expected_low, expected_high): + similarity = json.loads(get_text_criteria_analysis_openai(input_text, features_list, azure=True)) + assert expected_low <= similarity['overall_match'] <= expected_high,\ + f"Overall match {similarity['overall_match']} not in range" + + +extra_task = """ + Additional task: + + Extract all verbs from the input text and add them to the JSON under data.verbs. + + Rules: + - Use the same language as the input text. + - Return verbs in their base/infinitive form when possible. + - Do not repeat verbs (no duplicates). + - Preserve the order in which they first appear in the text. + - Verbs should be in this base/infinitive form. + + The data field must include: + "data": { + "verbs": [ "", "", ... ] + } + If no verbs are found, set "verbs" to an empty array: "verbs": []. +""" + +get_extra_examples = ( + ('How are you today?', ["is a greeting phrase", "is a question"], ['be']), + ('I wrote a letter', ["is an affirmation", "talks about the weather"], ['write']), + ('I have to go', ["expresses a positive sentiment"], ['have', 'go']), + ('I went to Madrid', ["is an affirmation", "talks about the weather"], ['go']), + ('Oops I did it again', ["is a greeting phrase", "is a question"], ['do']) +) + +@pytest.mark.skipif(os.getenv("AZURE_OPENAI_API_KEY") is None, + reason="AZURE_OPENAI_API_KEY environment variable not set") +@pytest.mark.parametrize('input_text, features_list, verb_list', get_extra_examples) +def test_get_text_analysis_extra_features(input_text, features_list, verb_list): + similarity = json.loads(get_text_criteria_analysis_openai(input_text, features_list, + azure=True, extra_tasks=extra_task)) + assert similarity['data']['verbs'] == verb_list + + +examples_sentence_transformers = ( + ('How are you today?', ["hello!", "What's up"], 0.4, 1), + ('Today is not sunny', ["it's raining"], 0.4, 1), + ('I love programming', ["I like code", "I love to cook"], 0.4, 1), + ('How are you today?', ["it's raining", "this text is an affirmation"], 0.0, 0.3), + ('Today is sunny', ["I like code", "I love to cook"], 0.0, 0.3), + ('I love programming', ["hello!", "What's up"], 0.0, 0.3), +) + + +# @pytest.mark.skip(reason='Sentence Transformers model is not available in the CI environment') +@pytest.mark.parametrize('input_text, features_list, expected_low, expected_high', examples_sentence_transformers) +def test_get_text_analysis_sentence_transformers(input_text, features_list, expected_low, expected_high): + similarity = get_text_criteria_analysis_sentence_transformers(input_text, features_list) + assert expected_low <= similarity['overall_match'] <= expected_high diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 79d5f031..a720309f 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -46,7 +46,7 @@ def build_system_message(characteristics): Tasks: 1) For EACH characteristic, decide how well the text satisfies it on a scale from 0.0 (does not satisfy it at all) to 1.0 (perfectly satisfies it). Consider style, tone and content when relevant. - 2) Only for each low scored characteristic (<=0.2), output: + 2) ONLY for each low scored characteristic (<=0.2), output: - "name": the exact characteristic name as listed above. - "score": a float between 0.0 and 0.2. 3) Compute an overall score "overall_match" between 0.0 and 1.0 that summarizes how well the text matches the whole set. It does not have to be a simple arithmetic mean, but must still be in [0.0, 1.0]. @@ -75,6 +75,7 @@ def build_system_message(characteristics): }} Constraints: + - Do NOT include scores for high valued (<=0.2) features at features list. - The "data" field must ALWAYS be present. If there are no extra sections, it MUST be: "data": {{}}. - Use a dot as decimal separator (e.g. 0.75, not 0,75). - Use at most 2 decimal places for all scores. @@ -109,14 +110,17 @@ def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=N msg.append(task) else: msg.append(extra_tasks) - return openai_request(system_message, text_input, model_name, azure, **kwargs) + return openai_request(msg, text_input, model_name, azure, **kwargs) def get_text_criteria_analysis_sentence_transformers(text_input, target_features, extra_tasks=None, model_name=None, azure=True, **kwargs): """ - Get text criteria analysis using Sentence Transformers. To analyze how well a given text - matches a set of target characteristics. + Get text criteria analysis using Sentence Transformers. Sentence Transformers works better using examples + that are semantically similar, so this method is more suitable for evaluating characteristics like + "is a greeting phrase", "talks about the weather", etc. + The response is a structured JSON object with overall match score, individual feature scores, + and additional data sections. :param text_input: text to analyze :param target_features: list of target characteristics to evaluate @@ -128,6 +132,12 @@ def get_text_criteria_analysis_sentence_transformers(text_input, target_features if SentenceTransformer is None: raise ImportError("Sentence Transformers is not installed. Please run 'pip install toolium[ai]'" " to use Sentence Transformers features") + + def similarity_to_score(cos_sim): + if cos_sim <= 0.1: + return 0.0 + return cos_sim / 0.7 + config = DriverWrappersPool.get_default_wrapper().config model_name = model_name or config.get_optional('AI', 'sentence_transformers_model', 'all-mpnet-base-v2') model = SentenceTransformer(model_name, **kwargs) @@ -141,10 +151,10 @@ def get_text_criteria_analysis_sentence_transformers(text_input, target_features # Generate contracted results for f, sim in zip(target_features, sims): # Normalize similarity from [-1, 1] to [0, 1] - score = (sim + 1.0) / 2.0 + score = similarity_to_score(sim) results.append({ "name": f, - "score": round(score, 2), + "score": round(score, 2) }) # overall score as average of feature scores From 9c09d43882912ff0028bf6368baa4007d1f33ce4 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Fri, 12 Dec 2025 13:17:57 +0100 Subject: [PATCH 4/8] lint fix --- toolium/test/utils/ai_utils/test_text_analysis.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/toolium/test/utils/ai_utils/test_text_analysis.py b/toolium/test/utils/ai_utils/test_text_analysis.py index 8ed61e23..8ccd4cba 100644 --- a/toolium/test/utils/ai_utils/test_text_analysis.py +++ b/toolium/test/utils/ai_utils/test_text_analysis.py @@ -52,22 +52,21 @@ def configure_default_openai_model(): @pytest.mark.parametrize('input_text, features_list, expected_low, expected_high', get_analysis_examples) def test_get_text_analysis(input_text, features_list, expected_low, expected_high): similarity = json.loads(get_text_criteria_analysis_openai(input_text, features_list, azure=True)) - assert expected_low <= similarity['overall_match'] <= expected_high,\ - f"Overall match {similarity['overall_match']} not in range" + assert expected_low <= similarity['overall_match'] <= expected_high extra_task = """ Additional task: Extract all verbs from the input text and add them to the JSON under data.verbs. - + Rules: - Use the same language as the input text. - Return verbs in their base/infinitive form when possible. - Do not repeat verbs (no duplicates). - Preserve the order in which they first appear in the text. - Verbs should be in this base/infinitive form. - + The data field must include: "data": { "verbs": [ "", "", ... ] @@ -83,6 +82,7 @@ def test_get_text_analysis(input_text, features_list, expected_low, expected_hig ('Oops I did it again', ["is a greeting phrase", "is a question"], ['do']) ) + @pytest.mark.skipif(os.getenv("AZURE_OPENAI_API_KEY") is None, reason="AZURE_OPENAI_API_KEY environment variable not set") @pytest.mark.parametrize('input_text, features_list, verb_list', get_extra_examples) @@ -102,7 +102,7 @@ def test_get_text_analysis_extra_features(input_text, features_list, verb_list): ) -# @pytest.mark.skip(reason='Sentence Transformers model is not available in the CI environment') +@pytest.mark.skip(reason='Sentence Transformers model is not available in the CI environment') @pytest.mark.parametrize('input_text, features_list, expected_low, expected_high', examples_sentence_transformers) def test_get_text_analysis_sentence_transformers(input_text, features_list, expected_low, expected_high): similarity = get_text_criteria_analysis_sentence_transformers(input_text, features_list) From 86d81e2c05502c478bde4faeabb702d54ced1d06 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Fri, 12 Dec 2025 15:52:57 +0100 Subject: [PATCH 5/8] CHANGELOG typo --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d6a0cd2f..218a89c3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ v3.7.1 *Release date: In development* -- Add text analysis tool to get an overall match of a text against a list of expected caractersitics +- Add text analysis tool to get an overall match of a text against a list of expected characteristics using AI libraries that come with the `ai` extra dependency v3.7.0 From fce1dc978fdb5f3021649e042d34847fd980e92a Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Thu, 18 Dec 2025 13:51:37 +0100 Subject: [PATCH 6/8] text analysis isolated --- .../test/utils/ai_utils/test_text_analysis.py | 61 +----------- toolium/utils/ai_utils/text_analysis.py | 96 ++++--------------- 2 files changed, 23 insertions(+), 134 deletions(-) diff --git a/toolium/test/utils/ai_utils/test_text_analysis.py b/toolium/test/utils/ai_utils/test_text_analysis.py index 8ccd4cba..3d677a4c 100644 --- a/toolium/test/utils/ai_utils/test_text_analysis.py +++ b/toolium/test/utils/ai_utils/test_text_analysis.py @@ -21,8 +21,7 @@ import pytest from toolium.driver_wrappers_pool import DriverWrappersPool -from toolium.utils.ai_utils.text_analysis import (get_text_criteria_analysis_openai, - get_text_criteria_analysis_sentence_transformers) +from toolium.utils.ai_utils.text_analysis import get_text_criteria_analysis def configure_default_openai_model(): @@ -34,7 +33,7 @@ def configure_default_openai_model(): config.add_section('AI') except Exception: pass - config.set('AI', 'openai_model', 'gpt-4o-mini') + config.set('AI', 'openai_model', 'gpt-4.1-mini') get_analysis_examples = ( @@ -51,59 +50,5 @@ def configure_default_openai_model(): reason="AZURE_OPENAI_API_KEY environment variable not set") @pytest.mark.parametrize('input_text, features_list, expected_low, expected_high', get_analysis_examples) def test_get_text_analysis(input_text, features_list, expected_low, expected_high): - similarity = json.loads(get_text_criteria_analysis_openai(input_text, features_list, azure=True)) - assert expected_low <= similarity['overall_match'] <= expected_high - - -extra_task = """ - Additional task: - - Extract all verbs from the input text and add them to the JSON under data.verbs. - - Rules: - - Use the same language as the input text. - - Return verbs in their base/infinitive form when possible. - - Do not repeat verbs (no duplicates). - - Preserve the order in which they first appear in the text. - - Verbs should be in this base/infinitive form. - - The data field must include: - "data": { - "verbs": [ "", "", ... ] - } - If no verbs are found, set "verbs" to an empty array: "verbs": []. -""" - -get_extra_examples = ( - ('How are you today?', ["is a greeting phrase", "is a question"], ['be']), - ('I wrote a letter', ["is an affirmation", "talks about the weather"], ['write']), - ('I have to go', ["expresses a positive sentiment"], ['have', 'go']), - ('I went to Madrid', ["is an affirmation", "talks about the weather"], ['go']), - ('Oops I did it again', ["is a greeting phrase", "is a question"], ['do']) -) - - -@pytest.mark.skipif(os.getenv("AZURE_OPENAI_API_KEY") is None, - reason="AZURE_OPENAI_API_KEY environment variable not set") -@pytest.mark.parametrize('input_text, features_list, verb_list', get_extra_examples) -def test_get_text_analysis_extra_features(input_text, features_list, verb_list): - similarity = json.loads(get_text_criteria_analysis_openai(input_text, features_list, - azure=True, extra_tasks=extra_task)) - assert similarity['data']['verbs'] == verb_list - - -examples_sentence_transformers = ( - ('How are you today?', ["hello!", "What's up"], 0.4, 1), - ('Today is not sunny', ["it's raining"], 0.4, 1), - ('I love programming', ["I like code", "I love to cook"], 0.4, 1), - ('How are you today?', ["it's raining", "this text is an affirmation"], 0.0, 0.3), - ('Today is sunny', ["I like code", "I love to cook"], 0.0, 0.3), - ('I love programming', ["hello!", "What's up"], 0.0, 0.3), -) - - -@pytest.mark.skip(reason='Sentence Transformers model is not available in the CI environment') -@pytest.mark.parametrize('input_text, features_list, expected_low, expected_high', examples_sentence_transformers) -def test_get_text_analysis_sentence_transformers(input_text, features_list, expected_low, expected_high): - similarity = get_text_criteria_analysis_sentence_transformers(input_text, features_list) + similarity = json.loads(get_text_criteria_analysis(input_text, features_list, azure=True)) assert expected_low <= similarity['overall_match'] <= expected_high diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index a720309f..72511b27 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -16,14 +16,9 @@ limitations under the License. """ import logging - -try: - from sentence_transformers import SentenceTransformer, util -except ImportError: - SentenceTransformer = None +import json from toolium.utils.ai_utils.openai import openai_request -from toolium.driver_wrappers_pool import DriverWrappersPool # Configure logger logger = logging.getLogger(__name__) @@ -50,11 +45,6 @@ def build_system_message(characteristics): - "name": the exact characteristic name as listed above. - "score": a float between 0.0 and 0.2. 3) Compute an overall score "overall_match" between 0.0 and 1.0 that summarizes how well the text matches the whole set. It does not have to be a simple arithmetic mean, but must still be in [0.0, 1.0]. - 4) Produce a "data" object that can contain extra structured analysis sections: - - "data" MUST always be present. - - "data" MUST be a JSON object. - - Each key in "data" is the title/name of a section (e.g. "genres", "entities", "style_breakdown"). - - Each value is a JSON array (the structure of its objects will be defined by additional system instructions). Output format (IMPORTANT): Return ONLY a single valid JSON object with this exact top-level structure and property names: @@ -66,17 +56,11 @@ def build_system_message(characteristics): "name": string, "score": float }} - ], - "data": {{ - "": [ - {{}} - ] - }} + ] }} Constraints: - Do NOT include scores for high valued (<=0.2) features at features list. - - The "data" field must ALWAYS be present. If there are no extra sections, it MUST be: "data": {{}}. - Use a dot as decimal separator (e.g. 0.75, not 0,75). - Use at most 2 decimal places for all scores. - Do NOT include any text outside the JSON (no Markdown, no comments, no explanations). @@ -85,8 +69,7 @@ def build_system_message(characteristics): return base_prompt.strip() -def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=None, model_name=None, - azure=False, **kwargs): +def get_text_criteria_analysis(text_input, text_criteria, model_name=None, azure=False, **kwargs): """ Get text criteria analysis using Azure OpenAI. To analyze how well a given text matches a set of target characteristics. @@ -94,7 +77,7 @@ def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=N and additional data sections. :param text_input: text to analyze - :param target_features: list of target characteristics to evaluate + :param text_criteria: list of target characteristics to evaluate :param extra_tasks: additional system messages for extra analysis sections (optional) :param model_name: name of the Azure OpenAI model to use :param azure: whether to use Azure OpenAI or standard OpenAI @@ -102,66 +85,27 @@ def get_text_criteria_analysis_openai(text_input, target_features, extra_tasks=N :returns: response from Azure OpenAI """ # Build prompt using base prompt and target features - system_message = build_system_message(target_features) + system_message = build_system_message(text_criteria) msg = [system_message] - if extra_tasks: - if isinstance(extra_tasks, list): - for task in extra_tasks: - msg.append(task) - else: - msg.append(extra_tasks) return openai_request(msg, text_input, model_name, azure, **kwargs) -def get_text_criteria_analysis_sentence_transformers(text_input, target_features, extra_tasks=None, - model_name=None, azure=True, **kwargs): +def assert_text_criteria(text_input, text_criteria, threshold, model_name=None, azure=False, **kwargs): """ - Get text criteria analysis using Sentence Transformers. Sentence Transformers works better using examples - that are semantically similar, so this method is more suitable for evaluating characteristics like - "is a greeting phrase", "talks about the weather", etc. - The response is a structured JSON object with overall match score, individual feature scores, - and additional data sections. + Get text criteria analysis and assert if overall match score is above threshold. :param text_input: text to analyze - :param target_features: list of target characteristics to evaluate - :param extra_tasks: additional system messages for extra analysis sections (not used here, for compatibility) - :param model_name: name of the Sentence Transformers model to use - :param azure: whether to use Azure OpenAI or standard OpenAI (not used here, for compatibility) - :param kwargs: additional parameters to be used by Sentence Transformers client + :param text_criteria: list of target characteristics to evaluate + :param threshold: minimum overall match score to consider the text acceptable + :param model_name: name of the Azure OpenAI model to use + :param azure: whether to use Azure OpenAI or standard OpenAI + :param kwargs: additional parameters to be used by Azure OpenAI client + :raises AssertionError: if overall match score is below threshold """ - if SentenceTransformer is None: - raise ImportError("Sentence Transformers is not installed. Please run 'pip install toolium[ai]'" - " to use Sentence Transformers features") - - def similarity_to_score(cos_sim): - if cos_sim <= 0.1: - return 0.0 - return cos_sim / 0.7 - - config = DriverWrappersPool.get_default_wrapper().config - model_name = model_name or config.get_optional('AI', 'sentence_transformers_model', 'all-mpnet-base-v2') - model = SentenceTransformer(model_name, **kwargs) - # Pre-compute feature embeddings - feature_embs = model.encode([f for f in target_features], normalize_embeddings=True) - # text_input embedding - text_emb = model.encode(text_input, normalize_embeddings=True) - # Computes cosine-similarities between the text and features tensors (range [-1, 1]) - sims = util.cos_sim(text_emb, feature_embs)[0].tolist() - results = [] - # Generate contracted results - for f, sim in zip(target_features, sims): - # Normalize similarity from [-1, 1] to [0, 1] - score = similarity_to_score(sim) - results.append({ - "name": f, - "score": round(score, 2) - }) - - # overall score as average of feature scores - overall = sum(r["score"] for r in results) / len(results) - - return { - "overall_match": round(overall, 2), - "features": results, - "data": {} - } + analysis = json.loads(get_text_criteria_analysis(text_input, text_criteria, model_name, azure, **kwargs)) + overall_match = analysis.get("overall_match", 0.0) + if overall_match < threshold: + raise AssertionError(f"Text criteria analysis failed: overall match {overall_match} " + f"is below threshold {threshold}") + logger.info(f"Text criteria analysis passed: overall match {overall_match} " + f"is above threshold {threshold}") From a4d0d8a2314fba8ff8b0dafea6b2a75016e0ed88 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Thu, 18 Dec 2025 15:18:00 +0100 Subject: [PATCH 7/8] delete extra var --- toolium/utils/ai_utils/text_analysis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 72511b27..4f71d4a0 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -86,8 +86,7 @@ def get_text_criteria_analysis(text_input, text_criteria, model_name=None, azure """ # Build prompt using base prompt and target features system_message = build_system_message(text_criteria) - msg = [system_message] - return openai_request(msg, text_input, model_name, azure, **kwargs) + return openai_request(system_message, text_input, model_name, azure, **kwargs) def assert_text_criteria(text_input, text_criteria, threshold, model_name=None, azure=False, **kwargs): From 7ee6156d8113c9ab98c75020e262019a1cd567a6 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Fri, 19 Dec 2025 23:40:24 +0100 Subject: [PATCH 8/8] requested fixes --- toolium/utils/ai_utils/text_analysis.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 4f71d4a0..33b016c0 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -78,11 +78,10 @@ def get_text_criteria_analysis(text_input, text_criteria, model_name=None, azure :param text_input: text to analyze :param text_criteria: list of target characteristics to evaluate - :param extra_tasks: additional system messages for extra analysis sections (optional) - :param model_name: name of the Azure OpenAI model to use + :param model_name: name of the OpenAI model to use :param azure: whether to use Azure OpenAI or standard OpenAI - :param kwargs: additional parameters to be used by Azure OpenAI client - :returns: response from Azure OpenAI + :param kwargs: additional parameters to be used by OpenAI client + :returns: response from OpenAI """ # Build prompt using base prompt and target features system_message = build_system_message(text_criteria) @@ -96,15 +95,20 @@ def assert_text_criteria(text_input, text_criteria, threshold, model_name=None, :param text_input: text to analyze :param text_criteria: list of target characteristics to evaluate :param threshold: minimum overall match score to consider the text acceptable - :param model_name: name of the Azure OpenAI model to use + :param model_name: name of the OpenAI model to use :param azure: whether to use Azure OpenAI or standard OpenAI - :param kwargs: additional parameters to be used by Azure OpenAI client + :param kwargs: additional parameters to be used by OpenAI client :raises AssertionError: if overall match score is below threshold """ analysis = json.loads(get_text_criteria_analysis(text_input, text_criteria, model_name, azure, **kwargs)) overall_match = analysis.get("overall_match", 0.0) if overall_match < threshold: + logger.error(f"Text criteria analysis failed: overall match {overall_match} " + f"is below threshold {threshold}\n" + f"Failed features: {analysis.get('features', [])}") raise AssertionError(f"Text criteria analysis failed: overall match {overall_match} " - f"is below threshold {threshold}") + f"is below threshold {threshold}\n" + f"Failed features: {analysis.get('features', [])}") logger.info(f"Text criteria analysis passed: overall match {overall_match} " - f"is above threshold {threshold}") + f"is above threshold {threshold}." + f"Low scored features: {analysis.get('features', [])}")