From fddbf2689a46133a50b9c6accaaaf9845d414a6c Mon Sep 17 00:00:00 2001 From: Jiri Podivin Date: Wed, 26 Mar 2025 20:06:34 +0100 Subject: [PATCH 1/2] Setting up pytest * test target in makefile * pytest added to dev dependencies * simple test case Signed-off-by: Jiri Podivin --- .github/workflows/run-pytest.yml | 23 ++++++++++ Makefile | 5 ++ pdm.lock | 58 ++++++++++++++++++++---- pyproject.toml | 1 + tests/mocks/__init__.py | 1 + tests/mocks/mock_classes.py | 16 +++++++ tests/utils/__init__.py | 1 + tests/utils/test_similarity_score_llm.py | 9 ++++ 8 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/run-pytest.yml create mode 100644 tests/mocks/__init__.py create mode 100644 tests/mocks/mock_classes.py create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/test_similarity_score_llm.py diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml new file mode 100644 index 0000000..a63421e --- /dev/null +++ b/.github/workflows/run-pytest.yml @@ -0,0 +1,23 @@ +name: Pytest +on: + push: + pull_request: + workflow_dispatch: + +jobs: + run-pytest: + name: Execute Pytest tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup PDM + uses: pdm-project/setup-pdm@v4 + + - name: Install dependencies + run: make install-tools + + - name: Run linters + run: make test diff --git a/Makefile b/Makefile index 735b1b5..7c9e9fa 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ install-tools: ## Install required utilities/tools pdm run black --version # check that correct Ruff version is installed pdm run ruff --version + # check that Pytest is installed + pdm run pytest --version pdm-lock-check: ## Check that the pdm.lock file is in a good shape pdm lock --check @@ -58,6 +60,9 @@ verify-packages-completeness: requirements.txt ## Verify that requirements.txt f distribution-archives: ## Generate distribution archives to be uploaded into Python registry pdm run python -m build +test: install-deps-test ## Execute tests with Pytest + pdm run pytest tests + help: ## Show this help screen @echo 'Usage: make ... ' @echo '' diff --git a/pdm.lock b/pdm.lock index b50121b..6778d2c 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,20 +5,20 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:7b5c0ba86a340c969adba02c6a8f3679008cca8bcacf6848f37991b64b74504b" +content_hash = "sha256:c63774322b7541911172368c9e2c170b3d9b5ebe35d6aacc0836d7dab3ff7ac5" [[metadata.targets]] requires_python = ">=3.11.1,<=3.12.8" [[package]] name = "absl-py" -version = "2.2.0" +version = "2.2.1" requires_python = ">=3.8" summary = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." groups = ["default"] files = [ - {file = "absl_py-2.2.0-py3-none-any.whl", hash = "sha256:5c432cdf7b045f89c4ddc3bba196cabb389c0c321322f8dec68eecdfa732fdad"}, - {file = "absl_py-2.2.0.tar.gz", hash = "sha256:2aabeae1403380e338fba88d4f8c9bf9925c20ad04c1c96d4a26930d034c507b"}, + {file = "absl_py-2.2.1-py3-none-any.whl", hash = "sha256:ca8209abd5005ae6e700ef36e2edc84ad5338678f95625a3f15275410a89ffbc"}, + {file = "absl_py-2.2.1.tar.gz", hash = "sha256:4c7bc50d42d021c12d4f31b7001167925e0bd71ade853069f64af410f5565ff9"}, ] [[package]] @@ -1031,6 +1031,17 @@ files = [ {file = "importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580"}, ] +[[package]] +name = "iniconfig" +version = "2.1.0" +requires_python = ">=3.8" +summary = "brain-dead simple config-ini parsing" +groups = ["dev"] +files = [ + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, +] + [[package]] name = "installer" version = "0.7.0" @@ -2269,6 +2280,17 @@ files = [ {file = "platformdirs-4.3.7.tar.gz", hash = "sha256:eb437d586b6a0986388f0d6f74aa0cde27b48d0e3d66843640bfb6bdcdb6e351"}, ] +[[package]] +name = "pluggy" +version = "1.5.0" +requires_python = ">=3.8" +summary = "plugin and hook calling mechanisms for python" +groups = ["dev"] +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + [[package]] name = "prometheus-client" version = "0.21.1" @@ -2561,6 +2583,25 @@ files = [ {file = "pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8"}, ] +[[package]] +name = "pytest" +version = "8.3.5" +requires_python = ">=3.8" +summary = "pytest: simple powerful testing with Python" +groups = ["dev"] +dependencies = [ + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "iniconfig", + "packaging", + "pluggy<2,>=1.5", + "tomli>=1; python_version < \"3.11\"", +] +files = [ + {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, + {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2936,9 +2977,9 @@ files = [ [[package]] name = "sentence-transformers" -version = "3.4.1" +version = "4.0.1" requires_python = ">=3.9" -summary = "State-of-the-Art Text Embeddings" +summary = "Embeddings, Retrieval, and Reranking" groups = ["default"] dependencies = [ "Pillow", @@ -2948,10 +2989,11 @@ dependencies = [ "torch>=1.11.0", "tqdm", "transformers<5.0.0,>=4.41.0", + "typing-extensions>=4.5.0", ] files = [ - {file = "sentence_transformers-3.4.1-py3-none-any.whl", hash = "sha256:e026dc6d56801fd83f74ad29a30263f401b4b522165c19386d8bc10dcca805da"}, - {file = "sentence_transformers-3.4.1.tar.gz", hash = "sha256:68daa57504ff548340e54ff117bd86c1d2f784b21e0fb2689cf3272b8937b24b"}, + {file = "sentence_transformers-4.0.1-py3-none-any.whl", hash = "sha256:bb037b22d2766b94fa60f5c4fb9a876679b2311b509ae266259434d6de0875fd"}, + {file = "sentence_transformers-4.0.1.tar.gz", hash = "sha256:0c0080a08dc5670b17c0c88791d15b06c0fa1a598d26714732760d1832bc7d88"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index c940780..84b7377 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dev = [ "mypy>=1.15.0", "ruff>=0.8.0", "pylint>=3.3.2", + "pytest>=8.3.2", ] [project.scripts] diff --git a/tests/mocks/__init__.py b/tests/mocks/__init__.py new file mode 100644 index 0000000..7e79b48 --- /dev/null +++ b/tests/mocks/__init__.py @@ -0,0 +1 @@ +"""Mocks for tests.""" diff --git a/tests/mocks/mock_classes.py b/tests/mocks/mock_classes.py new file mode 100644 index 0000000..8b5c0a8 --- /dev/null +++ b/tests/mocks/mock_classes.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from langchain_core.runnables import Runnable + + +@dataclass +class MockResult: + """Mock of LLM response""" + + content: str + + +class MockLLM(Runnable): + """Mock LLM class""" + + def invoke(self, input, config=None, **kwargs): + return MockResult("5") diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..9fb5a03 --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1 @@ +"""Tests for evalutation utilities.""" diff --git a/tests/utils/test_similarity_score_llm.py b/tests/utils/test_similarity_score_llm.py new file mode 100644 index 0000000..fd1574a --- /dev/null +++ b/tests/utils/test_similarity_score_llm.py @@ -0,0 +1,9 @@ +from road_core_eval.utils.similarity_score_llm import AnswerSimilarityScore +from tests.mocks.mock_classes import MockLLM + + +def test_similarity_score(): + mock_llm = MockLLM() + + similarity_scorer = AnswerSimilarityScore(mock_llm) + assert similarity_scorer.get_score("question", "answer", "response") == 0.5 From f822e40dfe64390bf9ece2adb8e718b2fdce2c36 Mon Sep 17 00:00:00 2001 From: Jiri Podivin Date: Fri, 28 Mar 2025 11:00:08 +0100 Subject: [PATCH 2/2] Formatting tests Signed-off-by: Jiri Podivin --- Makefile | 2 +- tests/mocks/mock_classes.py | 4 +++- tests/utils/test_similarity_score_llm.py | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7c9e9fa..b9e5309 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ format: install-deps-test ## Format the code into unified format verify: install-deps-test ## Verify the code using various linters pdm run black . --check pdm run ruff check . --per-file-ignores=tests/*:S101 --per-file-ignores=scripts/*:S101 - pdm run pylint src + pdm run pylint src tests requirements.txt: pyproject.toml pdm.lock ## Generate requirements.txt file containing hashes for all non-devel packages pdm export --prod --format requirements --output requirements.txt diff --git a/tests/mocks/mock_classes.py b/tests/mocks/mock_classes.py index 8b5c0a8..d76e54f 100644 --- a/tests/mocks/mock_classes.py +++ b/tests/mocks/mock_classes.py @@ -1,3 +1,5 @@ +"""Mocks for various classes for use during testing.""" + from dataclasses import dataclass from langchain_core.runnables import Runnable @@ -12,5 +14,5 @@ class MockResult: class MockLLM(Runnable): """Mock LLM class""" - def invoke(self, input, config=None, **kwargs): + def invoke(self, input, config=None, **kwargs): # pylint: disable=W0622 return MockResult("5") diff --git a/tests/utils/test_similarity_score_llm.py b/tests/utils/test_similarity_score_llm.py index fd1574a..85f9840 100644 --- a/tests/utils/test_similarity_score_llm.py +++ b/tests/utils/test_similarity_score_llm.py @@ -1,8 +1,11 @@ +"""Tests for similarity score utilities""" + from road_core_eval.utils.similarity_score_llm import AnswerSimilarityScore from tests.mocks.mock_classes import MockLLM def test_similarity_score(): + """Test similarity score calculation.""" mock_llm = MockLLM() similarity_scorer = AnswerSimilarityScore(mock_llm)