From 5edb0ece1e8f7ac6424eb86ce25c3014a41c9c85 Mon Sep 17 00:00:00 2001 From: Adityatorgal17 Date: Thu, 5 Feb 2026 02:17:38 +0530 Subject: [PATCH] Add support for PEP 751 pylock.toml lockfiles Signed-off-by: Adityatorgal17 --- src/packagedcode/__init__.py | 4 + src/packagedcode/pylock.py | 143 +++++++++++++++ .../data/plugin/plugins_list_linux.txt | 7 + .../data/pylock/pylock-empty.toml | 5 + .../data/pylock/pylock-invalid-format.toml | 4 + .../data/pylock/pylock-large.toml | 86 +++++++++ .../pylock/pylock-malformed-packages.toml | 15 ++ .../data/pylock/pylock-medium.toml | 41 +++++ .../data/pylock/pylock-nolock-version.toml | 6 + .../data/pylock/pylock-small.toml | 8 + tests/packagedcode/test_pylock.py | 163 ++++++++++++++++++ 11 files changed, 482 insertions(+) create mode 100644 src/packagedcode/pylock.py create mode 100644 tests/packagedcode/data/pylock/pylock-empty.toml create mode 100644 tests/packagedcode/data/pylock/pylock-invalid-format.toml create mode 100644 tests/packagedcode/data/pylock/pylock-large.toml create mode 100644 tests/packagedcode/data/pylock/pylock-malformed-packages.toml create mode 100644 tests/packagedcode/data/pylock/pylock-medium.toml create mode 100644 tests/packagedcode/data/pylock/pylock-nolock-version.toml create mode 100644 tests/packagedcode/data/pylock/pylock-small.toml create mode 100644 tests/packagedcode/test_pylock.py diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index d3c48b6e25..b16738665b 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -40,6 +40,7 @@ from packagedcode import swift from packagedcode import win_pe from packagedcode import windows +from packagedcode import pylock if on_linux: from packagedcode import msi @@ -212,6 +213,9 @@ # These are handlers for deplock generated files pypi.PipInspectDeplockHandler, + + # These are handlers for pylock generated files + pylock.PyLockHandler, ] if on_linux: diff --git a/src/packagedcode/pylock.py b/src/packagedcode/pylock.py new file mode 100644 index 0000000000..9661db63f7 --- /dev/null +++ b/src/packagedcode/pylock.py @@ -0,0 +1,143 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +try: + import tomllib +except ImportError: + # Fallback for Python < 3.11 + import tomli as tomllib + +from packagedcode import models +from packaging.utils import canonicalize_name +from packageurl import PackageURL + +logger = logging.getLogger(__name__) + + +class PyLockHandler(models.DatafileHandler): + """ + Handler for PEP 751 pylock.toml lockfiles. + + PyLock is a standardized lockfile format for Python packages defined in PEP 751. + It provides a machine-readable record of the exact package versions and metadata + used in a Python environment. + See https://peps.python.org/pep-0751/ + """ + + datasource_id = 'pylock' + path_patterns = ('*/pylock.toml',) + default_package_type = 'pypi' + default_primary_language = 'Python' + is_lockfile = True + description = 'Python pylock.toml lockfile (PEP 751)' + documentation_url = 'https://peps.python.org/pep-0751/' + + @classmethod + def parse(cls, location, package_only=False): + """ + Parse a pylock.toml file and extract package information. + + Returns a generator of Package objects representing the packages + listed in the lockfile. + """ + try: + with open(location, 'rb') as f: + data = tomllib.load(f) + except (OSError, tomllib.TOMLDecodeError) as e: + logger.warning( + 'Failed to parse pylock.toml at %r: %s', + location, + e, + ) + return + + # Validate PEP 751 requirement - 'lock-version' is required + if 'lock-version' not in data: + logger.warning( + 'Missing required lock-version in pylock.toml at %r', + location, + ) + return + + packages = data.get('packages', []) + if not packages: + logger.debug( + 'No packages found in pylock.toml at %r', + location, + ) + return + + dependencies = [] + if not package_only: + for pkg in packages: + if not isinstance(pkg, dict): + continue + + raw_name = pkg.get('name') + version = pkg.get('version') + + # Required fields validation + if not raw_name or not version: + logger.debug( + 'Skipping package with missing name or version in %r', + location, + ) + continue + + name = canonicalize_name(raw_name) + + purl = PackageURL( + type='pypi', + name=name, + version=version, + ).to_string() + + # Extract optional dependency metadata + dep_extra_data = {} + if 'marker' in pkg: + dep_extra_data['marker'] = pkg['marker'] + if 'requires-python' in pkg: + dep_extra_data['requires_python'] = pkg['requires-python'] + + # Check if dependency is optional based on marker + is_optional = 'marker' in pkg and 'extra' in pkg.get('marker', '') + + dependencies.append( + models.DependentPackage( + purl=purl, + extracted_requirement=version, + scope='runtime', + is_runtime=True, + is_optional=is_optional, + is_pinned=True, + extra_data=dep_extra_data, + ) + ) + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name='python-environment', + version=None, + primary_language=cls.default_primary_language, + dependencies=dependencies, + is_virtual=True, + extra_data={ + 'lock_version': data.get('lock-version'), + 'package_count': len(packages), + 'created_by': data.get('created-by'), + 'requires_python': data.get('requires-python'), + 'environments': data.get('environments'), + 'dependency_groups': data.get('dependency-groups'), + 'default_groups': data.get('default-groups'), + } + ) + + yield models.PackageData(**package_data) diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt index eb4763d6c7..8fb41b3f1b 100755 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -720,6 +720,13 @@ Package type: pypi description: Pipfile.lock path_patterns: '*Pipfile.lock' -------------------------------------------- +Package type: pypi + datasource_id: pylock + documentation URL: https://peps.python.org/pep-0751/ + primary language: Python + description: Python pylock.toml lockfile (PEP 751) + path_patterns: '*/pylock.toml' +-------------------------------------------- Package type: pypi datasource_id: pypi_editable_egg_pkginfo documentation URL: https://peps.python.org/pep-0376/ diff --git a/tests/packagedcode/data/pylock/pylock-empty.toml b/tests/packagedcode/data/pylock/pylock-empty.toml new file mode 100644 index 0000000000..2712c0fa22 --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-empty.toml @@ -0,0 +1,5 @@ +lock-version = "1.0" +created-by = "test" +requires-python = ">=3.8" + +# Empty packages array - no actual package entries \ No newline at end of file diff --git a/tests/packagedcode/data/pylock/pylock-invalid-format.toml b/tests/packagedcode/data/pylock/pylock-invalid-format.toml new file mode 100644 index 0000000000..94b1a42f2c --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-invalid-format.toml @@ -0,0 +1,4 @@ +This is not TOML at all! +Invalid syntax [[ without closing +Missing quotes and = signs +Complete garbage content \ No newline at end of file diff --git a/tests/packagedcode/data/pylock/pylock-large.toml b/tests/packagedcode/data/pylock/pylock-large.toml new file mode 100644 index 0000000000..71b88bf213 --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-large.toml @@ -0,0 +1,86 @@ +# This file was autogenerated by uv via the following command: +# uv export -o pylock.toml +lock-version = "1.0" +created-by = "uv" +requires-python = ">=3.13.2" + +[[packages]] +name = "aiohappyeyeballs" +version = "2.6.1" +index = "https://pypi.org/simple" +sdist = { + url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", + upload-time = 2025-03-12T01:42:48Z, + size = 22760, + hashes = { sha256 = "c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558" } +} +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", + upload-time = 2025-03-12T01:42:47Z, + size = 15265, + hashes = { sha256 = "f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8" } + } +] + +[[packages]] +name = "aiohttp" +version = "3.11.14" +index = "https://pypi.org/simple" +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ca/6ce3da7c3295e0655b3404a309c7002099ca3619aeb04d305cedc77a0a14/aiohttp-3.11.14-cp313-cp313-manylinux_x86_64.whl" }, + { url = "https://files.pythonhosted.org/packages/4a/e0/2f9e77ef2d4a1dbf05f40b7edf1e1ce9be72bdbe6037cf1db1712b455e3e/aiohttp-3.11.14-cp313-cp313-win_amd64.whl" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/d7f353c5aaf9f868ab382c3d3320dc6efaa639b6b30d5a686bed83196115/aiohttp-3.11.14-cp313-cp313-macosx_10_13_universal2.whl" } +] + +[[packages]] +name = "aiosignal" +version = "1.3.2" +index = "https://pypi.org/simple" +sdist = { + url = "https://files.pythonhosted.org/packages/ba/b5/6d55e80f6d8a08ce22b982eafa278d823b541c925f11ee774b0b9c43473d/aiosignal-1.3.2.tar.gz", + hashes = { sha256 = "a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54" } +} +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/6a/bc7e17a3e87a2985d3e8f4da4cd0f481060eb78fb08596c42be62c90a4d9/aiosignal-1.3.2-py2.py3-none-any.whl" } +] + +[[packages]] +name = "attrs" +version = "25.1.0" +index = "https://pypi.org/simple" +sdist = { + url = "https://files.pythonhosted.org/packages/49/7c/fdf464bcc51d23881d110abd74b512a42b3d5d376a55a831b44c603ae17f/attrs-25.1.0.tar.gz", + hashes = { sha256 = "1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e" } +} +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl" } +] + +[[packages]] +name = "colorama" +version = "0.4.6" +marker = "sys_platform == 'win32'" +index = "https://pypi.org/simple" +sdist = { + url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", + hashes = { sha256 = "08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44" } +} +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl" } +] + +[[packages]] +name = "multidict" +version = "6.1.0" +index = "https://pypi.org/simple" +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/64/43c5e4cfd4c0e6f8b5f6e63f9d88e9e4a9fd1c5f1f87e4d6c9d0e2/multidict-6.1.0-cp313-cp313-manylinux_x86_64.whl" } +] + +[[packages]] +name = "yarl" +version = "1.9.7" +index = "https://pypi.org/simple" +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/cc/9b3e0b94c9f38d8b47c0b41a1b0f1c4fd87b9dbb4bba1d4a2bdb7f/yarl-1.9.7-cp313-cp313-manylinux_x86_64.whl" } +] diff --git a/tests/packagedcode/data/pylock/pylock-malformed-packages.toml b/tests/packagedcode/data/pylock/pylock-malformed-packages.toml new file mode 100644 index 0000000000..3a580de494 --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-malformed-packages.toml @@ -0,0 +1,15 @@ +lock-version = "1.0" +created-by = "uv" +requires-python = ">=3.13" + +[[packages]] +name = "aiohttp" +# missing version + +[[packages]] +version = "1.0.0" +# missing name + +[[packages]] +name = "attrs" +version = "25.1.0" diff --git a/tests/packagedcode/data/pylock/pylock-medium.toml b/tests/packagedcode/data/pylock/pylock-medium.toml new file mode 100644 index 0000000000..6e65986803 --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-medium.toml @@ -0,0 +1,41 @@ +lock-version = '1.0' +environments = ["sys_platform == 'win32'", "sys_platform == 'linux'"] +requires-python = '==3.12' +created-by = 'mousebender' + +[[packages]] +name = 'attrs' +version = '25.1.0' +requires-python = '>=3.8' +wheels = [ + {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}}, +] +[[packages.attestation-identities]] +environment = 'release-pypi' +kind = 'GitHub' +repository = 'python-attrs/attrs' +workflow = 'pypi-package.yml' + +[[packages]] +name = 'cattrs' +version = '24.1.2' +requires-python = '>=3.8' +dependencies = [ + {name = 'attrs'}, +] +wheels = [ + {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, +] + +[[packages]] +name = 'numpy' +version = '2.2.3' +requires-python = '>=3.10' +wheels = [ + {name = 'numpy-2.2.3-cp312-cp312-win_amd64.whl', upload-time = 2025-02-13T16:51:21.821880+00:00, url = 'https://files.pythonhosted.org/packages/42/6e/55580a538116d16ae7c9aa17d4edd56e83f42126cb1dfe7a684da7925d2c/numpy-2.2.3-cp312-cp312-win_amd64.whl', size = 12626357, hashes = {sha256 = '83807d445817326b4bcdaaaf8e8e9f1753da04341eceec705c001ff342002e5d'}}, + {name = 'numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', upload-time = 2025-02-13T16:50:00.079662+00:00, url = 'https://files.pythonhosted.org/packages/39/04/78d2e7402fb479d893953fb78fa7045f7deb635ec095b6b4f0260223091a/numpy-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl', size = 16116679, hashes = {sha256 = '3b787adbf04b0db1967798dba8da1af07e387908ed1553a0d6e74c084d1ceafe'}}, +] + +[tool.mousebender] +command = ['.', 'lock', '--platform', 'cpython3.12-windows-x64', '--platform', 'cpython3.12-manylinux2014-x64', 'cattrs', 'numpy'] +run-on = 2025-03-06T12:28:57.760769 \ No newline at end of file diff --git a/tests/packagedcode/data/pylock/pylock-nolock-version.toml b/tests/packagedcode/data/pylock/pylock-nolock-version.toml new file mode 100644 index 0000000000..c70d1da983 --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-nolock-version.toml @@ -0,0 +1,6 @@ +created-by = "uv" +requires-python = ">=3.13" + +[[packages]] +name = "aiohttp" +version = "3.11.14" diff --git a/tests/packagedcode/data/pylock/pylock-small.toml b/tests/packagedcode/data/pylock/pylock-small.toml new file mode 100644 index 0000000000..501af27455 --- /dev/null +++ b/tests/packagedcode/data/pylock/pylock-small.toml @@ -0,0 +1,8 @@ +lock-version = "1.0" +created-by = "uv" +requires-python = ">=3.13" + +[[packages]] +name = "attrs" +version = "25.1.0" +index = "https://pypi.org/simple" diff --git a/tests/packagedcode/test_pylock.py b/tests/packagedcode/test_pylock.py new file mode 100644 index 0000000000..e0fd40ec65 --- /dev/null +++ b/tests/packagedcode/test_pylock.py @@ -0,0 +1,163 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +""" +Tests for PEP 751 pylock.toml parsing. +""" + +import os + +from packagedcode.pylock import PyLockHandler +from packages_test_utils import PackageTester + + +class TestPyLockHandler(PackageTester): + + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_small_pylock(self): + test_file = self.get_test_loc('pylock/pylock-small.toml') + results = list(PyLockHandler.parse(test_file)) + + assert len(results) == 1 + package = results[0] + + assert package.datasource_id == 'pylock' + assert package.type == 'pypi' + assert package.name == 'python-environment' + assert package.version is None + assert package.primary_language == 'Python' + assert package.is_virtual + + assert len(package.dependencies) == 1 + dep = package.dependencies[0] + + assert dep.purl == 'pkg:pypi/attrs@25.1.0' + assert dep.extracted_requirement == '25.1.0' + assert dep.scope == 'runtime' + assert dep.is_runtime + assert dep.is_optional is False + assert dep.is_pinned + + assert package.extra_data['lock_version'] == '1.0' + assert package.extra_data['created_by'] == 'uv' + assert package.extra_data['requires_python'] == '>=3.13' + assert package.extra_data['package_count'] >= 1 + + def test_parse_medium_pylock(self): + test_file = self.get_test_loc('pylock/pylock-medium.toml') + results = list(PyLockHandler.parse(test_file)) + + assert len(results) == 1 + package = results[0] + + assert package.datasource_id == 'pylock' + assert package.type == 'pypi' + assert package.is_virtual + assert package.extra_data['package_count'] >= 3 + + purls = {dep.purl for dep in package.dependencies} + assert { + 'pkg:pypi/attrs@25.1.0', + 'pkg:pypi/cattrs@24.1.2', + 'pkg:pypi/numpy@2.2.3', + } == purls + + for dep in package.dependencies: + assert dep.scope == 'runtime' + assert dep.is_runtime + assert dep.is_optional is False + assert dep.is_pinned + + # Check optional fields are extracted + assert package.extra_data['created_by'] == 'mousebender' + assert package.extra_data['requires_python'] == '==3.12' + + def test_package_only_mode(self): + test_file = self.get_test_loc('pylock/pylock-small.toml') + results = list(PyLockHandler.parse(test_file, package_only=True)) + + assert len(results) == 1 + package = results[0] + + assert package.datasource_id == 'pylock' + assert package.type == 'pypi' + assert package.is_virtual + assert package.name == 'python-environment' + assert package.dependencies == [] + + assert package.extra_data['lock_version'] == '1.0' + assert package.extra_data['package_count'] >= 1 + + def test_large_pylock_invalid_toml(self): + """ + pylock-large.toml contains invalid TOML syntax. + Parsing should fail gracefully. + """ + test_file = self.get_test_loc('pylock/pylock-large.toml') + results = list(PyLockHandler.parse(test_file)) + + assert results == [] + + def test_malformed_packages_pylock(self): + """ + Malformed pylock files should not crash parsing. + Should skip packages with missing required fields but parse valid ones. + """ + test_file = self.get_test_loc('pylock/pylock-malformed-packages.toml') + results = list(PyLockHandler.parse(test_file)) + + assert len(results) == 1 + package = results[0] + assert package.is_virtual + assert package.name == 'python-environment' + + # Should only have one valid dependency (attrs) and skip malformed ones + assert len(package.dependencies) == 1 + dep = package.dependencies[0] + assert dep.purl == 'pkg:pypi/attrs@25.1.0' + assert dep.is_pinned + + assert package.extra_data['lock_version'] == '1.0' + assert package.extra_data['package_count'] == 3 + + def test_missing_lock_version(self): + test_file = self.get_test_loc('pylock/pylock-nolock-version.toml') + results = list(PyLockHandler.parse(test_file)) + + assert results == [] + + def test_handler_metadata(self): + assert PyLockHandler.datasource_id == 'pylock' + assert PyLockHandler.path_patterns == ('*/pylock.toml',) + assert PyLockHandler.default_package_type == 'pypi' + assert PyLockHandler.default_primary_language == 'Python' + assert PyLockHandler.is_lockfile is True + assert 'PEP 751' in PyLockHandler.description + assert PyLockHandler.documentation_url == 'https://peps.python.org/pep-0751/' + + def test_empty_packages_list(self): + """ + Test pylock files with empty packages array. + """ + test_file = self.get_test_loc('pylock/pylock-empty.toml') + results = list(PyLockHandler.parse(test_file)) + assert results == [] + + def test_file_not_found(self): + results = list(PyLockHandler.parse('/non/existent/file.toml')) + assert results == [] + + def test_invalid_file_format(self): + """ + Test files with invalid TOML format should fail gracefully. + """ + test_file = self.get_test_loc('pylock/pylock-invalid-format.toml') + results = list(PyLockHandler.parse(test_file)) + assert results == []