diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 40ace8b..8557222 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,13 +5,6 @@ ################################################################################ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu20_cpython - image_name: ubuntu-20.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: @@ -21,14 +14,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos12_cpython - image_name: macOS-12 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython diff --git a/src/fetchcode/__init__.py b/src/fetchcode/__init__.py index 5d05242..82523d6 100644 --- a/src/fetchcode/__init__.py +++ b/src/fetchcode/__init__.py @@ -44,6 +44,7 @@ def fetch_http(url, location): `url` URL string saving the content in a file at `location` """ r = requests.get(url) + with open(location, "wb") as f: f.write(r.content) @@ -106,3 +107,17 @@ def fetch(url): return fetchers.get(scheme)(url, location) raise Exception("Not a supported/known scheme.") + + +def fetch_json_response(url): + """ + Fetch a JSON response from the given URL and return the parsed JSON data. + """ + response = requests.get(url) + if response.status_code != 200: + raise Exception(f"Failed to fetch {url}: {response.status_code} {response.reason}") + + try: + return response.json() + except ValueError as e: + raise Exception(f"Failed to parse JSON from {url}: {str(e)}") diff --git a/src/fetchcode/download_urls.py b/src/fetchcode/download_urls.py new file mode 100644 index 0000000..8594006 --- /dev/null +++ b/src/fetchcode/download_urls.py @@ -0,0 +1,41 @@ +# fetchcode is a free software tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/fetchcode for support and download. +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and http://aboutcode.org +# +# This software is licensed under the Apache License version 2.0. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: +# http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from packageurl.contrib.route import NoRouteAvailable +from packageurl.contrib.route import Router + +from fetchcode.pypi import Pypi + +package_registry = [ + Pypi, +] + +router = Router() + +for pkg_class in package_registry: + router.append(pattern=pkg_class.purl_pattern, endpoint=pkg_class.get_download_url) + + +def download_url(purl): + """ + Return package metadata for a URL or PURL. + Return None if there is no URL, or the URL or PURL is not supported. + """ + if purl: + try: + return router.process(purl) + except NoRouteAvailable: + return diff --git a/src/fetchcode/pypi.py b/src/fetchcode/pypi.py new file mode 100644 index 0000000..96b6916 --- /dev/null +++ b/src/fetchcode/pypi.py @@ -0,0 +1,58 @@ +# fetchcode is a free software tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/fetchcode for support and download. +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and http://aboutcode.org +# +# This software is licensed under the Apache License version 2.0. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: +# http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +from urllib.parse import urljoin + +from packageurl import PackageURL + +from fetchcode import fetch_json_response + + +class Pypi: + """ + This class handles Cargo PURLs. + """ + + purl_pattern = "pkg:pypi/.*" + base_url = "https://pypi.org/pypi/" + + @classmethod + def get_download_url(cls, purl): + """ + Return the download URL for a Pypi PURL. + """ + purl = PackageURL.from_string(purl) + + name = purl.name + version = purl.version + + if not name or not version: + raise ValueError("Pypi PURL must specify a name and version") + + url = urljoin(cls.base_url, f"{name}/{version}/json") + data = fetch_json_response(url) + + download_urls = data.get("urls", [{}]) + + if not download_urls: + raise ValueError(f"No download URLs found for {name} version {version}") + + download_url = next((url["url"] for url in download_urls if url.get("url")), None) + + if not download_url: + raise ValueError(f"No download URL found for {name} version {version}") + + return download_url diff --git a/tests/test_pypi.py b/tests/test_pypi.py new file mode 100644 index 0000000..93b9eb2 --- /dev/null +++ b/tests/test_pypi.py @@ -0,0 +1,76 @@ +import unittest +from unittest.mock import patch + +from fetchcode.pypi import Pypi + + +class TestGetDownloadURL(unittest.TestCase): + @patch("fetchcode.pypi.fetch_json_response") + def test_valid_purl_returns_download_url(self, mock_fetch_json_response): + mock_response = { + "urls": [ + { + "url": "https://files.pythonhosted.org/packages/source/r/requests/requests-2.31.0.tar.gz" + } + ] + } + mock_fetch_json_response.return_value = mock_response + + purl = "pkg:pypi/requests@2.31.0" + result = Pypi.get_download_url(purl) + self.assertEqual( + result, + "https://files.pythonhosted.org/packages/source/r/requests/requests-2.31.0.tar.gz", + ) + + @patch("fetchcode.pypi.fetch_json_response") + def test_missing_version_raises_value_error(self, mock_fetch_json_response): + purl = "pkg:pypi/requests" + with self.assertRaises(ValueError) as context: + Pypi.get_download_url(purl) + self.assertIn("Pypi PURL must specify a name and version", str(context.exception)) + + @patch("fetchcode.pypi.fetch_json_response") + def test_missing_name_raises_value_error(self, mock_fetch_json_response): + purl = "pkg:pypi/@2.31.0" + with self.assertRaises(ValueError) as context: + Pypi.get_download_url(purl) + self.assertIn("purl is missing the required name component", str(context.exception)) + + @patch("fetchcode.pypi.fetch_json_response") + def test_missing_urls_field_raises_value_error(self, mock_fetch_json_response): + mock_fetch_json_response.return_value = {} + purl = "pkg:pypi/requests@2.31.0" + with self.assertRaises(ValueError) as context: + Pypi.get_download_url(purl) + self.assertIn("No download URL found", str(context.exception)) + + @patch("fetchcode.pypi.fetch_json_response") + def test_empty_urls_list_raises_value_error(self, mock_fetch_json_response): + mock_fetch_json_response.return_value = {"urls": []} + purl = "pkg:pypi/requests@2.31.0" + with self.assertRaises(ValueError) as context: + Pypi.get_download_url(purl) + self.assertIn("No download URLs found", str(context.exception)) + + @patch("fetchcode.pypi.fetch_json_response") + def test_first_url_object_missing_url_key(self, mock_fetch_json_response): + mock_fetch_json_response.return_value = {"urls": [{}]} + purl = "pkg:pypi/requests@2.31.0" + with self.assertRaises(ValueError) as context: + Pypi.get_download_url(purl) + self.assertIn("No download URL found", str(context.exception)) + + @patch("fetchcode.pypi.fetch_json_response") + def test_url_fallback_when_multiple_urls_provided(self, mock_fetch_json_response): + mock_fetch_json_response.return_value = { + "urls": [{}, {"url": "https://example.com/fallback-url.tar.gz"}] + } + + purl = "pkg:pypi/requests@2.31.0" + download_url = Pypi.get_download_url(purl) + self.assertEqual(download_url, "https://example.com/fallback-url.tar.gz") + + def test_malformed_purl_raises_exception(self): + with self.assertRaises(ValueError): + Pypi.get_download_url("this-is-not-a-valid-purl")