Skip to content

Commit b56b3c1

Browse files
TG1999TG1999
committed
Migrate Istio importer
Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 6f16ab5 commit b56b3c1

4 files changed

Lines changed: 264 additions & 0 deletions

File tree

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
elixir_security_importer as elixir_security_importer_v2,
4848
)
4949
from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
50+
from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2
5051
from vulnerabilities.pipelines.v2_importers import npm_importer as npm_importer_v2
5152
from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2
5253
from vulnerabilities.pipelines.v2_importers import oss_fuzz as oss_fuzz_v2
@@ -69,6 +70,7 @@
6970
xen_importer_v2.XenImporterPipeline,
7071
curl_importer_v2.CurlImporterPipeline,
7172
oss_fuzz_v2.OSSFuzzImporterPipeline,
73+
istio_importer_v2.IstioImporterPipeline,
7274
nvd_importer.NVDImporterPipeline,
7375
github_importer.GitHubAPIImporterPipeline,
7476
gitlab_importer.GitLabImporterPipeline,

‎vulnerabilities/pipelines/v2_importers/curl_importer.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ class CurlImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
3737
license_url = "https://curl.se/docs/copyright.html"
3838
repo_url = "https://github.com/curl/curl-www/"
3939
url = "https://curl.se/docs/vuln.json"
40+
unfurl_version_ranges = True
4041

4142
@classmethod
4243
def steps(cls):

‎vulnerabilities/pipelines/v2_importers/istio_importer.py‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import re
11+
from pathlib import Path
12+
from typing import Iterable
13+
from typing import List
14+
15+
import pytz
16+
import saneyaml
17+
from dateutil import parser
18+
from fetchcode.vcs import fetch_via_vcs
19+
from packageurl import PackageURL
20+
from univers.version_constraint import VersionConstraint
21+
from univers.version_range import GitHubVersionRange
22+
from univers.version_range import GolangVersionRange
23+
from univers.versions import SemverVersion
24+
25+
from vulnerabilities.importer import AdvisoryData
26+
from vulnerabilities.importer import AffectedPackage
27+
from vulnerabilities.importer import ReferenceV2
28+
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
29+
from vulnerabilities.utils import get_advisory_url
30+
from vulnerabilities.utils import split_markdown_front_matter
31+
32+
is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
33+
34+
35+
class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
36+
"""
37+
Importer for Istio.io security advisories.
38+
"""
39+
40+
pipeline_id = "istio_importer_v2"
41+
spdx_license_expression = "Apache-2.0"
42+
license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
43+
repo_url = "git+https://github.com/istio/istio.io"
44+
unfurl_version_ranges = True
45+
46+
@classmethod
47+
def steps(cls):
48+
return (
49+
cls.clone,
50+
cls.collect_and_store_advisories,
51+
cls.clean_downloads,
52+
)
53+
54+
def advisories_count(self) -> int:
55+
base_path = Path(self.vcs_response.dest_dir)
56+
advisories_dir = base_path / "content/en/news/security"
57+
return sum(
58+
1 for file in advisories_dir.rglob("*.md") if not file.name.endswith("_index.md")
59+
)
60+
61+
def clone(self):
62+
self.log(f"Cloning `{self.repo_url}`")
63+
self.vcs_response = fetch_via_vcs(self.repo_url)
64+
65+
def collect_advisories(self) -> Iterable[AdvisoryData]:
66+
base_path = Path(self.vcs_response.dest_dir)
67+
advisories_dir = base_path / "content/en/news/security"
68+
69+
for md_file in advisories_dir.rglob("*.md"):
70+
if md_file.name.endswith("_index.md"):
71+
continue
72+
73+
data = self.parse_markdown(md_file)
74+
advisory_url = get_advisory_url(
75+
file=md_file,
76+
base_path=base_path,
77+
url="https://github.com/istio/istio.io/blob/master/",
78+
)
79+
published_date = data.get("publishdate")
80+
release_date = (
81+
parser.parse(published_date).replace(tzinfo=pytz.UTC) if published_date else None
82+
)
83+
constraints = self.get_version_constraints(data.get("releases", []))
84+
85+
cves = data.get("cves", [])
86+
87+
affected_packages = []
88+
if constraints:
89+
affected_packages.extend(
90+
[
91+
AffectedPackage(
92+
package=PackageURL(type="golang", namespace="istio.io", name="istio"),
93+
affected_version_range=GolangVersionRange(constraints=constraints),
94+
),
95+
AffectedPackage(
96+
package=PackageURL(type="github", namespace="istio", name="istio"),
97+
affected_version_range=GitHubVersionRange(constraints=constraints),
98+
),
99+
]
100+
)
101+
102+
title = data.get("title") or ""
103+
summary = data.get("description") or ""
104+
references = []
105+
if title:
106+
references.append(
107+
ReferenceV2(
108+
reference_id=title,
109+
url=f"https://istio.io/latest/news/security/{title}/",
110+
)
111+
)
112+
113+
yield AdvisoryData(
114+
advisory_id=title,
115+
aliases=cves,
116+
summary=summary,
117+
affected_packages=affected_packages,
118+
references_v2=references,
119+
date_published=release_date,
120+
url=advisory_url,
121+
original_advisory_text=md_file.read_text(encoding="utf-8"),
122+
)
123+
124+
def parse_markdown(self, path: Path) -> dict:
125+
"""Return a mapping of vulnerability data extracted from an advisory."""
126+
text = path.read_text(encoding="utf-8")
127+
front_matter, _ = split_markdown_front_matter(text)
128+
return saneyaml.load(front_matter)
129+
130+
def get_version_constraints(self, releases: List[str]) -> List[VersionConstraint]:
131+
constraints = []
132+
for release in releases:
133+
release = release.strip()
134+
135+
if "All releases prior" in release:
136+
_, _, version = release.rpartition(" ")
137+
constraints.append(
138+
VersionConstraint(version=SemverVersion(version), comparator="<")
139+
)
140+
141+
elif "All releases" in release and "and later" in release:
142+
version = release.replace("All releases", "").replace("and later", "").strip()
143+
if is_release(version):
144+
constraints.append(
145+
VersionConstraint(version=SemverVersion(version), comparator=">=")
146+
)
147+
148+
elif "to" in release:
149+
lower, _, upper = release.partition("to")
150+
constraints.append(
151+
VersionConstraint(version=SemverVersion(lower.strip()), comparator=">=")
152+
)
153+
constraints.append(
154+
VersionConstraint(version=SemverVersion(upper.strip()), comparator="<=")
155+
)
156+
157+
elif is_release(release):
158+
constraints.append(
159+
VersionConstraint(version=SemverVersion(release), comparator="=")
160+
)
161+
162+
return constraints
163+
164+
def clean_downloads(self):
165+
if self.vcs_response:
166+
self.log("Removing cloned repository")
167+
self.vcs_response.delete()
168+
169+
def on_failure(self):
170+
self.clean_downloads()

‎vulnerabilities/tests/pipelines/test_istio_importer_v2.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import tempfile
2+
from pathlib import Path
3+
from textwrap import dedent
4+
5+
import pytest
6+
from packageurl import PackageURL
7+
from univers.version_constraint import VersionConstraint
8+
from univers.version_range import GitHubVersionRange
9+
from univers.version_range import GolangVersionRange
10+
from univers.versions import SemverVersion
11+
12+
from vulnerabilities.importer import AdvisoryData
13+
from vulnerabilities.importer import AffectedPackage
14+
from vulnerabilities.importer import ReferenceV2
15+
from vulnerabilities.pipelines.v2_importers.istio_importer import IstioImporterPipeline
16+
17+
18+
@pytest.mark.django_db
19+
def test_istio_advisory_parsing():
20+
sample_md = dedent(
21+
"""\
22+
---
23+
title: ISTIO-SECURITY-2019-002
24+
subtitle: Security Bulletin
25+
description: Denial of service affecting JWT access token parsing.
26+
cves: [CVE-2019-12995]
27+
cvss: "7.5"
28+
vector: "AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H/E:F/RL:O/RC:C"
29+
cvss_version: "3.0"
30+
releases: ["1.0 to 1.0.8", "1.1 to 1.1.9", "1.2 to 1.2.1"]
31+
publishdate: 2019-06-28
32+
keywords: [CVE]
33+
skip_seealso: true
34+
aliases:
35+
- /blog/2019/cve-2019-12995
36+
- /news/2019/cve-2019-12995
37+
---
38+
39+
A bug in Istio’s JWT validation filter causes Envoy to crash...
40+
"""
41+
)
42+
43+
with tempfile.TemporaryDirectory() as tmp_dir:
44+
base_path = Path(tmp_dir)
45+
advisory_dir = base_path / "content/en/news/security"
46+
advisory_dir.mkdir(parents=True)
47+
advisory_file = advisory_dir / "ISTIO-SECURITY-2019-002.md"
48+
advisory_file.write_text(sample_md, encoding="utf-8")
49+
50+
importer = IstioImporterPipeline()
51+
importer.vcs_response = type(
52+
"FakeVCS", (), {"dest_dir": tmp_dir, "delete": lambda x: None}
53+
)()
54+
55+
advisories = list(importer.collect_advisories())
56+
57+
assert len(advisories) == 1
58+
advisory = advisories[0]
59+
60+
assert isinstance(advisory, AdvisoryData)
61+
assert advisory.advisory_id == "ISTIO-SECURITY-2019-002"
62+
assert advisory.aliases == ["CVE-2019-12995"]
63+
assert advisory.summary.startswith("Denial of service affecting JWT access token")
64+
assert advisory.date_published.isoformat() == "2019-06-28T00:00:00+00:00"
65+
assert advisory.url.endswith("ISTIO-SECURITY-2019-002.md")
66+
assert advisory.references_v2[0] == ReferenceV2(
67+
reference_id="ISTIO-SECURITY-2019-002",
68+
url="https://istio.io/latest/news/security/ISTIO-SECURITY-2019-002/",
69+
)
70+
71+
expected_versions = [
72+
VersionConstraint(version=SemverVersion("1.0"), comparator=">="),
73+
VersionConstraint(version=SemverVersion("1.0.8"), comparator="<="),
74+
VersionConstraint(version=SemverVersion("1.1"), comparator=">="),
75+
VersionConstraint(version=SemverVersion("1.1.9"), comparator="<="),
76+
VersionConstraint(version=SemverVersion("1.2"), comparator=">="),
77+
VersionConstraint(version=SemverVersion("1.2.1"), comparator="<="),
78+
]
79+
80+
expected_packages = [
81+
AffectedPackage(
82+
package=PackageURL(type="golang", namespace="istio.io", name="istio"),
83+
affected_version_range=GolangVersionRange(constraints=expected_versions),
84+
),
85+
AffectedPackage(
86+
package=PackageURL(type="github", namespace="istio", name="istio"),
87+
affected_version_range=GitHubVersionRange(constraints=expected_versions),
88+
),
89+
]
90+
91+
assert advisory.affected_packages == expected_packages

0 commit comments

Comments
 (0)