diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml index 95c1bb2bc..885afa196 100644 --- a/.github/workflows/snapshot-publish.yml +++ b/.github/workflows/snapshot-publish.yml @@ -69,7 +69,7 @@ jobs: run: dotnet run scan --Verbosity Verbose --SourceDirectory ${{ github.workspace }}/test/Microsoft.ComponentDetection.VerificationTests/resources --Output ${{ github.workspace }}/output --DockerImagesToScan "docker.io/library/debian@sha256:9b0e3056b8cd8630271825665a0613cc27829d6a24906dc0122b3b4834312f7d,mcr.microsoft.com/cbl-mariner/base/core@sha256:c1bc83a3d385eccbb2f7f7da43a726c697e22a996f693a407c35ac7b4387cd59,docker.io/library/alpine@sha256:1304f174557314a7ed9eddb4eab12fed12cb0cd9809e4c28f29af86979a3c870" - --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff --DirectoryExclusionList "**/pip/parallel/**;**/pip/roots/**;**/pip/pre-generated/**;**/pip/fallback/**;**/pip/index-removal/**;**/pip/simple-extras/**;**/pip/pytestresultpkg/**" + --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff,DockerCompose=EnableIfDefaultOff,Helm=EnableIfDefaultOff,Kubernetes=EnableIfDefaultOff --DirectoryExclusionList "**/pip/parallel/**;**/pip/roots/**;**/pip/pre-generated/**;**/pip/fallback/**;**/pip/index-removal/**;**/pip/simple-extras/**;**/pip/pytestresultpkg/**" - name: Upload output folder uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 diff --git a/.github/workflows/snapshot-verify.yml b/.github/workflows/snapshot-verify.yml index 9645328f1..a9810c87a 100644 --- a/.github/workflows/snapshot-verify.yml +++ b/.github/workflows/snapshot-verify.yml @@ -101,7 +101,7 @@ jobs: run: dotnet run scan --Verbosity Verbose --SourceDirectory ${{ github.workspace }}/test/Microsoft.ComponentDetection.VerificationTests/resources --Output ${{ github.workspace }}/output --DockerImagesToScan "docker.io/library/debian@sha256:9b0e3056b8cd8630271825665a0613cc27829d6a24906dc0122b3b4834312f7d,mcr.microsoft.com/cbl-mariner/base/core@sha256:c1bc83a3d385eccbb2f7f7da43a726c697e22a996f693a407c35ac7b4387cd59,docker.io/library/alpine@sha256:1304f174557314a7ed9eddb4eab12fed12cb0cd9809e4c28f29af86979a3c870" - --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff --MaxDetectionThreads 5 --DirectoryExclusionList "**/pip/parallel/**;**/pip/roots/**;**/pip/pre-generated/**;**/pip/fallback/**;**/pip/index-removal/**;**/pip/simple-extras/**;**/pip/pytestresultpkg/**" + --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff,DockerCompose=EnableIfDefaultOff,Helm=EnableIfDefaultOff,Kubernetes=EnableIfDefaultOff --MaxDetectionThreads 5 --DirectoryExclusionList "**/pip/parallel/**;**/pip/roots/**;**/pip/pre-generated/**;**/pip/fallback/**;**/pip/index-removal/**;**/pip/simple-extras/**;**/pip/pytestresultpkg/**" - name: Run Verification Tests working-directory: test/Microsoft.ComponentDetection.VerificationTests diff --git a/.gitignore b/.gitignore index 533090632..1a86942c2 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,4 @@ launchSettings.json # Dev nupkgs dev-source +.nuget/ diff --git a/docs/detectors/README.md b/docs/detectors/README.md index 3309e9eed..d2f23eb87 100644 --- a/docs/detectors/README.md +++ b/docs/detectors/README.md @@ -18,6 +18,12 @@ | -------------------------- | ---------- | | CondaLockComponentDetector | DefaultOff | +- [Docker Compose](dockercompose.md) + +| Detector | Status | +| ------------------------------- | ---------- | +| DockerComposeComponentDetector | DefaultOff | + - [Dockerfile](dockerfile.md) | Detector | Status | @@ -42,12 +48,24 @@ | ----------------------- | ------ | | GradleComponentDetector | Stable | +- [Helm](helm.md) + +| Detector | Status | +| ---------------------- | ---------- | +| HelmComponentDetector | DefaultOff | + - [Ivy](ivy.md) | Detector | Status | | ----------- | ------------ | | IvyDetector | Experimental | +- [Kubernetes](kubernetes.md) + +| Detector | Status | +| ------------------------------ | ---------- | +| KubernetesComponentDetector | DefaultOff | + - [Linux](linux.md) | Detector | Status | diff --git a/docs/detectors/dockercompose.md b/docs/detectors/dockercompose.md new file mode 100644 index 000000000..7e523b44a --- /dev/null +++ b/docs/detectors/dockercompose.md @@ -0,0 +1,48 @@ +# Docker Compose Detection + +## Requirements + +Docker Compose detection depends on the following to successfully run: + +- One or more Docker Compose files matching the patterns: `docker-compose.yml`, `docker-compose.yaml`, `docker-compose.*.yml`, `docker-compose.*.yaml`, `compose.yml`, `compose.yaml`, `compose.*.yml`, `compose.*.yaml` + +The `DockerComposeComponentDetector` is a **DefaultOff** detector and must be explicitly enabled via the `--DetectorArgs` parameter. + +## Detection strategy + +The Docker Compose detector parses YAML compose files to extract Docker image references from service definitions. + +### Service Image Detection + +The detector looks for the `services` section and extracts the `image` field from each service: + +```yaml +services: + web: + image: nginx:1.21 + db: + image: postgres:14 +``` + +Services that only define a `build` directive without an `image` field are skipped, as they do not reference external Docker images. + +### Full Registry References + +The detector supports full registry image references: + +```yaml +services: + app: + image: ghcr.io/myorg/myapp:v2.0 +``` + +### Variable Resolution + +Images containing unresolved variables (e.g., `${TAG}` or `{{ .Values.tag }}`) are skipped to avoid reporting incomplete or incorrect references. The detector checks for `$`, `{`, or `}` characters in image references. + +## Known limitations + +- **DefaultOff Status**: This detector must be explicitly enabled using `--DetectorArgs DockerCompose=EnableIfDefaultOff` +- **Variable Resolution**: Image references containing unresolved environment variables or template expressions are not reported, which may lead to under-reporting in compose files that heavily use variable substitution +- **Build-Only Services**: Services that only specify a `build` directive without an `image` field are not reported +- **No Dependency Graph**: All detected images are registered as independent components without parent-child relationships diff --git a/docs/detectors/helm.md b/docs/detectors/helm.md new file mode 100644 index 000000000..cbad0e94f --- /dev/null +++ b/docs/detectors/helm.md @@ -0,0 +1,51 @@ +# Helm Detection + +## Requirements + +Helm detection depends on the following to successfully run: + +- One or more Helm values files matching the patterns: `*values*.yaml`, `*values*.yml` +- Chart metadata files (`Chart.yaml`, `Chart.yml`, `chart.yaml`, `chart.yml`) are matched for file discovery but only values files are parsed for image references + +The `HelmComponentDetector` is a **DefaultOff** detector and must be explicitly enabled via the `--DetectorArgs` parameter. + +## Detection strategy + +The Helm detector parses Helm values YAML files to extract Docker image references. It recursively walks the YAML tree looking for `image` keys. + +### Direct Image Strings + +The detector recognizes image references specified as simple strings: + +```yaml +image: nginx:1.21 +``` + +### Structured Image Objects + +The detector also supports the common Helm chart pattern of structured image definitions: + +```yaml +image: + registry: ghcr.io + repository: org/myimage + tag: v1.0 +``` + +The `registry` and `tag` fields are optional. When present, the detector reconstructs the full image reference. The `digest` field is also supported. + +### Recursive Search + +The detector recursively traverses all nested mappings and sequences in the values file, detecting image references at any depth in the YAML structure. + +### Variable Resolution + +Images containing unresolved variables (e.g., `{{ .Values.tag }}`) are skipped to avoid reporting incomplete or incorrect references. The detector checks for `$`, `{`, or `}` characters in image references. + +## Known limitations + +- **DefaultOff Status**: This detector must be explicitly enabled using `--DetectorArgs Helm=EnableIfDefaultOff` +- **Values Files Only**: Only files with `values` in the name are parsed for image references. Chart.yaml files are matched but not processed +- **Same-Directory Co-location**: Values files are only processed when a `Chart.yaml` (or `Chart.yml`) exists in the **same directory**. Values files in subdirectories of a chart root (e.g., `mychart/subdir/values.yaml`) will not be detected, even if a `Chart.yaml` exists in the parent directory +- **Variable Resolution**: Image references containing unresolved Helm template expressions are not reported +- **No Dependency Graph**: All detected images are registered as independent components without parent-child relationships diff --git a/docs/detectors/kubernetes.md b/docs/detectors/kubernetes.md new file mode 100644 index 000000000..78c53b706 --- /dev/null +++ b/docs/detectors/kubernetes.md @@ -0,0 +1,59 @@ +# Kubernetes Detection + +## Requirements + +Kubernetes detection depends on the following to successfully run: + +- One or more Kubernetes manifest files matching the patterns: `*.yaml`, `*.yml` +- Manifests must contain both `apiVersion` and `kind` fields to be recognized as Kubernetes resources + +The `KubernetesComponentDetector` is a **DefaultOff** detector and must be explicitly enabled via the `--DetectorArgs` parameter. + +## Detection strategy + +The Kubernetes detector parses Kubernetes manifest YAML files to extract Docker image references from container specifications. + +### Supported Resource Kinds + +The detector recognizes the following Kubernetes resource kinds: + +- `Pod` +- `PodTemplate` +- `Deployment` +- `StatefulSet` +- `DaemonSet` +- `ReplicaSet` +- `Job` +- `CronJob` +- `ReplicationController` + +Files with an unrecognized `kind` or missing `apiVersion`/`kind` fields are skipped. + +### Container Image Detection + +The detector extracts image references from all container types within pod specifications: + +- **containers**: Main application containers +- **initContainers**: Initialization containers that run before app containers +- **ephemeralContainers**: Ephemeral debugging containers + +### Pod Spec Locations + +The detector handles different pod spec locations depending on the resource kind: + +- **Pod**: `spec.containers` +- **Deployment, StatefulSet, DaemonSet, ReplicaSet, ReplicationController**: `spec.template.spec.containers` +- **Job**: `spec.template.spec.containers` +- **CronJob**: `spec.jobTemplate.spec.template.spec.containers` + +### Variable Resolution + +Images containing unresolved variables (e.g., `${TAG}`) are skipped to avoid reporting incomplete or incorrect references. The detector checks for `$`, `{`, or `}` characters in image references. + +## Known limitations + +- **DefaultOff Status**: This detector must be explicitly enabled using `--DetectorArgs Kubernetes=EnableIfDefaultOff` +- **Broad File Matching**: The `*.yaml` and `*.yml` search patterns match all YAML files, so the detector relies on content-based filtering (`apiVersion` and `kind` fields) to identify Kubernetes manifests +- **Variable Resolution**: Image references containing unresolved template variables are not reported +- **Limited Resource Kinds**: Only the eight resource kinds listed above are supported. Custom resources (CRDs) or other workload types are not detected +- **No Dependency Graph**: All detected images are registered as independent components without parent-child relationships diff --git a/src/Microsoft.ComponentDetection.Common/DockerReference/DockerReferenceUtility.cs b/src/Microsoft.ComponentDetection.Common/DockerReference/DockerReferenceUtility.cs index 022f8b612..63ce6f04a 100644 --- a/src/Microsoft.ComponentDetection.Common/DockerReference/DockerReferenceUtility.cs +++ b/src/Microsoft.ComponentDetection.Common/DockerReference/DockerReferenceUtility.cs @@ -38,6 +38,53 @@ public static class DockerReferenceUtility private const string LEGACYDEFAULTDOMAIN = "index.docker.io"; private const string OFFICIALREPOSITORYNAME = "library"; + /// + /// Returns true if the reference contains unresolved variable placeholders (e.g., ${VAR}, {{ .Values.tag }}). + /// Such references should be skipped before calling or . + /// + /// The image reference string to check. + /// true if the reference contains variable placeholder characters; otherwise false. + public static bool HasUnresolvedVariables(string reference) => + reference.IndexOfAny(['$', '{', '}']) >= 0; + + /// + /// Attempts to parse an image reference string into a . + /// Returns null if the reference contains unresolved variables or cannot be parsed. + /// + /// The image reference string to parse. + /// A if parsing succeeds; otherwise null. + public static DockerReference? TryParseImageReference(string imageReference) + { + if (HasUnresolvedVariables(imageReference)) + { + return null; + } + + try + { + return ParseFamiliarName(imageReference); + } + catch + { + return null; + } + } + + /// + /// Parses an image reference and registers it with the recorder if valid. + /// Silently skips references with unresolved variables or that cannot be parsed. + /// + /// The image reference string to parse. + /// The component recorder to register the image with. + public static void TryRegisterImageReference(string imageReference, ISingleFileComponentRecorder recorder) + { + var dockerRef = TryParseImageReference(imageReference); + if (dockerRef != null) + { + recorder.RegisterUsage(new DetectedComponent(dockerRef.ToTypedDockerReferenceComponent())); + } + } + public static DockerReference ParseQualifiedName(string qualifiedName) { var regexp = DockerRegex.ReferenceRegexp; diff --git a/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs b/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs index becfa4f9f..859be21c3 100644 --- a/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs +++ b/src/Microsoft.ComponentDetection.Contracts/DetectorClass.cs @@ -47,4 +47,13 @@ public enum DetectorClass /// Indicates a detector applies to Swift packages. Swift, + + /// Indicates a detector applies to Docker Compose image references. + DockerCompose, + + /// Indicates a detector applies to Helm chart image references. + Helm, + + /// Indicates a detector applies to Kubernetes manifest image references. + Kubernetes, } diff --git a/src/Microsoft.ComponentDetection.Detectors/dockercompose/DockerComposeComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dockercompose/DockerComposeComponentDetector.cs new file mode 100644 index 000000000..dba84ce7d --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/dockercompose/DockerComposeComponentDetector.cs @@ -0,0 +1,124 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.DockerCompose; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; +using YamlDotNet.RepresentationModel; + +public class DockerComposeComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +{ + public DockerComposeComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "DockerCompose"; + + public override IList SearchPatterns { get; } = + [ + "docker-compose.yml", "docker-compose.yaml", + "docker-compose.*.yml", "docker-compose.*.yaml", + "compose.yml", "compose.yaml", + "compose.*.yml", "compose.*.yaml", + ]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.DockerReference]; + + public override int Version => 1; + + public override IEnumerable Categories => [nameof(DetectorClass.DockerCompose)]; + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var file = processRequest.ComponentStream; + + try + { + this.Logger.LogInformation("Discovered Docker Compose file: {Location}", file.Location); + + string contents; + using (var reader = new StreamReader(file.Stream)) + { + contents = await reader.ReadToEndAsync(cancellationToken); + } + + var yaml = new YamlStream(); + yaml.Load(new StringReader(contents)); + + if (yaml.Documents.Count == 0) + { + return; + } + + foreach (var document in yaml.Documents) + { + if (document.RootNode is YamlMappingNode rootMapping) + { + this.ExtractImageReferences(rootMapping, singleFileComponentRecorder, file.Location); + } + } + } + catch (Exception e) + { + this.Logger.LogError(e, "Failed to parse Docker Compose file: {Location}", file.Location); + } + } + + private static YamlMappingNode? GetMappingChild(YamlMappingNode parent, string key) + { + foreach (var entry in parent.Children) + { + if (entry.Key is YamlScalarNode scalarKey && string.Equals(scalarKey.Value, key, StringComparison.OrdinalIgnoreCase)) + { + return entry.Value as YamlMappingNode; + } + } + + return null; + } + + private void ExtractImageReferences(YamlMappingNode rootMapping, ISingleFileComponentRecorder recorder, string fileLocation) + { + var services = GetMappingChild(rootMapping, "services"); + if (services == null) + { + return; + } + + foreach (var serviceEntry in services.Children) + { + if (serviceEntry.Value is not YamlMappingNode serviceMapping) + { + continue; + } + + // Extract direct image: references + foreach (var entry in serviceMapping.Children) + { + var key = (entry.Key as YamlScalarNode)?.Value; + if (string.Equals(key, "image", StringComparison.OrdinalIgnoreCase)) + { + var imageRef = (entry.Value as YamlScalarNode)?.Value; + if (!string.IsNullOrWhiteSpace(imageRef)) + { + DockerReferenceUtility.TryRegisterImageReference(imageRef, recorder); + } + } + } + } + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs index d1d19f3a9..46c42469f 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs @@ -5,7 +5,6 @@ namespace Microsoft.ComponentDetection.Detectors.Dockerfile; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common; @@ -143,20 +142,10 @@ private DockerReference ParseFromInstruction(DockerfileConstruct construct, char if (!string.IsNullOrEmpty(stageNameReference)) { - if (this.HasUnresolvedVariables(stageNameReference)) - { - return null; - } - - return DockerReferenceUtility.ParseFamiliarName(stageNameReference); - } - - if (this.HasUnresolvedVariables(reference)) - { - return null; + return DockerReferenceUtility.TryParseImageReference(stageNameReference); } - return DockerReferenceUtility.ParseFamiliarName(reference); + return DockerReferenceUtility.TryParseImageReference(reference); } private DockerReference ParseCopyInstruction(DockerfileConstruct construct, char escapeChar, Dictionary stageNameMap) @@ -172,26 +161,9 @@ private DockerReference ParseCopyInstruction(DockerfileConstruct construct, char stageNameMap.TryGetValue(reference, out var stageNameReference); if (!string.IsNullOrEmpty(stageNameReference)) { - if (this.HasUnresolvedVariables(stageNameReference)) - { - return null; - } - else - { - return DockerReferenceUtility.ParseFamiliarName(stageNameReference); - } + return DockerReferenceUtility.TryParseImageReference(stageNameReference); } - if (this.HasUnresolvedVariables(reference)) - { - return null; - } - - return DockerReferenceUtility.ParseFamiliarName(reference); - } - - private bool HasUnresolvedVariables(string reference) - { - return new Regex("[${}]").IsMatch(reference); + return DockerReferenceUtility.TryParseImageReference(reference); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs new file mode 100644 index 000000000..49f977a72 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs @@ -0,0 +1,233 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.Helm; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; +using YamlDotNet.RepresentationModel; + +public class HelmComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +{ + private readonly ConcurrentDictionary helmChartDirectories = new(StringComparer.OrdinalIgnoreCase); + + public HelmComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "Helm"; + + public override IList SearchPatterns { get; } = + [ + "Chart.yaml", "Chart.yml", + "*values*.yaml", "*values*.yml", + ]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.DockerReference]; + + public override int Version => 1; + + public override IEnumerable Categories => [nameof(DetectorClass.Helm)]; + + public override async Task ExecuteDetectorAsync(ScanRequest request, CancellationToken cancellationToken = default) + { + this.helmChartDirectories.Clear(); + return await base.ExecuteDetectorAsync(request, cancellationToken); + } + + protected override async Task> OnPrepareDetectionAsync( + IObservable processRequests, + IDictionary detectorArgs, + CancellationToken cancellationToken = default) + { + // Materialize all matching files first so that chart directories are fully + // known before any values file is decided on, regardless of enumeration order. + var allRequests = await processRequests.ToList().ToTask(cancellationToken); + + // Pass 1: record every directory that contains a Chart.yaml / Chart.yml. + foreach (var request in allRequests) + { + if (IsChartFile(Path.GetFileName(request.ComponentStream.Location))) + { + this.helmChartDirectories.TryAdd( + Path.GetDirectoryName(request.ComponentStream.Location)!, true); + } + } + + // Pass 2: emit only the values files that sit in a known chart directory. + return allRequests + .Where(r => + IsValuesFile(Path.GetFileName(r.ComponentStream.Location)) && + this.helmChartDirectories.ContainsKey(Path.GetDirectoryName(r.ComponentStream.Location)!)) + .ToObservable(); + } + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var file = processRequest.ComponentStream; + + // OnPrepareDetectionAsync has already filtered to values files co-located + // with a Chart.yaml — no further filename/directory checks are needed. + try + { + this.Logger.LogInformation("Discovered Helm values file: {Location}", file.Location); + + string contents; + using (var reader = new StreamReader(file.Stream)) + { + contents = await reader.ReadToEndAsync(cancellationToken); + } + + var yaml = new YamlStream(); + yaml.Load(new StringReader(contents)); + + if (yaml.Documents.Count == 0) + { + return; + } + + this.ExtractImageReferencesFromValues(yaml, processRequest.SingleFileComponentRecorder, file.Location); + } + catch (Exception e) + { + this.Logger.LogError(e, "Failed to parse Helm file: {Location}", file.Location); + } + } + + private static bool IsChartFile(string fileName) => + fileName.Equals("Chart.yaml", StringComparison.OrdinalIgnoreCase) || + fileName.Equals("Chart.yml", StringComparison.OrdinalIgnoreCase); + + private static bool IsValuesFile(string fileName) => + fileName.Contains("values", StringComparison.OrdinalIgnoreCase) && + (fileName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase)); + + private void ExtractImageReferencesFromValues(YamlStream yaml, ISingleFileComponentRecorder recorder, string fileLocation) + { + foreach (var document in yaml.Documents) + { + if (document.RootNode is YamlMappingNode rootMapping) + { + this.WalkYamlForImages(rootMapping, recorder, fileLocation); + } + } + } + + /// + /// Walks the YAML tree looking for image references. Handles two common patterns: + /// 1. Direct image string: `image: nginx:1.21` + /// 2. Structured image object: `image: { repository: nginx, tag: "1.21" }`. + /// + private void WalkYamlForImages(YamlMappingNode mapping, ISingleFileComponentRecorder recorder, string fileLocation) + { + foreach (var entry in mapping.Children) + { + var key = (entry.Key as YamlScalarNode)?.Value; + + if (string.Equals(key, "image", StringComparison.OrdinalIgnoreCase)) + { + switch (entry.Value) + { + // image: nginx:1.21 + case YamlScalarNode scalarValue when !string.IsNullOrWhiteSpace(scalarValue.Value): + DockerReferenceUtility.TryRegisterImageReference(scalarValue.Value, recorder); + break; + + // image: + // repository: nginx + // tag: "1.21" + case YamlMappingNode imageMapping: + this.TryRegisterStructuredImageReference(imageMapping, recorder, fileLocation); + break; + + default: + break; + } + } + else if (entry.Value is YamlMappingNode childMapping) + { + this.WalkYamlForImages(childMapping, recorder, fileLocation); + } + else if (entry.Value is YamlSequenceNode sequenceNode) + { + foreach (var item in sequenceNode) + { + if (item is YamlMappingNode sequenceMapping) + { + this.WalkYamlForImages(sequenceMapping, recorder, fileLocation); + } + } + } + } + } + + private void TryRegisterStructuredImageReference(YamlMappingNode imageMapping, ISingleFileComponentRecorder recorder, string fileLocation) + { + string? repository = null; + string? tag = null; + string? digest = null; + string? registry = null; + + foreach (var child in imageMapping.Children) + { + var childKey = (child.Key as YamlScalarNode)?.Value; + var childValue = (child.Value as YamlScalarNode)?.Value; + + switch (childKey?.ToUpperInvariant()) + { + case "REPOSITORY": + repository = childValue; + break; + case "TAG": + tag = childValue; + break; + case "DIGEST": + digest = childValue; + break; + case "REGISTRY": + registry = childValue; + break; + default: + break; + } + } + + if (string.IsNullOrWhiteSpace(repository)) + { + return; + } + + var imageRef = !string.IsNullOrWhiteSpace(registry) + ? $"{registry}/{repository}" + : repository; + + if (!string.IsNullOrWhiteSpace(tag)) + { + imageRef = $"{imageRef}:{tag}"; + } + + if (!string.IsNullOrWhiteSpace(digest)) + { + imageRef = $"{imageRef}@{digest}"; + } + + DockerReferenceUtility.TryRegisterImageReference(imageRef, recorder); + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/kubernetes/KubernetesComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/kubernetes/KubernetesComponentDetector.cs new file mode 100644 index 000000000..7127d70c2 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/kubernetes/KubernetesComponentDetector.cs @@ -0,0 +1,291 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.Kubernetes; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; +using YamlDotNet.Core; +using YamlDotNet.RepresentationModel; + +public class KubernetesComponentDetector : FileComponentDetector, IDefaultOffComponentDetector +{ + private static readonly HashSet KubernetesKinds = new(StringComparer.OrdinalIgnoreCase) + { + "Pod", + "PodTemplate", + "Deployment", + "StatefulSet", + "DaemonSet", + "ReplicaSet", + "Job", + "CronJob", + "ReplicationController", + }; + + public KubernetesComponentDetector( + IComponentStreamEnumerableFactory componentStreamEnumerableFactory, + IObservableDirectoryWalkerFactory walkerFactory, + ILogger logger) + { + this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; + this.Scanner = walkerFactory; + this.Logger = logger; + } + + public override string Id => "Kubernetes"; + + public override IList SearchPatterns { get; } = ["*.yaml", "*.yml"]; + + public override IEnumerable SupportedComponentTypes => [ComponentType.DockerReference]; + + public override int Version => 1; + + public override IEnumerable Categories => [nameof(DetectorClass.Kubernetes)]; + + protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs, CancellationToken cancellationToken = default) + { + var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; + var file = processRequest.ComponentStream; + + try + { + string contents; + using (var reader = new StreamReader(file.Stream)) + { + contents = await reader.ReadToEndAsync(cancellationToken); + } + + // Fast text-based rejection before expensive YAML parsing. + if (!LooksLikeKubernetesManifest(contents)) + { + return; + } + + var yaml = new YamlStream(); + yaml.Load(new StringReader(contents)); + + foreach (var document in yaml.Documents) + { + if (document.RootNode is not YamlMappingNode rootMapping) + { + continue; + } + + if (!IsKubernetesManifest(rootMapping)) + { + continue; + } + + this.Logger.LogInformation("Discovered Kubernetes manifest: {Location}", file.Location); + this.ExtractImageReferences(rootMapping, singleFileComponentRecorder, file.Location); + } + } + catch (YamlException e) + { + this.Logger.LogWarning(e, "Failed to parse YAML file: {Location}", file.Location); + } + catch (Exception e) + { + this.Logger.LogError(e, "Unexpected error processing file: {Location}", file.Location); + } + } + + private static YamlMappingNode? GetMappingChild(YamlMappingNode parent, string key) + { + foreach (var entry in parent.Children) + { + if (entry.Key is YamlScalarNode scalarKey && string.Equals(scalarKey.Value, key, StringComparison.OrdinalIgnoreCase)) + { + return entry.Value as YamlMappingNode; + } + } + + return null; + } + + private static YamlSequenceNode? GetSequenceChild(YamlMappingNode parent, string key) + { + foreach (var entry in parent.Children) + { + if (entry.Key is YamlScalarNode scalarKey && string.Equals(scalarKey.Value, key, StringComparison.OrdinalIgnoreCase)) + { + return entry.Value as YamlSequenceNode; + } + } + + return null; + } + + /// + /// Fast text-based pre-filter. Checks for "apiVersion" and a known "kind: <K8sKind>" + /// pattern using line-based scanning to reject non-Kubernetes YAML without YAML parsing. + /// Tolerates varied whitespace (e.g. "kind:Deployment", "kind: Deployment") and + /// optional quotes around the value. + /// + private static bool LooksLikeKubernetesManifest(string contents) + { + var span = contents.AsSpan(); + + // Must contain apiVersion to be any kind of K8s manifest. + if (span.IndexOf("apiVersion".AsSpan(), StringComparison.Ordinal) < 0) + { + return false; + } + + // Scan line-by-line for a "kind: " entry, tolerating varied + // whitespace and quotes to avoid false negatives on valid manifests. + foreach (var line in span.EnumerateLines()) + { + var trimmed = line.TrimStart(); + if (!trimmed.StartsWith("kind", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var afterKind = trimmed.Slice(4).TrimStart(); + if (afterKind.IsEmpty || afterKind[0] != ':') + { + continue; + } + + var value = afterKind.Slice(1).Trim(); + + // Strip inline YAML comments (K8s kind values never contain '#'). + var commentIdx = value.IndexOf('#'); + if (commentIdx >= 0) + { + value = value.Slice(0, commentIdx).TrimEnd(); + } + + // Strip optional surrounding quotes (e.g. kind: "Deployment"). + if (value.Length >= 2 && + ((value[0] == '"' && value[^1] == '"') || + (value[0] == '\'' && value[^1] == '\''))) + { + value = value.Slice(1, value.Length - 2).Trim(); + } + + if (!value.IsEmpty && KubernetesKinds.Contains(value.ToString())) + { + return true; + } + } + + return false; + } + + private static bool IsKubernetesManifest(YamlMappingNode rootMapping) + { + string? apiVersion = null; + string? kind = null; + + foreach (var entry in rootMapping.Children) + { + var entryKey = (entry.Key as YamlScalarNode)?.Value; + if (string.Equals(entryKey, "apiVersion", StringComparison.OrdinalIgnoreCase)) + { + apiVersion = (entry.Value as YamlScalarNode)?.Value; + } + else if (string.Equals(entryKey, "kind", StringComparison.OrdinalIgnoreCase)) + { + kind = (entry.Value as YamlScalarNode)?.Value; + } + + // Both fields found — stop iterating remaining keys. + if (apiVersion != null && kind != null) + { + break; + } + } + + return !string.IsNullOrEmpty(apiVersion) && !string.IsNullOrEmpty(kind) && KubernetesKinds.Contains(kind); + } + + private void ExtractImageReferences(YamlMappingNode rootMapping, ISingleFileComponentRecorder recorder, string fileLocation) + { + // For Pod, the spec is at the top level + // For Deployment/StatefulSet/etc, the pod spec is at spec.template.spec + var spec = GetMappingChild(rootMapping, "spec"); + if (spec == null) + { + return; + } + + // Direct pod spec (kind: Pod) + this.ExtractContainerImages(spec, recorder, fileLocation); + + // Templated pod spec (kind: Deployment, StatefulSet, etc.) + var template = GetMappingChild(spec, "template"); + if (template != null) + { + var templateSpec = GetMappingChild(template, "spec"); + if (templateSpec != null) + { + this.ExtractContainerImages(templateSpec, recorder, fileLocation); + } + } + + // CronJob has spec.jobTemplate.spec.template.spec + var jobTemplate = GetMappingChild(spec, "jobTemplate"); + if (jobTemplate != null) + { + var jobSpec = GetMappingChild(jobTemplate, "spec"); + if (jobSpec != null) + { + var jobPodTemplate = GetMappingChild(jobSpec, "template"); + if (jobPodTemplate != null) + { + var jobPodSpec = GetMappingChild(jobPodTemplate, "spec"); + if (jobPodSpec != null) + { + this.ExtractContainerImages(jobPodSpec, recorder, fileLocation); + } + } + } + } + } + + private void ExtractContainerImages(YamlMappingNode podSpec, ISingleFileComponentRecorder recorder, string fileLocation) + { + this.ExtractImagesFromContainerList(podSpec, "containers", recorder, fileLocation); + this.ExtractImagesFromContainerList(podSpec, "initContainers", recorder, fileLocation); + this.ExtractImagesFromContainerList(podSpec, "ephemeralContainers", recorder, fileLocation); + } + + private void ExtractImagesFromContainerList(YamlMappingNode podSpec, string containerKey, ISingleFileComponentRecorder recorder, string fileLocation) + { + var containers = GetSequenceChild(podSpec, containerKey); + if (containers == null) + { + return; + } + + foreach (var container in containers) + { + if (container is not YamlMappingNode containerMapping) + { + continue; + } + + foreach (var entry in containerMapping.Children) + { + var key = (entry.Key as YamlScalarNode)?.Value; + if (string.Equals(key, "image", StringComparison.OrdinalIgnoreCase)) + { + var imageRef = (entry.Value as YamlScalarNode)?.Value; + if (!string.IsNullOrWhiteSpace(imageRef)) + { + DockerReferenceUtility.TryRegisterImageReference(imageRef, recorder); + } + } + } + } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 9a9a2b2c3..2f11c522e 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -5,11 +5,14 @@ namespace Microsoft.ComponentDetection.Orchestrator.Extensions; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Detectors.CocoaPods; using Microsoft.ComponentDetection.Detectors.Conan; +using Microsoft.ComponentDetection.Detectors.DockerCompose; using Microsoft.ComponentDetection.Detectors.Dockerfile; using Microsoft.ComponentDetection.Detectors.DotNet; using Microsoft.ComponentDetection.Detectors.Go; using Microsoft.ComponentDetection.Detectors.Gradle; +using Microsoft.ComponentDetection.Detectors.Helm; using Microsoft.ComponentDetection.Detectors.Ivy; +using Microsoft.ComponentDetection.Detectors.Kubernetes; using Microsoft.ComponentDetection.Detectors.Linux; using Microsoft.ComponentDetection.Detectors.Linux.Factories; using Microsoft.ComponentDetection.Detectors.Linux.Filters; @@ -87,6 +90,9 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // Dockerfile services.AddSingleton(); + // Docker Compose + services.AddSingleton(); + // DotNet services.AddSingleton(); @@ -97,6 +103,9 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // Gradle services.AddSingleton(); + // Helm + services.AddSingleton(); + // Ivy services.AddSingleton(); @@ -171,6 +180,9 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s // uv services.AddSingleton(); + // Kubernetes + services.AddSingleton(); + return services; } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DockerComposeComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DockerComposeComponentDetectorTests.cs new file mode 100644 index 000000000..64fcbd56a --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DockerComposeComponentDetectorTests.cs @@ -0,0 +1,276 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.Linq; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.DockerCompose; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class DockerComposeComponentDetectorTests : BaseDetectorTest +{ + [TestMethod] + public async Task TestCompose_SingleServiceImageAsync() + { + var composeYaml = @" +version: '3' +services: + web: + image: nginx:1.21 + ports: + - ""80:80"" +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/nginx"); + dockerRef.Tag.Should().Be("1.21"); + } + + [TestMethod] + public async Task TestCompose_MultipleServicesAsync() + { + var composeYaml = @" +version: '3' +services: + web: + image: nginx:1.21 + db: + image: postgres:15 + cache: + image: redis:7-alpine +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(3); + } + + [TestMethod] + public async Task TestCompose_FullRegistryImageAsync() + { + var composeYaml = @" +services: + app: + image: ghcr.io/myorg/myapp:v2.0 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("compose.yaml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Domain.Should().Be("ghcr.io"); + dockerRef.Repository.Should().Be("myorg/myapp"); + dockerRef.Tag.Should().Be("v2.0"); + } + + [TestMethod] + public async Task TestCompose_BuildOnlyServiceIgnoredAsync() + { + var composeYaml = @" +version: '3' +services: + app: + build: ./app + ports: + - ""3000:3000"" +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestCompose_MixedBuildAndImageAsync() + { + var composeYaml = @" +version: '3' +services: + app: + build: ./app + db: + image: postgres:15 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/postgres"); + dockerRef.Tag.Should().Be("15"); + } + + [TestMethod] + public async Task TestCompose_NoServicesKeyAsync() + { + var composeYaml = @" +version: '3' +networks: + frontend: +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestCompose_ImageWithDigestAsync() + { + var composeYaml = @" +services: + app: + image: nginx@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yaml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestCompose_ImageWithTagAndDigestAsync() + { + var composeYaml = @" +services: + app: + image: nginx:1.21@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yaml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Tag.Should().Be("1.21"); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestCompose_OverrideFileAsync() + { + var composeYaml = @" +services: + web: + image: myregistry.io/web:latest +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.override.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + } + + [TestMethod] + public async Task TestCompose_UnresolvedVariableSkippedAsync() + { + var composeYaml = @" +services: + app: + image: ${REGISTRY}/app:${TAG} + db: + image: postgres:15 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("docker-compose.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + + // Only the literal image reference (postgres:15) should be registered; + // the variable-interpolated image (${REGISTRY}/app:${TAG}) should be silently skipped. + components.Should().ContainSingle(); + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/postgres"); + } + + [TestMethod] + public async Task TestCompose_ComposeOverrideFileAsync() + { + var composeYaml = @" +services: + web: + image: nginx:1.21 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("compose.override.yml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + } + + [TestMethod] + public async Task TestCompose_ComposeOverrideYamlAsync() + { + var composeYaml = @" +services: + db: + image: postgres:15 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("compose.prod.yaml", composeYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/DockerfileComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/DockerfileComponentDetectorTests.cs new file mode 100644 index 000000000..0ade5d6f5 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/DockerfileComponentDetectorTests.cs @@ -0,0 +1,224 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.Linq; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Dockerfile; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class DockerfileComponentDetectorTests : BaseDetectorTest +{ + private readonly Mock mockCommandLineInvocationService = new(); + private readonly Mock mockEnvironmentVariableService = new(); + + [TestInitialize] + public void TestInitialize() + { + this.DetectorTestUtility + .AddServiceMock(this.mockCommandLineInvocationService) + .AddServiceMock(this.mockEnvironmentVariableService); + } + + [TestMethod] + public async Task TestDockerfile_SimpleFromInstructionAsync() + { + var dockerfile = "FROM nginx:1.21\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/nginx"); + dockerRef.Tag.Should().Be("1.21"); + } + + [TestMethod] + public async Task TestDockerfile_MultiStageFromAsync() + { + var dockerfile = @"FROM node:18-alpine AS build +RUN npm ci +FROM nginx:1.21 +COPY --from=build /app/dist /usr/share/nginx/html +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestDockerfile_FullRegistryAsync() + { + var dockerfile = "FROM gcr.io/my-project/my-app:latest\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Domain.Should().Be("gcr.io"); + dockerRef.Repository.Should().Be("my-project/my-app"); + dockerRef.Tag.Should().Be("latest"); + } + + [TestMethod] + public async Task TestDockerfile_WithDigestAsync() + { + var dockerfile = "FROM nginx@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestDockerfile_WithTagAndDigestAsync() + { + var dockerfile = "FROM nginx:1.21@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Tag.Should().Be("1.21"); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestDockerfile_CopyFromExternalImageAsync() + { + var dockerfile = @"FROM nginx:1.21 +COPY --from=busybox:1.35 /bin/busybox /bin/busybox +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestDockerfile_CopyFromNamedStageNotDuplicatedAsync() + { + var dockerfile = @"FROM node:18-alpine AS builder +RUN echo hello +FROM nginx:1.21 +COPY --from=builder /app /app +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + // COPY --from=builder should resolve to the existing stage, not register a new component + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestDockerfile_UnresolvedVariableSkippedAsync() + { + var dockerfile = @"ARG BASE_IMAGE=nginx +FROM ${BASE_IMAGE}:latest +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + // Unresolved variables are skipped + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestDockerfile_ScratchBaseImageAsync() + { + var dockerfile = "FROM scratch\nCOPY myapp /\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + // scratch is a valid Docker base but it resolves to docker.io/library/scratch + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + } + + [TestMethod] + public async Task TestDockerfile_EmptyFileAsync() + { + var dockerfile = "# just a comment\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestDockerfile_NamedDockerfilePatternAsync() + { + var dockerfile = "FROM alpine:3.18\n"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("app.dockerfile", dockerfile) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/alpine"); + dockerRef.Tag.Should().Be("3.18"); + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/HelmComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/HelmComponentDetectorTests.cs new file mode 100644 index 000000000..9918eef12 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/HelmComponentDetectorTests.cs @@ -0,0 +1,432 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Helm; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class HelmComponentDetectorTests : BaseDetectorTest +{ + private const string MinimalChartYaml = @" +apiVersion: v2 +name: my-chart +version: 0.1.0 +"; + + [TestMethod] + public async Task TestHelm_DirectImageStringAsync() + { + var valuesYaml = @" +replicaCount: 1 +image: nginx:1.21 +service: + type: ClusterIP +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/nginx"); + dockerRef.Tag.Should().Be("1.21"); + } + + [TestMethod] + public async Task TestHelm_StructuredImageReferenceAsync() + { + var valuesYaml = @" +replicaCount: 1 +image: + repository: myregistry.io/myapp + tag: ""2.0.0"" +service: + type: ClusterIP +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("myapp"); + dockerRef.Domain.Should().Be("myregistry.io"); + dockerRef.Tag.Should().Be("2.0.0"); + } + + [TestMethod] + public async Task TestHelm_StructuredImageWithRegistryAsync() + { + var valuesYaml = @" +image: + registry: ghcr.io + repository: org/myimage + tag: ""v1.0"" +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Domain.Should().Be("ghcr.io"); + dockerRef.Repository.Should().Be("org/myimage"); + dockerRef.Tag.Should().Be("v1.0"); + } + + [TestMethod] + public async Task TestHelm_NestedImageReferencesAsync() + { + var valuesYaml = @" +app: + frontend: + image: nginx:1.21 + backend: + image: + repository: node + tag: ""18-alpine"" +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestHelm_EmptyValuesYamlAsync() + { + var valuesYaml = @" +replicaCount: 1 +service: + type: ClusterIP +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestHelm_ChartYamlIgnoredAsync() + { + var chartYaml = @" +apiVersion: v2 +name: my-chart +version: 0.1.0 +dependencies: + - name: postgresql + version: ""11.0.0"" + repository: https://charts.bitnami.com/bitnami +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", chartYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestHelm_ValuesFileObservedBeforeChartYamlAsync() + { + // Verify that values files are processed even when they are enumerated + // before the co-located Chart.yaml (non-deterministic file order). + var valuesYaml = @" +image: nginx:1.21 +"; + + // values.yaml is registered first (before Chart.yaml) to simulate the + // problematic enumeration order the two-pass OnPrepareDetectionAsync fixes. + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("values.yaml", valuesYaml) + .WithFile("Chart.yaml", MinimalChartYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + + var dockerRef = componentRecorder.GetDetectedComponents().First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/nginx"); + dockerRef.Tag.Should().Be("1.21"); + } + + [TestMethod] + public async Task TestHelm_ValuesWithoutChartYamlSkippedAsync() + { + var valuesYaml = @" +image: nginx:1.21 +"; + + // No Chart.yaml provided — the values file should be skipped entirely. + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestHelm_ValuesInDifferentDirectoryFromChartSkippedAsync() + { + var valuesYaml = @" +image: nginx:1.21 +"; + + // Chart.yaml exists but in a different directory than values.yaml. + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml, fileLocation: Path.Combine(Path.GetTempPath(), "chartA", "Chart.yaml")) + .WithFile("values.yaml", valuesYaml, fileLocation: Path.Combine(Path.GetTempPath(), "chartB", "values.yaml")) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestHelm_ImageWithDigestAsync() + { + var valuesYaml = @" +image: + repository: nginx + digest: ""sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"" +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestHelm_StructuredImageWithTagAndDigestAsync() + { + var valuesYaml = @" +image: + repository: nginx + tag: ""1.21"" + digest: ""sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"" +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestHelm_DirectImageStringWithDigestAsync() + { + var valuesYaml = @" +image: nginx@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestHelm_DirectImageStringWithTagAndDigestAsync() + { + var valuesYaml = @" +image: nginx:1.21@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Tag.Should().Be("1.21"); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestHelm_ImagesInSequenceAsync() + { + var valuesYaml = @" +sidecars: + - name: sidecar1 + image: busybox:1.35 + - name: sidecar2 + image: alpine:3.18 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestHelm_UnresolvedVariableSkippedAsync() + { + var valuesYaml = @" +image: ${REGISTRY}/app:${TAG} +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestHelm_ValuesYmlExtensionAsync() + { + var valuesYaml = @" +image: nginx:1.21 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.yml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + } + + [TestMethod] + public async Task TestHelm_ValuesOverrideFileAsync() + { + var valuesYaml = @" +image: redis:7-alpine +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("values.production.yaml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + } + + [TestMethod] + public async Task TestHelm_CustomValuesFilenameAsync() + { + var valuesYaml = @" +image: postgres:15 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yaml", MinimalChartYaml) + .WithFile("myapp-values-dev.yml", valuesYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().ContainSingle(); + } + + [TestMethod] + public async Task TestHelm_LowercaseChartYamlAsync() + { + var chartYaml = @" +apiVersion: v2 +name: my-chart +version: 0.1.0 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("chart.yaml", chartYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestHelm_ChartYmlExtensionAsync() + { + var chartYaml = @" +apiVersion: v2 +name: my-chart +version: 0.1.0 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("Chart.yml", chartYaml) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } +} diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/KubernetesComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/KubernetesComponentDetectorTests.cs new file mode 100644 index 000000000..105aedbaf --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/KubernetesComponentDetectorTests.cs @@ -0,0 +1,390 @@ +#nullable enable +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System.Linq; +using System.Threading.Tasks; +using AwesomeAssertions; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Kubernetes; +using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +[TestCategory("Governance/All")] +[TestCategory("Governance/ComponentDetection")] +public class KubernetesComponentDetectorTests : BaseDetectorTest +{ + [TestMethod] + public async Task TestK8s_PodWithContainerImageAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: web + image: nginx:1.21 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/nginx"); + dockerRef.Tag.Should().Be("1.21"); + } + + [TestMethod] + public async Task TestK8s_DeploymentWithMultipleContainersAsync() + { + var manifest = @" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-app +spec: + replicas: 3 + selector: + matchLabels: + app: my-app + template: + metadata: + labels: + app: my-app + spec: + containers: + - name: app + image: myregistry.io/myapp:v2.0 + - name: sidecar + image: busybox:1.35 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("deployment.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestK8s_InitContainersAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + initContainers: + - name: init + image: busybox:1.35 + containers: + - name: app + image: nginx:1.21 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().HaveCount(2); + } + + [TestMethod] + public async Task TestK8s_CronJobAsync() + { + var manifest = @" +apiVersion: batch/v1 +kind: CronJob +metadata: + name: my-cronjob +spec: + schedule: ""*/5 * * * *"" + jobTemplate: + spec: + template: + spec: + containers: + - name: job + image: python:3.11-slim + restartPolicy: OnFailure +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("cronjob.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/python"); + dockerRef.Tag.Should().Be("3.11-slim"); + } + + [TestMethod] + public async Task TestK8s_NonKubernetesYamlIgnoredAsync() + { + var manifest = @" +name: my-config +settings: + debug: true + image: nginx:latest +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("config.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestK8s_ServiceIgnoredAsync() + { + var manifest = @" +apiVersion: v1 +kind: Service +metadata: + name: my-service +spec: + selector: + app: my-app + ports: + - port: 80 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("service.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestK8s_StatefulSetAsync() + { + var manifest = @" +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: my-db +spec: + serviceName: my-db + replicas: 3 + selector: + matchLabels: + app: my-db + template: + metadata: + labels: + app: my-db + spec: + containers: + - name: db + image: postgres:15 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("statefulset.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/postgres"); + dockerRef.Tag.Should().Be("15"); + } + + [TestMethod] + public async Task TestK8s_DaemonSetAsync() + { + var manifest = @" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd +spec: + selector: + matchLabels: + name: fluentd + template: + metadata: + labels: + name: fluentd + spec: + containers: + - name: fluentd + image: fluentd:v1.16 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("daemonset.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/fluentd"); + dockerRef.Tag.Should().Be("v1.16"); + } + + [TestMethod] + public async Task TestK8s_ImageWithFullRegistryAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: app + image: gcr.io/my-project/my-app:latest +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Domain.Should().Be("gcr.io"); + dockerRef.Repository.Should().Be("my-project/my-app"); + dockerRef.Tag.Should().Be("latest"); + } + + [TestMethod] + public async Task TestK8s_EmptyContainersAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: [] +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + componentRecorder.GetDetectedComponents().Should().BeEmpty(); + } + + [TestMethod] + public async Task TestK8s_ImageWithDigestOnlyAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: app + image: nginx@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestK8s_ImageWithTagAndDigestAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: app + image: nginx:1.21@sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + components.Should().ContainSingle(); + + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Tag.Should().Be("1.21"); + dockerRef.Digest.Should().Be("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abc1"); + } + + [TestMethod] + public async Task TestK8s_UnresolvedVariablesSkippedAsync() + { + var manifest = @" +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: app + image: ${REGISTRY}/app:${TAG} + - name: sidecar + image: nginx:1.21 +"; + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile("pod.yaml", manifest) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var components = componentRecorder.GetDetectedComponents(); + + // Only the literal image reference (nginx:1.21) should be registered; + // the variable-interpolated image (${REGISTRY}/app:${TAG}) should be silently skipped. + components.Should().ContainSingle(); + var dockerRef = components.First().Component as DockerReferenceComponent; + dockerRef.Should().NotBeNull(); + dockerRef.Repository.Should().Be("library/nginx"); + dockerRef.Tag.Should().Be("1.21"); + } +} diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 b/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 index 9e9dcd0d9..4581fb224 100755 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/VerificationTest.ps1 @@ -41,7 +41,7 @@ function main() Set-Location ((Get-Item $repoPath).FullName + "\src\Microsoft.ComponentDetection") dotnet run scan --SourceDirectory $verificationTestRepo --Output $output ` --DockerImagesToScan $dockerImagesToScan ` - --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff,CondaLock=EnableIfDefaultOff,ConanLock=EnableIfDefaultOff ` + --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff,CondaLock=EnableIfDefaultOff,ConanLock=EnableIfDefaultOff,DockerCompose=EnableIfDefaultOff,Helm=EnableIfDefaultOff,Kubernetes=EnableIfDefaultOff ` --MaxDetectionThreads 5 --DebugTelemetry ` --DirectoryExclusionList "**/pip/parallel/**;**/pip/roots/**;**/pip/pre-generated/**" @@ -50,7 +50,7 @@ function main() Set-Location ($CDRelease + "\src\Microsoft.ComponentDetection") dotnet run scan --SourceDirectory $verificationTestRepo --Output $releaseOutput ` --DockerImagesToScan $dockerImagesToScan ` - --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff,CondaLock=EnableIfDefaultOff,ConanLock=EnableIfDefaultOff ` + --DetectorArgs DockerReference=EnableIfDefaultOff,SPDX22SBOM=EnableIfDefaultOff,CondaLock=EnableIfDefaultOff,ConanLock=EnableIfDefaultOff,DockerCompose=EnableIfDefaultOff,Helm=EnableIfDefaultOff,Kubernetes=EnableIfDefaultOff ` --MaxDetectionThreads 5 --DebugTelemetry ` --DirectoryExclusionList "**/pip/parallel/**;**/pip/roots/**;**/pip/pre-generated/**" diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/dockercompose/docker-compose.override.yml b/test/Microsoft.ComponentDetection.VerificationTests/resources/dockercompose/docker-compose.override.yml new file mode 100644 index 000000000..77260786a --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/dockercompose/docker-compose.override.yml @@ -0,0 +1,5 @@ +version: '3.8' +services: + debug: + image: busybox:1.35 + command: ['sh', '-c', 'sleep 3600'] diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/dockercompose/docker-compose.yml b/test/Microsoft.ComponentDetection.VerificationTests/resources/dockercompose/docker-compose.yml new file mode 100644 index 000000000..068f30b72 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/dockercompose/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.8' +services: + web: + image: nginx:1.21 + ports: + - "8080:80" + depends_on: + - api + - db + + api: + image: ghcr.io/myorg/myapp:v2.0 + environment: + - DATABASE_URL=postgres://db:5432/app + depends_on: + - db + + db: + image: postgres:15 + volumes: + - db-data:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=secret + + cache: + image: redis:7-alpine + ports: + - "6379:6379" + + digest-only: + image: nginx@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31 + + tag-and-digest: + image: redis:7-alpine@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31 + +volumes: + db-data: diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/helm/Chart.yaml b/test/Microsoft.ComponentDetection.VerificationTests/resources/helm/Chart.yaml new file mode 100644 index 000000000..3906c37a8 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: my-app +description: A sample Helm chart +version: 0.1.0 +appVersion: "1.0.0" diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/helm/values.yaml b/test/Microsoft.ComponentDetection.VerificationTests/resources/helm/values.yaml new file mode 100644 index 000000000..865e5e160 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/helm/values.yaml @@ -0,0 +1,36 @@ +replicaCount: 3 + +image: + repository: myregistry.io/myapp + tag: "2.0.0" + +sidecar: + image: nginx:1.21 + +backend: + image: + registry: ghcr.io + repository: org/backend + tag: "v1.0" + +monitoring: + image: + repository: prom/prometheus + tag: "v2.45.0" + +digestOnly: + image: nginx@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31 + +tagAndDigest: + image: redis:7-alpine@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31 + +structuredDigest: + image: + repository: busybox + digest: "sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31" + +structuredTagAndDigest: + image: + repository: alpine + tag: "3.18" + digest: "sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31" diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/cronjob.yaml b/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/cronjob.yaml new file mode 100644 index 000000000..f535e6631 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/cronjob.yaml @@ -0,0 +1,15 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cleanup +spec: + schedule: "0 2 * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: cleanup + image: python:3.11-slim + command: ['python', '-c', 'print("cleanup done")'] + restartPolicy: OnFailure diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/deployment.yaml b/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/deployment.yaml new file mode 100644 index 000000000..b7f3bc534 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web-app + labels: + app: web +spec: + replicas: 3 + selector: + matchLabels: + app: web + template: + metadata: + labels: + app: web + spec: + initContainers: + - name: init-db + image: busybox:1.35 + command: ['sh', '-c', 'until nslookup db; do sleep 2; done'] + containers: + - name: web + image: nginx:1.21 + ports: + - containerPort: 80 + - name: sidecar + image: gcr.io/my-project/log-collector:v1.0 + volumeMounts: + - name: logs + mountPath: /var/log + - name: digest-only + image: nginx@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31 + - name: tag-and-digest + image: redis:7-alpine@sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31 + volumes: + - name: logs + emptyDir: {} diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/statefulset.yaml b/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/statefulset.yaml new file mode 100644 index 000000000..78b3671f3 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/kubernetes/statefulset.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: database +spec: + serviceName: db + replicas: 1 + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + spec: + containers: + - name: postgres + image: postgres:15 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: db-secret + key: password