-
Notifications
You must be signed in to change notification settings - Fork 126
Add component detectors for Docker Compose, Helm, and Kubernetes #1759
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
1a3f651
077d1d0
fd74cb5
d5c91ed
a111856
70776f8
a099d56
0bbc3db
5c4e487
b5d3fbf
5158b75
a26f434
0873316
ea3b0a4
5984b1c
9d56f38
8bce22b
ad592b2
9ae1dd4
70762f1
d220257
dff7d5f
569babe
f651b13
7f08f43
b77c789
70ada53
e5ffbb5
c0f586c
5d270f5
df6c196
ce8e98b
5b148b7
84b4383
6478e2e
a3fa9b8
146759c
7fc72c1
9b8e1b7
bac8dc8
e0e81f9
ccfdd1b
1cc4383
6b7d6fe
035bdc9
2f0d13c
7307efc
5c56424
2e5d0f5
8ab1513
77a8af2
ade40f8
b52999a
d20304a
0ef45f1
d438845
f66238d
b0a4661
f102880
4bc3c0a
78aacbf
40942a9
930d0b2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| #nullable disable | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
| namespace Microsoft.ComponentDetection.Detectors.DockerCompose; | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
|
jpinz marked this conversation as resolved.
|
||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.ComponentDetection.Common; | ||
| using Microsoft.ComponentDetection.Contracts; | ||
| using Microsoft.ComponentDetection.Contracts.Internal; | ||
| using Microsoft.ComponentDetection.Contracts.TypedComponent; | ||
|
jpinz marked this conversation as resolved.
|
||
| using Microsoft.Extensions.Logging; | ||
| using YamlDotNet.RepresentationModel; | ||
|
|
||
| public class DockerComposeComponentDetector : FileComponentDetector, IDefaultOffComponentDetector | ||
| { | ||
| public DockerComposeComponentDetector( | ||
| IComponentStreamEnumerableFactory componentStreamEnumerableFactory, | ||
| IObservableDirectoryWalkerFactory walkerFactory, | ||
| ILogger<DockerComposeComponentDetector> logger) | ||
| { | ||
| this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; | ||
| this.Scanner = walkerFactory; | ||
| this.Logger = logger; | ||
| } | ||
|
|
||
| public override string Id => "DockerCompose"; | ||
|
|
||
| public override IList<string> SearchPatterns { get; } = | ||
| [ | ||
| "docker-compose.yml", "docker-compose.yaml", | ||
| "docker-compose.*.yml", "docker-compose.*.yaml", | ||
| "compose.yml", "compose.yaml", | ||
|
jpinz marked this conversation as resolved.
|
||
| ]; | ||
|
jpinz marked this conversation as resolved.
|
||
|
|
||
| public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.DockerReference]; | ||
|
|
||
| public override int Version => 1; | ||
|
|
||
| public override IEnumerable<string> Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.DockerCompose)]; | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
|
|
||
| protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default) | ||
| { | ||
| var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; | ||
| var file = processRequest.ComponentStream; | ||
|
|
||
| try | ||
| { | ||
| this.Logger.LogInformation("Discovered Docker Compose file: {Location}", file.Location); | ||
|
|
||
| string contents; | ||
| using (var reader = new StreamReader(file.Stream)) | ||
| { | ||
| contents = await reader.ReadToEndAsync(cancellationToken); | ||
| } | ||
|
|
||
| var yaml = new YamlStream(); | ||
| yaml.Load(new StringReader(contents)); | ||
|
|
||
| if (yaml.Documents.Count == 0) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| foreach (var document in yaml.Documents) | ||
| { | ||
| if (document.RootNode is YamlMappingNode rootMapping) | ||
| { | ||
| this.ExtractImageReferences(rootMapping, singleFileComponentRecorder, file.Location); | ||
| } | ||
| } | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| this.Logger.LogError(e, "Failed to parse Docker Compose file: {Location}", file.Location); | ||
| } | ||
| } | ||
|
|
||
| private void ExtractImageReferences(YamlMappingNode rootMapping, ISingleFileComponentRecorder recorder, string fileLocation) | ||
| { | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
| var services = GetMappingChild(rootMapping, "services"); | ||
| if (services == null) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| foreach (var serviceEntry in services.Children) | ||
| { | ||
| if (serviceEntry.Value is not YamlMappingNode serviceMapping) | ||
| { | ||
| continue; | ||
| } | ||
|
|
||
| // Extract direct image: references | ||
| foreach (var entry in serviceMapping.Children) | ||
| { | ||
| var key = (entry.Key as YamlScalarNode)?.Value; | ||
| if (string.Equals(key, "image", StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| var imageRef = (entry.Value as YamlScalarNode)?.Value; | ||
| if (!string.IsNullOrWhiteSpace(imageRef)) | ||
| { | ||
| this.TryRegisterImageReference(imageRef, recorder, fileLocation); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void TryRegisterImageReference(string imageReference, ISingleFileComponentRecorder recorder, string fileLocation) | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
| { | ||
| try | ||
| { | ||
| var dockerRef = DockerReferenceUtility.ParseFamiliarName(imageReference); | ||
| if (dockerRef != null) | ||
| { | ||
| recorder.RegisterUsage(new DetectedComponent(dockerRef.ToTypedDockerReferenceComponent())); | ||
| } | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| this.Logger.LogWarning(e, "Failed to parse image reference '{ImageReference}' in {Location}", imageReference, fileLocation); | ||
| } | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
| } | ||
|
|
||
| private static YamlMappingNode GetMappingChild(YamlMappingNode parent, string key) | ||
|
Check failure on line 127 in src/Microsoft.ComponentDetection.Detectors/dockercompose/DockerComposeComponentDetector.cs
|
||
| { | ||
| foreach (var entry in parent.Children) | ||
| { | ||
| if (entry.Key is YamlScalarNode scalarKey && string.Equals(scalarKey.Value, key, StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| return entry.Value as YamlMappingNode; | ||
| } | ||
| } | ||
|
|
||
| return null; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| #nullable disable | ||
| namespace Microsoft.ComponentDetection.Detectors.Helm; | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
|
jpinz marked this conversation as resolved.
|
||
| using System.IO; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.ComponentDetection.Common; | ||
| using Microsoft.ComponentDetection.Contracts; | ||
| using Microsoft.ComponentDetection.Contracts.Internal; | ||
| using Microsoft.ComponentDetection.Contracts.TypedComponent; | ||
|
jpinz marked this conversation as resolved.
jpinz marked this conversation as resolved.
|
||
| using Microsoft.Extensions.Logging; | ||
| using YamlDotNet.RepresentationModel; | ||
|
|
||
| public class HelmComponentDetector : FileComponentDetector, IDefaultOffComponentDetector | ||
| { | ||
| public HelmComponentDetector( | ||
| IComponentStreamEnumerableFactory componentStreamEnumerableFactory, | ||
| IObservableDirectoryWalkerFactory walkerFactory, | ||
| ILogger<HelmComponentDetector> logger) | ||
| { | ||
| this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; | ||
| this.Scanner = walkerFactory; | ||
| this.Logger = logger; | ||
| } | ||
|
|
||
| public override string Id => "Helm"; | ||
|
|
||
| public override IList<string> SearchPatterns { get; } = | ||
| [ | ||
| "Chart.yaml", "Chart.yml", | ||
| "chart.yaml", "chart.yml", | ||
|
jpinz marked this conversation as resolved.
Outdated
jpinz marked this conversation as resolved.
Outdated
|
||
| "*values*.yaml", "*values*.yml", | ||
|
jpinz marked this conversation as resolved.
jpinz marked this conversation as resolved.
|
||
| ]; | ||
|
Comment on lines
+32
to
+36
|
||
|
|
||
| public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.DockerReference]; | ||
|
|
||
| public override int Version => 1; | ||
|
|
||
| public override IEnumerable<string> Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.Helm)]; | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
|
|
||
| protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default) | ||
| { | ||
| var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; | ||
| var file = processRequest.ComponentStream; | ||
|
|
||
| try | ||
| { | ||
| this.Logger.LogInformation("Discovered Helm file: {Location}", file.Location); | ||
|
|
||
| string contents; | ||
| using (var reader = new StreamReader(file.Stream)) | ||
| { | ||
| contents = await reader.ReadToEndAsync(cancellationToken); | ||
| } | ||
|
|
||
|
jpinz marked this conversation as resolved.
|
||
| var yaml = new YamlStream(); | ||
| yaml.Load(new StringReader(contents)); | ||
|
|
||
| if (yaml.Documents.Count == 0) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| var fileName = Path.GetFileName(file.Location); | ||
|
|
||
| if (fileName.Contains("values", StringComparison.OrdinalIgnoreCase) && | ||
| (fileName.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase) || fileName.EndsWith(".yml", StringComparison.OrdinalIgnoreCase))) | ||
| { | ||
| this.ExtractImageReferencesFromValues(yaml, singleFileComponentRecorder, file.Location); | ||
| } | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
| } | ||
| catch (Exception e) | ||
| { | ||
| this.Logger.LogError(e, "Failed to parse Helm file: {Location}", file.Location); | ||
| } | ||
| } | ||
|
|
||
| private void ExtractImageReferencesFromValues(YamlStream yaml, ISingleFileComponentRecorder recorder, string fileLocation) | ||
| { | ||
| foreach (var document in yaml.Documents) | ||
| { | ||
| if (document.RootNode is YamlMappingNode rootMapping) | ||
| { | ||
| this.WalkYamlForImages(rootMapping, recorder, fileLocation); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Walks the YAML tree looking for image references. Handles two common patterns: | ||
| /// 1. Direct image string: `image: nginx:1.21` | ||
| /// 2. Structured image object: `image: { repository: nginx, tag: "1.21" }`. | ||
| /// </summary> | ||
| private void WalkYamlForImages(YamlMappingNode mapping, ISingleFileComponentRecorder recorder, string fileLocation) | ||
| { | ||
|
||
| foreach (var entry in mapping.Children) | ||
| { | ||
| var key = (entry.Key as YamlScalarNode)?.Value; | ||
|
|
||
| if (string.Equals(key, "image", StringComparison.OrdinalIgnoreCase)) | ||
| { | ||
| switch (entry.Value) | ||
|
Check failure on line 104 in src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs
|
||
| { | ||
| // image: nginx:1.21 | ||
| case YamlScalarNode scalarValue when !string.IsNullOrWhiteSpace(scalarValue.Value): | ||
| this.TryRegisterImageReference(scalarValue.Value, recorder, fileLocation); | ||
| break; | ||
|
|
||
| // image: | ||
| // repository: nginx | ||
| // tag: "1.21" | ||
| case YamlMappingNode imageMapping: | ||
| this.TryRegisterStructuredImageReference(imageMapping, recorder, fileLocation); | ||
| break; | ||
| } | ||
| } | ||
| else if (entry.Value is YamlMappingNode childMapping) | ||
| { | ||
| this.WalkYamlForImages(childMapping, recorder, fileLocation); | ||
| } | ||
| else if (entry.Value is YamlSequenceNode sequenceNode) | ||
| { | ||
| foreach (var item in sequenceNode) | ||
| { | ||
| if (item is YamlMappingNode sequenceMapping) | ||
| { | ||
| this.WalkYamlForImages(sequenceMapping, recorder, fileLocation); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void TryRegisterStructuredImageReference(YamlMappingNode imageMapping, ISingleFileComponentRecorder recorder, string fileLocation) | ||
| { | ||
|
jpinz marked this conversation as resolved.
|
||
| string repository = null; | ||
| string tag = null; | ||
| string digest = null; | ||
| string registry = null; | ||
|
|
||
| foreach (var child in imageMapping.Children) | ||
| { | ||
| var childKey = (child.Key as YamlScalarNode)?.Value; | ||
| var childValue = (child.Value as YamlScalarNode)?.Value; | ||
|
|
||
| switch (childKey?.ToLowerInvariant()) | ||
|
Check failure on line 148 in src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs
|
||
| { | ||
| case "repository": | ||
| repository = childValue; | ||
| break; | ||
| case "tag": | ||
| tag = childValue; | ||
| break; | ||
| case "digest": | ||
| digest = childValue; | ||
| break; | ||
| case "registry": | ||
| registry = childValue; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| if (string.IsNullOrWhiteSpace(repository)) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| var imageRef = !string.IsNullOrWhiteSpace(registry) | ||
| ? $"{registry}/{repository}" | ||
| : repository; | ||
|
|
||
| if (!string.IsNullOrWhiteSpace(tag)) | ||
| { | ||
| imageRef = $"{imageRef}:{tag}"; | ||
| } | ||
|
|
||
| if (!string.IsNullOrWhiteSpace(digest)) | ||
| { | ||
| imageRef = $"{imageRef}@{digest}"; | ||
| } | ||
|
|
||
| this.TryRegisterImageReference(imageRef, recorder, fileLocation); | ||
| } | ||
|
|
||
| private void TryRegisterImageReference(string imageReference, ISingleFileComponentRecorder recorder, string fileLocation) | ||
| { | ||
| try | ||
| { | ||
| var dockerRef = DockerReferenceUtility.ParseFamiliarName(imageReference); | ||
| if (dockerRef != null) | ||
| { | ||
| recorder.RegisterUsage(new DetectedComponent(dockerRef.ToTypedDockerReferenceComponent())); | ||
| } | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| this.Logger.LogWarning(e, "Failed to parse image reference '{ImageReference}' in {Location}", imageReference, fileLocation); | ||
| } | ||
|
jpinz marked this conversation as resolved.
Outdated
|
||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.