diff --git a/src/Microsoft.ComponentDetection.Detectors/conda/CondaDependencyResolver.cs b/src/Microsoft.ComponentDetection.Detectors/conda/CondaDependencyResolver.cs index c84637640..4dc3c422b 100644 --- a/src/Microsoft.ComponentDetection.Detectors/conda/CondaDependencyResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/conda/CondaDependencyResolver.cs @@ -18,7 +18,13 @@ public static class CondaDependencyResolver /// The full condaLock object. /// The SingleFileComponentRecorder. public static void RecordDependencyGraphFromFile(CondaLock condaLock, ISingleFileComponentRecorder singleFileComponentRecorder) - => GetPackages(condaLock).ForEach(package => RegisterPackageWithDependencies(package, null, condaLock, singleFileComponentRecorder)); + { + // Tracks components whose sub-tree has already been walked so that cyclic + // dependencies (e.g. A -> B -> A) don't cause infinite recursion and so that + // diamond-shaped graphs aren't re-walked exponentially. + var visited = new HashSet(); + GetPackages(condaLock).ForEach(package => RegisterPackageWithDependencies(package, null, condaLock, singleFileComponentRecorder, visited)); + } /// /// Updates all registered packages that don't have any ancestors. @@ -60,7 +66,8 @@ public static void UpdateDirectlyReferencedPackages(ISingleFileComponentRecorder /// The id of the parent package. /// The full condaLock object. /// The SingleFileComponentRecorder. - private static void RegisterPackageWithDependencies(CondaPackage package, string parentId, CondaLock condaLock, ISingleFileComponentRecorder singleFileComponentRecorder) + /// The set of component ids whose dependencies have already been walked. + private static void RegisterPackageWithDependencies(CondaPackage package, string parentId, CondaLock condaLock, ISingleFileComponentRecorder singleFileComponentRecorder, HashSet visited) { if (package == null) { @@ -69,16 +76,24 @@ private static void RegisterPackageWithDependencies(CondaPackage package, string var component = CreateComponent(package); - //// Register the package itself. + // Register the package itself. This also records the edge from the parent, + // so it must happen every time the package is reached, regardless of cycles. RegisterPackage(component, parentId, false, singleFileComponentRecorder); - //// Register all dependencies of the package. - package.Dependencies.Keys.ToList().ForEach(dependency => - RegisterPackageWithDependencies( - condaLock?.Package.FirstOrDefault(condaPackage => condaPackage.Name == dependency && condaPackage.Platform == package.Platform), - component.Id, - condaLock, - singleFileComponentRecorder)); + // Only walk a package's dependencies once. This guards against cyclic + // dependency graphs (which would otherwise recurse forever) and avoids + // re-walking shared sub-trees. + var visitKey = $"{component.Id}:{package.Platform}"; + if (!visited.Add(visitKey)) + { + return; + } + + foreach (var dependency in package.Dependencies?.Keys ?? Enumerable.Empty()) + { + var dependencyPackage = condaLock?.Package.FirstOrDefault(condaPackage => condaPackage.Name == dependency && condaPackage.Platform == package.Platform); + RegisterPackageWithDependencies(dependencyPackage, component.Id, condaLock, singleFileComponentRecorder, visited); + } } /// @@ -122,19 +137,9 @@ private static List GetPackages(CondaLock condaLock) /// The CondaPackage to convert. /// The TypedComponent. private static TypedComponent CreateComponent(CondaPackage package) - => IsPythonPackage(package) - ? new PipComponent(package.Name, package.Version) - : new CondaComponent(package.Name, package.Version, null, package.Category, null, null, null, null); - - /// - /// Checks if a package is a python package. - /// - /// If the package is either managed by pip, or if it depends on python - /// it is considered a python package. - /// - /// The CondaPackage. - /// True if the package is a python package. - private static bool IsPythonPackage(CondaPackage package) - => package.Manager.Equals("pip", StringComparison.OrdinalIgnoreCase) || - package.Dependencies.Keys.Any(dependency => dependency.Equals("python", StringComparison.OrdinalIgnoreCase)); + { + return package.Manager.Equals("pip", StringComparison.OrdinalIgnoreCase) + ? new PipComponent(package.Name, package.Version) + : new CondaComponent(package.Name, package.Version, null, package.Category, null, null, null, null); + } } diff --git a/src/Microsoft.ComponentDetection.Detectors/conda/CondaLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/conda/CondaLockComponentDetector.cs index 3394056b6..fff3118fe 100644 --- a/src/Microsoft.ComponentDetection.Detectors/conda/CondaLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/conda/CondaLockComponentDetector.cs @@ -30,7 +30,7 @@ public CondaLockComponentDetector( public override IEnumerable SupportedComponentTypes => [ComponentType.Conda, ComponentType.Pip]; - public override int Version { get; } = 2; + public override int Version { get; } = 3; public override IEnumerable Categories => ["Python"]; diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/CondaLockComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/CondaLockComponentDetectorTests.cs index 4902b205a..bd596a70e 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/CondaLockComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/CondaLockComponentDetectorTests.cs @@ -104,6 +104,342 @@ public async Task CondaComponentDetector_TestCondaLockFileAsync() detectedComponents.Should().HaveCount(4); } + [TestMethod] + [Timeout(3000, CooperativeCancellation = true)] // Fail after 3 seconds if we're stuck in a loop + public async Task CondaComponentDetector_TestCyclicalDependenciesAsync() + { + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + sources: + - environment.yml +package: +- name: pkg-a + version: 1.0.0 + manager: conda + platform: linux-64 + dependencies: + pkg-b: '>=1.0.0' + url: https://conda.anaconda.org/conda-forge/noarch/pkg-a-1.0.0-0.conda + hash: + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + category: main + optional: false +- name: pkg-b + version: 2.0.0 + manager: conda + platform: linux-64 + dependencies: + pkg-a: '>=1.0.0' + url: https://conda.anaconda.org/conda-forge/noarch/pkg-b-2.0.0-0.conda + hash: + sha256: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + category: main + optional: false +"; + + var detectorTask = this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var (scanResult, componentRecorder) = await detectorTask; + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "pkg-a", "1.0.0"); + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "pkg-b", "2.0.0"); + + detectedComponents.Should().HaveCount(2); + } + + [TestMethod] + public async Task CondaComponentDetector_ManagerDeterminesComponentTypeAsync() + { + // A conda-managed package that depends on python must still be a CondaComponent + // (it comes from a conda channel, not PyPI). Only pip-managed packages become PipComponents. + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + sources: + - environment.yml +package: +- name: numpy + version: 1.24.0 + manager: conda + platform: linux-64 + dependencies: + python: '>=3.8' + url: https://conda.anaconda.org/conda-forge/linux-64/numpy-1.24.0-py311.conda + hash: + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + category: main + optional: false +- name: python + version: 3.11.0 + manager: conda + platform: linux-64 + dependencies: {} + url: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.0.conda + hash: + sha256: cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc + category: main + optional: false +- name: boto3 + version: 1.28.0 + manager: pip + platform: linux-64 + dependencies: + python: '>=3.7' + url: https://files.pythonhosted.org/packages/boto3-1.28.0-py3-none-any.whl + hash: + sha256: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + category: main + optional: false +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + // conda-managed packages remain conda components even when they depend on python. + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "numpy", "1.24.0"); + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "python", "3.11.0"); + + // Only the pip-managed package is a pip component. + this.AssertPipComponentNameAndVersion(detectedComponents, "boto3", "1.28.0"); + detectedComponents.Count(c => c.Component is PipComponent).Should().Be(1); + + detectedComponents.Should().HaveCount(3); + } + + [TestMethod] + public async Task CondaComponentDetector_HandlesMissingDependenciesFieldAsync() + { + // A package may omit the "dependencies" field entirely (deserializes to null). + // The detector must not throw and must still register the package. + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + sources: + - environment.yml +package: +- name: solo + version: 1.0.0 + manager: conda + platform: linux-64 + url: https://conda.anaconda.org/conda-forge/linux-64/solo-1.0.0-0.conda + hash: + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + category: main + optional: false +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "solo", "1.0.0"); + detectedComponents.Should().HaveCount(1); + } + + [TestMethod] + public async Task CondaComponentDetector_DeduplicatesPackagesAcrossPlatformsAsync() + { + // The same package/version appears once per platform with platform-specific urls. + // Because the url is intentionally dropped, these collapse into a single component. + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + - win-64 + sources: + - environment.yml +package: +- name: zlib + version: 1.2.13 + manager: conda + platform: linux-64 + dependencies: {} + url: https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.13-0.conda + hash: + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + category: main + optional: false +- name: zlib + version: 1.2.13 + manager: conda + platform: win-64 + dependencies: {} + url: https://conda.anaconda.org/conda-forge/win-64/zlib-1.2.13-0.conda + hash: + sha256: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + category: main + optional: false +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "zlib", "1.2.13"); + detectedComponents.Should().HaveCount(1); + } + + [TestMethod] + public async Task CondaComponentDetector_RecordsDependencyGraphRelationshipsAsync() + { + // parent-pkg -> child-pkg. The root is explicitly referenced; the transitive + // dependency is not, and the parent->child edge is recorded. + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + sources: + - environment.yml +package: +- name: parent-pkg + version: 1.0.0 + manager: conda + platform: linux-64 + dependencies: + child-pkg: '>=1.0.0' + url: https://conda.anaconda.org/conda-forge/linux-64/parent-pkg-1.0.0-0.conda + hash: + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + category: main + optional: false +- name: child-pkg + version: 2.0.0 + manager: conda + platform: linux-64 + dependencies: {} + url: https://conda.anaconda.org/conda-forge/linux-64/child-pkg-2.0.0-0.conda + hash: + sha256: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + category: main + optional: false +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + detectedComponents.Should().HaveCount(2); + + var parentId = detectedComponents.Single(c => c.Component is CondaComponent { Name: "parent-pkg" }).Component.Id; + var childId = detectedComponents.Single(c => c.Component is CondaComponent { Name: "child-pkg" }).Component.Id; + + var graph = componentRecorder.GetDependencyGraphsByLocation().Values.Single(); + + graph.IsComponentExplicitlyReferenced(parentId).Should().BeTrue(); + graph.IsComponentExplicitlyReferenced(childId).Should().BeFalse(); + + graph.GetDependenciesForComponent(parentId).Should().Contain(childId); + graph.GetDependenciesForComponent(childId).Should().BeEmpty(); + } + + [TestMethod] + public async Task CondaComponentDetector_HandlesFileWithNoPackagesAsync() + { + // A lock file with no package section should produce no components and still succeed. + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + sources: + - environment.yml +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + detectedComponents.Should().BeEmpty(); + } + + [TestMethod] + public async Task CondaComponentDetector_DetectsNamedCondaLockFileAsync() + { + // The detector also matches "*.conda-lock.yml" (e.g. environment-specific lock files). + var condaLockContent = +@"version: 1 +metadata: + platforms: + - linux-64 + sources: + - environment.yml +package: +- name: openssl + version: 3.1.0 + manager: conda + platform: linux-64 + dependencies: {} + url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.0-0.conda + hash: + sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + category: main + optional: false +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("environment.conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + this.AssertCondaLockComponentNameAndVersion(detectedComponents, "openssl", "3.1.0"); + detectedComponents.Should().HaveCount(1); + } + + [TestMethod] + public async Task CondaComponentDetector_HandlesMalformedYamlGracefullyAsync() + { + // A malformed lock file should be caught, logged, and not fail the overall scan. + var condaLockContent = +@"version: 1 +package: +- name: [this is not valid + version: : : : +"; + + var (scanResult, componentRecorder) = await this.detectorTestUtility + .WithFile("conda-lock.yml", condaLockContent) + .ExecuteDetectorAsync(); + + var detectedComponents = componentRecorder.GetDetectedComponents(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + detectedComponents.Should().BeEmpty(); + } + private void AssertCondaLockComponentNameAndVersion(IEnumerable detectedComponents, string name, string version) { detectedComponents.SingleOrDefault(c =>