Skip to content

Update LinuxApplicationLayerExperiment control group #1616

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JamieMagee merged 1 commit into from
Jan 16, 2026
Merged

Update LinuxApplicationLayerExperiment control group #1616

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions ...ft.ComponentDetection.Orchestrator/Experiments/Configs/LinuxApplicationLayerExperiment.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@ namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
/// <summary>
/// Experiment to validate the <see cref="LinuxApplicationLayerDetector"/> which captures application-level
/// packages in addition to system packages from Linux containers.
/// Control group includes the standard file-based npm and pip detectors plus the Linux system package detector.
/// Experiment group uses container-based detection for all package types together.
/// Control group uses file-based detectors plus LinuxContainerDetector (system packages only).
/// Experiment group uses file-based detectors plus LinuxApplicationLayerDetector (system + application packages).
/// The diff reveals net-new application packages found only inside containers (e.g., RUN npm add lodash),
/// excluding both manifest-detected components (canceled by file-based detectors) and system packages
/// (canceled by LinuxContainerDetector).
/// </summary>
public class LinuxApplicationLayerExperiment : IExperimentConfiguration
{
Expand All @@ -30,20 +33,25 @@ or NuGetProjectModelProjectCentricComponentDetector

/// <inheritdoc />
public bool IsInExperimentGroup(IComponentDetector componentDetector) =>
componentDetector is LinuxApplicationLayerDetector;
componentDetector
is LinuxApplicationLayerDetector
or NpmComponentDetector
or NpmLockfileDetectorBase
or PipReportComponentDetector
or NuGetComponentDetector
or NuGetProjectModelProjectCentricComponentDetector
or NuGetPackagesConfigDetector;

/// <inheritdoc />
public bool ShouldRecord(IComponentDetector componentDetector, int numComponents)
{
// Only record telemetry if the experiment group detector (LinuxApplicationLayerDetector)
// actually found components.
if (componentDetector is LinuxApplicationLayerDetector)
// Only record telemetry if a Linux container detector found components,
// indicating containers were detected and scanned.
if (componentDetector is LinuxContainerDetector)
{
return numComponents > 0;
}

// For control group detectors, record if the experiment group found anything
// This will be determined by the orchestrator based on whether the experiment group had components
return true;
}
}
129 changes: 61 additions & 68 deletions ...ComponentDetection.Orchestrator.Tests/Experiments/LinuxApplicationLayerExperimentTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,50 +22,17 @@ public void IsInControlGroup_LinuxContainerDetector_ReturnsTrue()
}

[TestMethod]
public void IsInControlGroup_NpmComponentDetector_ReturnsTrue()
public void IsInControlGroup_FileBasedDetectors_ReturnsTrue()
{
var npmDetector = new NpmComponentDetector(null, null, null);
this.experiment.IsInControlGroup(npmDetector).Should().BeTrue();
}

[TestMethod]
public void IsInControlGroup_NpmLockfile3Detector_ReturnsTrue()
{
var npmLockfile3Detector = new NpmLockfile3Detector(null, null, null, null);
this.experiment.IsInControlGroup(npmLockfile3Detector).Should().BeTrue();
}

[TestMethod]
public void IsInControlGroup_NpmComponentDetectorWithRoots_ReturnsTrue()
{
var npmDetectorWithRoots = new NpmComponentDetectorWithRoots(null, null, null, null);
this.experiment.IsInControlGroup(npmDetectorWithRoots).Should().BeTrue();
}

[TestMethod]
public void IsInControlGroup_NuGetComponentDetector_ReturnsTrue()
{
var nuGetDetector = new NuGetComponentDetector(null, null, null);
this.experiment.IsInControlGroup(nuGetDetector).Should().BeTrue();
}

[TestMethod]
public void IsInControlGroup_NuGetProjectModelProjectCentricComponentDetector_ReturnsTrue()
{
var nuGetDetector = new NuGetProjectModelProjectCentricComponentDetector(null, null, null, null);
this.experiment.IsInControlGroup(nuGetDetector).Should().BeTrue();
}

[TestMethod]
public void IsInControlGroup_NuGetPackagesConfigDetector_ReturnsTrue()
{
var nuGetPackagesConfigDetector = new NuGetPackagesConfigDetector(null, null, null);
this.experiment.IsInControlGroup(nuGetPackagesConfigDetector).Should().BeTrue();
}

[TestMethod]
public void IsInControlGroup_PipReportComponentDetector_ReturnsTrue()
{
var pipDetector = new PipReportComponentDetector(
null,
null,
Expand All @@ -77,8 +44,21 @@ public void IsInControlGroup_PipReportComponentDetector_ReturnsTrue()
null,
null
);

this.experiment.IsInControlGroup(pipDetector).Should().BeTrue();

var nuGetDetector = new NuGetComponentDetector(null, null, null);
this.experiment.IsInControlGroup(nuGetDetector).Should().BeTrue();

var nuGetProjectCentricDetector = new NuGetProjectModelProjectCentricComponentDetector(
null,
null,
null,
null
);
this.experiment.IsInControlGroup(nuGetProjectCentricDetector).Should().BeTrue();

var nuGetPackagesConfigDetector = new NuGetPackagesConfigDetector(null, null, null);
this.experiment.IsInControlGroup(nuGetPackagesConfigDetector).Should().BeTrue();
}

[TestMethod]
Expand All @@ -103,29 +83,17 @@ public void IsInExperimentGroup_LinuxContainerDetector_ReturnsFalse()
}

[TestMethod]
public void IsInExperimentGroup_NpmComponentDetector_ReturnsFalse()
public void IsInExperimentGroup_FileBasedDetectors_ReturnsTrue()
{
var npmDetector = new NpmComponentDetector(null, null, null);
this.experiment.IsInExperimentGroup(npmDetector).Should().BeFalse();
}
this.experiment.IsInExperimentGroup(npmDetector).Should().BeTrue();

[TestMethod]
public void IsInExperimentGroup_NpmLockfile3Detector_ReturnsFalse()
{
var npmLockfile3Detector = new NpmLockfile3Detector(null, null, null, null);
this.experiment.IsInExperimentGroup(npmLockfile3Detector).Should().BeFalse();
}
this.experiment.IsInExperimentGroup(npmLockfile3Detector).Should().BeTrue();

[TestMethod]
public void IsInExperimentGroup_NpmComponentDetectorWithRoots_ReturnsFalse()
{
var npmDetectorWithRoots = new NpmComponentDetectorWithRoots(null, null, null, null);
this.experiment.IsInExperimentGroup(npmDetectorWithRoots).Should().BeFalse();
}
this.experiment.IsInExperimentGroup(npmDetectorWithRoots).Should().BeTrue();

[TestMethod]
public void IsInExperimentGroup_PipReportComponentDetector_ReturnsFalse()
{
var pipDetector = new PipReportComponentDetector(
null,
null,
Expand All @@ -137,22 +105,21 @@ public void IsInExperimentGroup_PipReportComponentDetector_ReturnsFalse()
null,
null
);
this.experiment.IsInExperimentGroup(pipDetector).Should().BeTrue();

this.experiment.IsInExperimentGroup(pipDetector).Should().BeFalse();
}

[TestMethod]
public void IsInExperimentGroup_NuGetComponentDetector_ReturnsFalse()
{
var nuGetDetector = new NuGetComponentDetector(null, null, null);
this.experiment.IsInExperimentGroup(nuGetDetector).Should().BeFalse();
}
this.experiment.IsInExperimentGroup(nuGetDetector).Should().BeTrue();

[TestMethod]
public void IsInExperimentGroup_NuGetProjectModelProjectCentricComponentDetector_ReturnsFalse()
{
var nuGetDetector = new NuGetProjectModelProjectCentricComponentDetector(null, null, null, null);
this.experiment.IsInExperimentGroup(nuGetDetector).Should().BeFalse();
var nuGetProjectCentricDetector = new NuGetProjectModelProjectCentricComponentDetector(
null,
null,
null,
null
);
this.experiment.IsInExperimentGroup(nuGetProjectCentricDetector).Should().BeTrue();

var nuGetPackagesConfigDetector = new NuGetPackagesConfigDetector(null, null, null);
this.experiment.IsInExperimentGroup(nuGetPackagesConfigDetector).Should().BeTrue();
}

[TestMethod]
Expand All @@ -170,21 +137,47 @@ public void ShouldRecord_ExperimentGroup_ReturnsFalse_WhenNumComponentsIsZero()
}

[TestMethod]
public void ShouldRecord_ControlGroup_AlwaysReturnsTrue()
public void ShouldRecord_ControlGroup_ReturnsTrue_WhenNumComponentsGreaterThanZero()
{
var linuxDetector = new LinuxContainerDetector(null, null, null);
this.experiment.ShouldRecord(linuxDetector, 0).Should().BeTrue();
this.experiment.ShouldRecord(linuxDetector, 1).Should().BeTrue();
}

[TestMethod]
public void ShouldRecord_ControlGroup_ReturnsFalse_WhenNumComponentsIsZero()
{
var linuxDetector = new LinuxContainerDetector(null, null, null);
this.experiment.ShouldRecord(linuxDetector, 0).Should().BeFalse();
}

[TestMethod]
public void ShouldRecord_FileBasedDetectors_AlwaysReturnsTrue()
{
var npmDetector = new NpmComponentDetector(null, null, null);
this.experiment.ShouldRecord(npmDetector, 0).Should().BeTrue();

var pipDetector = new PipReportComponentDetector(null, null, null, null, null, null, null, null, null);
var pipDetector = new PipReportComponentDetector(
null,
null,
null,
null,
null,
null,
null,
null,
null
);
this.experiment.ShouldRecord(pipDetector, 0).Should().BeTrue();

var nuGetDetector = new NuGetComponentDetector(null, null, null);
this.experiment.ShouldRecord(nuGetDetector, 0).Should().BeTrue();

var nuGetProjectCentricDetector = new NuGetProjectModelProjectCentricComponentDetector(null, null, null, null);
var nuGetProjectCentricDetector = new NuGetProjectModelProjectCentricComponentDetector(
null,
null,
null,
null
);
this.experiment.ShouldRecord(nuGetProjectCentricDetector, 0).Should().BeTrue();

var nuGetPackagesConfigDetector = new NuGetPackagesConfigDetector(null, null, null);
Expand Down