Add MAUI Android inner loop deploy measurement scenario

eng/performance/maui_scenarios_android_innerloop.proj

@@ -0,0 +1,100 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <Import Project="Scenarios.Common.props" />
+  <!-- TODO: Scenarios.Common.props sets HelixResultsDestinationDir to a path under
+       CorrelationPayloadDirectory, which is cleaned up after SendToHelix. As a result,
+       this file's DownloadFilesFromResults artifacts (e.g., output.log) may not appear
+       in published pipeline artifacts. A future fix should override
+       HelixResultsDestinationDir to a path under ArtifactStagingDirectory that the
+       pipeline's "Gather logs" step publishes. -->
+
+  <PropertyGroup>
+    <IncludeXHarnessCli>true</IncludeXHarnessCli>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'mono'">/p:UseMonoRuntime=true</_MSBuildArgs>
+    <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr'">/p:UseMonoRuntime=false</_MSBuildArgs>
+
+    <!-- CoreCLR JIT (all inner loop runs use codeGenType=Default; AOT/R2R/NativeAOT are unused) -->
+    <_MSBuildArgs Condition="'$(RuntimeFlavor)' == 'coreclr' and '$(CodegenType)' == 'JIT'">$(_MSBuildArgs) /p:PublishReadyToRun=false /p:PublishReadyToRunComposite=false</_MSBuildArgs>
+
+    <!-- AndroidRid: defaults based on target OS but can be overridden.
+         Windows targets physical Pixel 8 (ARM64); Linux targets x86_64 emulator. -->
+    <AndroidRid Condition="'$(AndroidRid)' == '' and '$(TargetsWindows)' == 'true'">android-arm64</AndroidRid>
+    <AndroidRid Condition="'$(AndroidRid)' == '' and '$(TargetsWindows)' != 'true'">android-x64</AndroidRid>
+    <_MSBuildArgs>$(_MSBuildArgs) /p:RuntimeIdentifier=$(AndroidRid)</_MSBuildArgs>
+
+    <!-- TODO: https://github.com/dotnet/maui/issues/34706 -->
+    <_MSBuildArgs>$(_MSBuildArgs) /p:SupportedOSPlatformVersion=23</_MSBuildArgs>
+    <!-- Add TargetFrameworks override so dotnet build's implicit restore
+         only evaluates the android TFM, not multi-platform TFMs in the csproj. -->
+    <_MSBuildArgs Condition="!$(_MSBuildArgs.Contains('/p:TargetFrameworks='))">$(_MSBuildArgs) /p:TargetFrameworks=$(PERFLAB_Framework)-android</_MSBuildArgs>
+
+    <RunConfigsString>$(RuntimeFlavor)_$(CodegenType)</RunConfigsString>
+    <InnerLoopIterations Condition="'$(InnerLoopIterations)' == ''">10</InnerLoopIterations>
+    <!-- Screen timeout must be large enough so the display stays on for the entire scenario.
+         If the screen turns off mid-run, am start reports LaunchState: UNKNOWN and omits
+         TotalTime, which breaks cold startup measurement. -->
+    <ScreenTimeoutMs Condition="'$(ScreenTimeoutMs)' == ''">1800000</ScreenTimeoutMs>
+  </PropertyGroup>
+
+  <!-- Remove .NET SDK packs from the Helix correlation payload to reduce transfer size.
+       The workload packs (~1-2 GB) are restored on the Helix machine during the run step,
+       so shipping them in the payload is unnecessary and slows down payload upload. -->
+  <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
+    <Message Text="Removing Dotnet Packs from Correlation Staging" Importance="high" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet\packs" Condition="'$(TargetsWindows)' == 'true'" />
+    <RemoveDir Directories="$(CorrelationPayloadDirectory)dotnet/packs" Condition="'$(TargetsWindows)' != 'true'" />
+  </Target>
+
+  <ItemDefinitionGroup>
+    <HelixWorkItem>
+      <Timeout>01:00</Timeout>
+    </HelixWorkItem>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <MAUIAndroidInnerLoopScenario Include="MAUI Android Inner Loop">
+      <ScenarioDirectoryName>mauiandroidinnerloop</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+    </MAUIAndroidInnerLoopScenario>
+  </ItemGroup>
+
+  <!-- PreparePayloadWorkItem: create template and modified source on the build machine.
+       pre.py creates the MAUI template in app/, fixes .csproj TFMs, and prepares the
+       modified MainPage.xaml.cs in src/ for the incremental deploy simulation. -->
+  <ItemGroup>
+    <PreparePayloadWorkItem Include="@(MAUIAndroidInnerLoopScenario)">
+      <Command>$(Python) pre.py default -f $(PERFLAB_Framework)</Command>
+      <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
+    </PreparePayloadWorkItem>
+  </ItemGroup>
+
+  <!-- Environment variables needed on Helix before setup_helix.py runs (so PATH includes adb). -->
+  <PropertyGroup>
+    <_WindowsEnvVars>set DOTNET_ROOT=%HELIX_CORRELATION_PAYLOAD%\dotnet;set DOTNET_CLI_TELEMETRY_OPTOUT=1;set DOTNET_MULTILEVEL_LOOKUP=0;set NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\.packages;set ANDROID_HOME=%HELIX_WORKITEM_ROOT%\android-sdk;set ANDROID_SDK_ROOT=%HELIX_WORKITEM_ROOT%\android-sdk;set JAVA_HOME=%HELIX_WORKITEM_ROOT%\jdk;set PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet;;%HELIX_WORKITEM_ROOT%\android-sdk\platform-tools;;%HELIX_WORKITEM_ROOT%\jdk\bin;;%PATH%</_WindowsEnvVars>
+    <_LinuxEnvVars>export DOTNET_ROOT=$HELIX_CORRELATION_PAYLOAD/dotnet;export DOTNET_CLI_TELEMETRY_OPTOUT=1;export DOTNET_MULTILEVEL_LOOKUP=0;export NUGET_PACKAGES=$HELIX_WORKITEM_ROOT/.packages;export ANDROID_HOME=$HELIX_WORKITEM_ROOT/android-sdk;export ANDROID_SDK_ROOT=$HELIX_WORKITEM_ROOT/android-sdk;export JAVA_HOME=$HELIX_WORKITEM_ROOT/jdk;export ANDROID_SERIAL=emulator-5554;export PATH=$HELIX_CORRELATION_PAYLOAD/dotnet:$HELIX_WORKITEM_ROOT/android-sdk/platform-tools:$HELIX_WORKITEM_ROOT/jdk/bin:$PATH</_LinuxEnvVars>
+  </PropertyGroup>
+
+  <ItemGroup Condition="'$(TargetsWindows)' == 'true'">
+    <HelixWorkItem Include="@(MAUIAndroidInnerLoopScenario -> 'Inner Loop Device - %(Identity)')">
+      <PreCommands>$(_WindowsEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android &quot;$(_MSBuildArgs)&quot;</PreCommands>
+      <Command>$(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src &quot;src/MainPage.xaml.cs;src/MainPage.xaml&quot; --edit-dest &quot;app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml&quot; --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args &quot;$(_MSBuildArgs)&quot; --scenario-name &quot;%(Identity)&quot; --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs)</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+      <DownloadFilesFromResults>output.log</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetsWindows)' != 'true'">
+    <HelixWorkItem Include="@(MAUIAndroidInnerLoopScenario -> 'Inner Loop Emulator - %(Identity)')">
+      <PreCommands>$(_LinuxEnvVars);$(Python) setup_helix.py $(PERFLAB_Framework)-android &quot;$(_MSBuildArgs)&quot;</PreCommands>
+      <Command>$(Python) test.py androidinnerloop --csproj-path app/MauiAndroidInnerLoop.csproj --edit-src &quot;src/MainPage.xaml.cs;src/MainPage.xaml&quot; --edit-dest &quot;app/Pages/MainPage.xaml.cs;app/Pages/MainPage.xaml&quot; --package-name com.companyname.mauiandroidinnerloop -f $(PERFLAB_Framework)-android -c Debug --msbuild-args &quot;$(_MSBuildArgs)&quot; --scenario-name &quot;%(Identity)&quot; --inner-loop-iterations $(InnerLoopIterations) --screen-timeout-ms $(ScreenTimeoutMs) $(ScenarioArgs)</Command>
+      <PostCommands>$(Python) post.py</PostCommands>
+      <DownloadFilesFromResults>output.log</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup>
+
+  <Import Project="PreparePayloadWorkItems.targets" />
+
+</Project>

eng/pipelines/sdk-perf-jobs.yml

@@ -527,6 +527,84 @@ jobs:
         ${{ each parameter in parameters.jobParameters }}:
           ${{ parameter.key }}: ${{ parameter.value }}
 
+  # Maui Android inner loop benchmarks on devices (Mono Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - win-x64-android-arm64-pixel
+        - win-x64-android-arm64-galaxy
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android_innerloop.proj
+        channels:
+          - main
+        runtimeFlavor: mono
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: Mono_Debug_InnerLoop
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Maui Android inner loop benchmarks on devices (CoreCLR Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - win-x64-android-arm64-pixel
+        - win-x64-android-arm64-galaxy
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android_innerloop.proj
+        channels:
+          - main
+        runtimeFlavor: coreclr
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: CoreCLR_Debug_InnerLoop
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Maui Android inner loop benchmarks on emulator (Mono Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - ubuntu-x64-android-emulator
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android_innerloop.proj
+        channels:
+          - main
+        runtimeFlavor: mono
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: Mono_Debug_InnerLoop_Emulator
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
+  # Maui Android inner loop benchmarks on emulator (CoreCLR Default) - Debug
+  - template: /eng/pipelines/templates/build-machine-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml
+      buildMachines:
+        - ubuntu-x64-android-emulator
+      isPublic: false
+      jobParameters:
+        runKind: maui_scenarios_android_innerloop
+        projectFileName: maui_scenarios_android_innerloop.proj
+        channels:
+          - main
+        runtimeFlavor: coreclr
+        codeGenType: Default
+        buildConfig: Debug
+        additionalJobIdentifier: CoreCLR_Debug_InnerLoop_Emulator
+        ${{ each parameter in parameters.jobParameters }}:
+          ${{ parameter.key }}: ${{ parameter.value }}
+
   # Maui iOS scenario benchmarks (Mono - Default) - Debug
   - template: /eng/pipelines/templates/build-machine-matrix.yml
     parameters:

eng/pipelines/templates/build-machine-matrix.yml

@@ -146,6 +146,19 @@ jobs:
       machinePool: GalaxyA16
       ${{ insert }}: ${{ parameters.jobParameters }}
 
+- ${{ if and(containsValue(parameters.buildMachines, 'ubuntu-x64-android-emulator'), not(eq(parameters.isPublic, true))) }}: # Ubuntu x64 Android emulator only used in private builds currently
+  - template: ${{ parameters.jobTemplate }}
+    parameters:
+      osGroup: ubuntu
+      archType: x64
+      osVersion: 2204
+      pool:
+        vmImage: ubuntu-latest
+      container: ubuntu_x64_build_container
+      queue: Ubuntu.2204.Amd64.Android.36
+      machinePool: AndroidEmulator
+      ${{ insert }}: ${{ parameters.jobParameters }}
+
 - ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 17 only used in private builds currently
   - template: ${{ parameters.jobTemplate }}
     parameters:

eng/pipelines/templates/run-performance-job.yml

@@ -190,7 +190,7 @@ jobs:
               - '--is-scenario'
             - ${{ if ne(length(parameters.runEnvVars), 0) }}:
               - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}"
-            - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android'), ne(parameters.runtimeFlavor, '')) }}:
+            - ${{ if and(in(parameters.runKind, 'maui_scenarios_ios', 'maui_scenarios_android', 'maui_scenarios_android_innerloop'), ne(parameters.runtimeFlavor, '')) }}:
               - '--runtime-flavor ${{ parameters.runtimeFlavor }}'
             - ${{ if ne(parameters.osVersion, '') }}:
               - '--os-version ${{ parameters.osVersion }}'

scripts/run_performance_job.py

@@ -570,9 +570,9 @@ def get_run_configurations(
             configurations["iOSLlvmBuild"] = str(ios_llvm_build)
 
     # .NET Android and .NET MAUI Android sample app scenarios
-    if run_kind == "maui_scenarios_android":
+    if run_kind in ["maui_scenarios_android", "maui_scenarios_android_innerloop"]:
         if not runtime_flavor in ("mono", "coreclr"):
-            raise Exception("Runtime flavor must be specified for maui_scenarios_android")
+            raise Exception(f"Runtime flavor must be specified for {run_kind}")
         configurations["CodegenType"] = str(codegen_type)
         configurations["RuntimeType"] = str(runtime_flavor)
         if build_config is not None and build_config != DEFAULT_BUILD_CONFIG:
@@ -1190,7 +1190,7 @@ def publish_dotnet_app_to_payload(payload_dir_name: str, csproj_path: str, self_
                 verbose=True).run()
 
             # Search for additional binlogs generated by the maui scenarios prepare payload work items to copy to the artifacts log dir
-            if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios"]:
+            if args.run_kind in ["maui_scenarios_android", "maui_scenarios_ios", "maui_scenarios_android_innerloop"]:
                 for binlog_path in glob(os.path.join(payload_dir, "scenarios_out", "**", "*.binlog"), recursive=True):
                     shutil.copy(binlog_path, ci_artifacts_log_dir)
 

src/scenarios/mauiandroidinnerloop/post.py

@@ -0,0 +1,28 @@
+'''
+post cleanup script
+'''
+
+import subprocess
+import sys
+import traceback
+from performance.logger import setup_loggers, getLogger
+from shared.postcommands import clean_directories
+from shared.util import xharness_adb
+from test import EXENAME
+
+setup_loggers(True)
+logger = getLogger(__name__)
+
+try:
+    # Uninstall the app from the connected device so re-runs start from a clean state
+    package_name = f'com.companyname.{EXENAME.lower()}'
+    logger.info(f"Uninstalling {package_name} from device")
+    subprocess.run(xharness_adb() + ['uninstall', package_name], check=False)
+
+    logger.info("Shutting down dotnet build servers")
+    subprocess.run(['dotnet', 'build-server', 'shutdown'], check=False)
+
+    clean_directories()
+except Exception as e:
+    logger.error(f"Post cleanup failed: {e}\n{traceback.format_exc()}")
+    sys.exit(1)