[DO NOT REVIEW] [DRAFT] Flakey pipeline fix

.pipelines/dwsql-pipelines.yml

@@ -76,7 +76,7 @@ jobs:
     inputs:
       targetType: 'inline'
       script: |
-        connectionString="Server=tcp:127.0.0.1,1433;Persist Security Info=False;User ID=SA;Password=$(dbPassword);MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;Encrypt=False;"
+        connectionString="Server=tcp:127.0.0.1,1433;Persist Security Info=False;User ID=SA;Password=$(dbPassword);MultipleActiveResultSets=False;Connection Timeout=60;TrustServerCertificate=True;Encrypt=False;"
         echo "##vso[task.setvariable variable=data-source.connection-string;]$connectionString"
 
   - task: Bash@3
@@ -158,7 +158,7 @@ jobs:
       # since windows needs a different string.
       # The variable setting on the pipeline UI sets the connection string
       # for the linux job above.
-      data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;
+      data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=60;TrustServerCertificate=True;
       InstallerUrl: https://download.microsoft.com/download/7/c/1/7c14e92e-bdcb-4f89-b7cf-93543e7112d1/SqlLocalDB.msi
       SqlVersionCode: '15.0'
 

.pipelines/mssql-pipelines.yml

@@ -78,7 +78,7 @@ jobs:
     inputs:
       targetType: 'inline'
       script: |
-        $connectionString="Server=tcp:127.0.0.1,1433;Persist Security Info=False;User ID=SA;Password=$(dbPassword);MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;Encrypt=False;"
+        $connectionString="Server=tcp:127.0.0.1,1433;Persist Security Info=False;User ID=SA;Password=$(dbPassword);MultipleActiveResultSets=False;Connection Timeout=60;TrustServerCertificate=True;Encrypt=False;"
         Write-Host "##vso[task.setvariable variable=data-source.connection-string]$connectionString"
 
   - task: Bash@3
@@ -175,7 +175,7 @@ jobs:
       # since windows needs a different string.
       # The variable setting on the pipeline UI sets the connection string
       # for the linux job above.
-      data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;
+      data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=60;TrustServerCertificate=True;
       InstallerUrl: https://download.microsoft.com/download/7/c/1/7c14e92e-bdcb-4f89-b7cf-93543e7112d1/SqlLocalDB.msi
       SqlVersionCode: '15.0'
 
@@ -198,7 +198,7 @@ jobs:
       # since windows needs a different string.
       # The variable setting on the pipeline UI sets the connection string
       # for the linux job above.
-      data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;
+      data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=60;TrustServerCertificate=True;
       InstallerUrl: https://download.microsoft.com/download/7/c/1/7c14e92e-bdcb-4f89-b7cf-93543e7112d1/SqlLocalDB.msi
       SqlVersionCode: '15.0'
 

.pipelines/templates/mssql-test-steps.yml

@@ -70,6 +70,49 @@ steps:
       SqlLocalDb.exe start MSSQLLocalDB
       SqlLocalDb.exe info "MSSQLLocalDB"
 
+# Warm up LocalDB so the first real test connection isn't paying the cold-start
+# cost. LocalDB's TDS pre-login phase can take 30-60s on a cold instance, which
+# was timing out class-init metadata fetches in tests like
+# MsSqlMultipleMutationBuilderTests (~60 fresh connections opened sequentially).
+# We retry SELECT 1 for up to ~2 minutes; failure here is non-fatal so the
+# pipeline still proceeds to the actual test step which will surface a clearer
+# error if LocalDB really is unreachable.
+- task: PowerShell@2
+  displayName: 'Warm up LocalDB (retry SELECT 1)'
+  inputs:
+    targetType: 'inline'
+    script: |
+      $connStr = "Server=(localdb)\MSSQLLocalDB;Integrated Security=True;TrustServerCertificate=True;Connection Timeout=60;"
+      $maxAttempts = 6
+      $attempt = 0
+      $success = $false
+      while (-not $success -and $attempt -lt $maxAttempts) {
+          $attempt++
+          $sw = [System.Diagnostics.Stopwatch]::StartNew()
+          try {
+              $connection = New-Object System.Data.SqlClient.SqlConnection($connStr)
+              $connection.Open()
+              $cmd = $connection.CreateCommand()
+              $cmd.CommandText = "SELECT 1"
+              $cmd.CommandTimeout = 60
+              [void]$cmd.ExecuteScalar()
+              $connection.Close()
+              $sw.Stop()
+              Write-Host "LocalDB warm-up succeeded on attempt $attempt in $($sw.ElapsedMilliseconds) ms."
+              $success = $true
+          }
+          catch {
+              $sw.Stop()
+              Write-Warning "LocalDB warm-up attempt $attempt failed after $($sw.ElapsedMilliseconds) ms: $_"
+              if ($attempt -lt $maxAttempts) {
+                  Start-Sleep -Seconds 5
+              }
+          }
+      }
+      if (-not $success) {
+          Write-Warning "LocalDB warm-up did not succeed after $maxAttempts attempts; continuing - test step will surface the real error."
+      }
+
 - ${{ if eq(parameters.initDbSchema, true) }}:
   - task: PowerShell@2
     displayName: 'Initialize database schema'
@@ -79,7 +122,7 @@ steps:
         Write-Host "Running DatabaseSchema-MsSql.sql against LocalDB via SqlClient..."
         $sqlFile = "$(System.DefaultWorkingDirectory)\src\Service.Tests\DatabaseSchema-MsSql.sql"
         $sql = Get-Content $sqlFile -Raw
-        $connStr = "Server=(localdb)\MSSQLLocalDB;Integrated Security=True;TrustServerCertificate=True;Connection Timeout=30;"
+        $connStr = "Server=(localdb)\MSSQLLocalDB;Integrated Security=True;TrustServerCertificate=True;Connection Timeout=60;"
         $connection = New-Object System.Data.SqlClient.SqlConnection($connStr)
         try {
             $connection.Open()

src/Service.Tests/Configuration/ConfigurationTests.cs

@@ -1168,6 +1168,22 @@ public async Task TestLongRunningConfigUpdatedHandlerConfigurations(string confi
         [DataRow(CONFIGURATION_ENDPOINT_V2)]
         public async Task TestSqlSettingPostStartupConfigurations(string configurationEndpoint)
         {
+            // Other tests in this class set ASPNETCORE_ENVIRONMENT (e.g. to "MsSql") and do not
+            // reset it. MSTest reuses the test process, so a leaked env-var causes the
+            // FileSystemRuntimeConfigLoader to auto-load dab-config.MsSql.json on TestServer
+            // construction. That defeats the "post-startup" premise of this test: the engine
+            // would already be hydrated, isRuntimeReady would be true, and pre-hydration GETs
+            // would not return 503. Clear the env-var so the late-configured branch of
+            // Startup.Configure runs, regardless of which tests ran before this one.
+            Environment.SetEnvironmentVariable(ASP_NET_CORE_ENVIRONMENT_VAR_NAME, null);
+
+            // Program.IsHttpsRedirectionDisabled is also static-state-leaky. Other tests
+            // call Program.CreateWebHostBuilder with args that flip this flag based on the
+            // command-line they pass; if a previous test left the flag false, the
+            // UseHttpsRedirection middleware can intercept pre-hydration GETs before the
+            // 503 gating middleware runs. Force-disable HTTPS redirection for this test so
+            // the assertions exercise the real "service unavailable" path.
+            Program.IsHttpsRedirectionDisabled = true;
             using TestServer server = new(Program.CreateWebHostFromInMemoryUpdatableConfBuilder(Array.Empty<string>()));
             HttpClient httpClient = server.CreateClient();
 

src/Service/Program.cs

@@ -35,7 +35,7 @@ namespace Azure.DataApiBuilder.Service
 {
     public class Program
     {
-        public static bool IsHttpsRedirectionDisabled { get; private set; }
+        public static bool IsHttpsRedirectionDisabled { get; internal set; }
         public static DynamicLogLevelProvider LogLevelProvider = new();
 
         public static void Main(string[] args)