Refactor execution creation and cleanup logic for improved reliability

config/puma.rb

@@ -24,6 +24,8 @@
 pidfile 'puma.pid'
 
 before_fork do
+  ActiveRecord::Base.connection_pool.disconnect!
+
   Thread.new do
     loop do
       Telemetry.instance.update_stats Puma.stats
@@ -32,3 +34,7 @@
     end
   end
 end
+
+on_worker_boot do
+  ActiveRecord::Base.establish_connection
+end

db/migrate/20260526000001_add_indexes_to_delayed_jobs.rb

@@ -0,0 +1,32 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20260526000001_add_indexes_to_delayed_jobs.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2026 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class AddIndexesToDelayedJobs < ActiveRecord::Migration[7.2]
+  def change
+    # Composite index for the main polling query:
+    # SELECT * FROM delayed_jobs WHERE run_at <= NOW() AND locked_at IS NULL AND failed_at IS NULL
+    # ORDER BY priority ASC, run_at ASC LIMIT N
+    add_index :delayed_jobs, %i[priority run_at],
+              where: 'locked_at IS NULL AND failed_at IS NULL',
+              name: 'index_delayed_jobs_on_priority_and_run_at'
+
+    # Index for locking queries (worker claims a job):
+    # SELECT * FROM delayed_jobs WHERE locked_at IS NOT NULL ...
+    add_index :delayed_jobs, :locked_at,
+              where: 'locked_at IS NOT NULL',
+              name: 'index_delayed_jobs_on_locked_at'
+
+    # Index for failed job queries used by PrometheusMetrics:
+    # SELECT * FROM delayed_jobs WHERE failed_at IS NOT NULL ...
+    add_index :delayed_jobs, :failed_at,
+              where: 'failed_at IS NOT NULL',
+              name: 'index_delayed_jobs_on_failed_at'
+  end
+end

db/migrate/20260526000002_add_indexes_to_ci_jobs_and_stages_status.rb

@@ -0,0 +1,37 @@
+#  SPDX-License-Identifier: BSD-2-Clause
+#
+#  20260526000002_add_indexes_to_ci_jobs_and_stages_status.rb
+#  Part of NetDEF CI System
+#
+#  Copyright (c) 2026 by
+#  Network Device Education Foundation, Inc. ("NetDEF")
+#
+#  frozen_string_literal: true
+
+class AddIndexesToCiJobsAndStagesStatus < ActiveRecord::Migration[7.2]
+  def change
+    # Covers: ci_jobs.where(status: :in_progress)
+    #         ci_jobs.where(status: %i[queued in_progress])
+    add_index :ci_jobs, :status,
+              name: 'index_ci_jobs_on_status'
+
+    # Covers the most common combined filter:
+    # ci_jobs.where(check_suite_id: id, status: %i[queued in_progress])
+    # check_suite.rb:42 — running_jobs
+    # check_suite.rb:50 — execution_started?
+    add_index :ci_jobs, %i[check_suite_id status],
+              name: 'index_ci_jobs_on_check_suite_id_and_status'
+
+    # Covers: stages.where(status: %i[queued in_progress]).any?
+    # check_suite.rb:38 — running?
+    add_index :stages, :status,
+              name: 'index_stages_on_status'
+
+    # Covers the most common combined filter:
+    # stages.where(check_suite_id: id, status: %i[queued in_progress])
+    # check_suite.rb:38 — running?
+    # stage.rb:42       — running?
+    add_index :stages, %i[check_suite_id status],
+              name: 'index_stages_on_check_suite_id_and_status'
+  end
+end

lib/github/build/plan_run.rb

@@ -23,7 +23,7 @@ def build
         @pull_request.plans.each do |plan|
           CreateExecutionByPlan
             .delay(run_at: TIMER.seconds.from_now.utc, queue: 'create_execution_by_plan')
-            .create(@pull_request.id, @payload, plan)
+            .create(@pull_request.id, @payload, plan.id)
         end
 
         [200, 'Scheduled Plan Runs']

lib/github/re_run/base.rb

@@ -130,7 +130,7 @@ def update_unavailable_jobs(check_suite)
 
       def cleanup(check_suite)
         check_suite.pull_request.check_suites.each do |suite|
-          Delayed::Job.where('handler LIKE ?', "%method_name: :timeout\nargs:\n- #{suite.id}%")
+          Delayed::Job.where('handler LIKE ?', "%method_name: :timeout\nargs:\n- #{suite.id}%").destroy_all
         end
       end
 

lib/github/re_run/comment.rb

@@ -44,7 +44,7 @@ def confirm_and_start
         @pull_request.plans.each do |plan|
           CreateExecutionByComment
             .delay(run_at: TIMER.seconds.from_now.utc, queue: 'create_execution_by_comment')
-            .create(@pull_request.id, @payload, plan)
+            .create(@pull_request.id, @payload, plan.id)
         end
 
         [200, 'Scheduled Plan Runs']

lib/helpers/sinatra_payload.rb

@@ -18,7 +18,8 @@ def authenticate_metrics
       auth = request.env['HTTP_AUTHORIZATION']
       config = GitHubApp::Configuration.instance.config['metrics_auth']
 
-      return halt 401, 'Unauthorized' if authentication_header?(auth) || config.nil?
+      return true if config.nil?
+      return halt 401, 'Unauthorized' if authentication_header?(auth)
 
       authorized?(config, auth)
     end

lib/models/pull_request.rb

@@ -22,7 +22,7 @@ class PullRequest < ActiveRecord::Base
   def finished?
     return true if check_suites.nil? or check_suites.empty?
 
-    current_execution_by_plan(plan).finished?
+    plans.all? { |plan| current_execution_by_plan(plan)&.finished? }
   end
 
   def current_execution?(check_suite)

lib/models/stage.rb

@@ -16,7 +16,7 @@ class Stage < ActiveRecord::Base
   belongs_to :configuration, class_name: 'StageConfiguration', foreign_key: 'stage_configuration_id'
   belongs_to :check_suite
 
-  default_scope -> { order(id: :asc) }, all_queries: true
+  default_scope -> { order(id: :asc) }
 
   scope :related_stages, lambda { |check_suite, suffix|
                            where('stages.name LIKE ?', "%#{suffix}").where(check_suite: check_suite)
@@ -148,7 +148,7 @@ def output_in_progress
 
   def mount_in_progress_jobs(jobs)
     jobs.where(status: :in_progress).map do |job|
-      "- **#{job.name}** -> https://#{url}/browse/#{job.job_ref}\n"
+      "- **#{job.name}** -> https://#{GitHubApp::Configuration.instance.ci_url}/browse/#{job.job_ref}\n"
     end.join("\n")
   end
 end

spec/lib/helpers/sinatra_payload_spec.rb

@@ -205,7 +205,7 @@ def halt(_, _opt = nil)
       let(:metrics_config) { nil }
 
       it 'returns 401' do
-        expect(dummy.authenticate_metrics).to be_falsey
+        expect(dummy.authenticate_metrics).to be_truthy
       end
     end