Add source_type filtering, source linking on status changes, and evidence date fixes

- Add source_type filter to commitments, dashboard, and burndown APIs
- Add source_id to commitment_status_changes with backfill from commitment sources
- Use source document date for status change changed_at instead of extraction date
- Include url and date in all source serializations (events, assessments, revisions, status history)
- Add rake tasks for backfilling, dedup, reevaluation, and database backup
- Update criterion assessor and status deriver models

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: xrendan

.claude/settings.local.json

@@ -6,8 +6,9 @@
       "Bash(git push:*)",
       "Bash(curl:*)",
       "Bash(mkdir:*)",
-      "Bash(grep:*)"
+      "Bash(grep:*)",
+      "Bash(gem pristine:*)"
     ],
     "deny": []
   }
-}
+}

.ruby-version

@@ -1 +1 @@
-ruby-3.4.4
+ruby-3.4.7

Procfile.dev

@@ -0,0 +1,2 @@
+web: bin/rails server -p ${PORT:-3000}
+worker: bundle exec good_job start --max-threads=15

app/controllers/api/burndown_controller.rb

@@ -1,12 +1,17 @@
 module Api
   class BurndownController < ApplicationController
-    STARTED_STATUSES = %w[in_progress partially_implemented implemented abandoned].freeze
-    COMPLETED_STATUSES = %w[implemented].freeze
+    STARTED_STATUSES = %w[in_progress completed abandoned].freeze
+    COMPLETED_STATUSES = %w[completed].freeze
+    ABANDONED_STATUSES = %w[abandoned].freeze
 
     def show
       government = Government.find(params[:government_id])
       commitments = government.commitments
 
+      if params[:source_type].present?
+        commitments = commitments.joins(:sources).where(sources: { source_type: params[:source_type] }).distinct
+      end
+
       policy_area = nil
       if params[:policy_area_slug].present?
         policy_area = PolicyArea.find_by!(slug: params[:policy_area_slug])
@@ -38,46 +43,65 @@ def show
         first_change_by_commitment[sc.commitment_id] ||= sc.previous_status
       end
 
-      # For commitments already in a started status without status_change records,
-      # use their earliest evidence date instead of date_promised
-      earliest_evidence_dates = earliest_evidence_dates_for(commitment_ids)
+      # Gather all real-world evidence dates per commitment
+      # (from CommitmentMatch → Bill/StatcanDataset and from CriterionAssessment → Source)
+      all_evidence_dates = evidence_dates_for(commitment_ids)
 
-      # Build sorted events
+      # Build sorted events, tracking effective state per commitment
+      # to avoid double-counting from duplicate status change records
       events = []
+      effective_state = {}
 
       commitment_records.each do |c|
         initial = first_change_by_commitment[c.id] || c.status
+        effective_state[c.id] = initial
         scope_date = c.date_promised || c.created_at.to_date
 
         # Scope always enters on date_promised
-        events << { date: scope_date, delta_scope: 1, delta_started: 0, delta_completed: 0 }
-
-        # Started/completed enter on evidence date (if no status_change drove it)
-        next if first_change_by_commitment.key?(c.id)
+        events << { date: scope_date, delta_scope: 1, delta_started: 0, delta_completed: 0, delta_abandoned: 0 }
 
+        # Add initial state event for all commitments that start in a non-default state
         if STARTED_STATUSES.include?(initial)
-          evidence_date = earliest_evidence_dates[c.id] || scope_date
+          evidence_date = all_evidence_dates[c.id]&.min || scope_date
           events << { date: evidence_date, delta_scope: 0, delta_started: 1,
-                      delta_completed: COMPLETED_STATUSES.include?(initial) ? 1 : 0 }
+                      delta_completed: COMPLETED_STATUSES.include?(initial) ? 1 : 0,
+                      delta_abandoned: ABANDONED_STATUSES.include?(initial) ? 1 : 0 }
         end
       end
 
       status_changes.each do |sc|
-        ds = 0
-        dc = 0
+        # Use effective state to calculate deltas, skipping duplicate transitions
+        current = effective_state[sc.commitment_id]
+        next if sc.new_status == current
 
-        was_started = STARTED_STATUSES.include?(sc.previous_status)
+        was_started = STARTED_STATUSES.include?(current)
         now_started = STARTED_STATUSES.include?(sc.new_status)
+        ds = 0
         ds = 1 if now_started && !was_started
         ds = -1 if !now_started && was_started
 
-        was_completed = COMPLETED_STATUSES.include?(sc.previous_status)
+        was_completed = COMPLETED_STATUSES.include?(current)
         now_completed = COMPLETED_STATUSES.include?(sc.new_status)
+        dc = 0
         dc = 1 if now_completed && !was_completed
         dc = -1 if !now_completed && was_completed
 
-        events << { date: sc.changed_at.to_date, delta_scope: 0,
-                    delta_started: ds, delta_completed: dc }
+        was_abandoned = ABANDONED_STATUSES.include?(current)
+        now_abandoned = ABANDONED_STATUSES.include?(sc.new_status)
+        da = 0
+        da = 1 if now_abandoned && !was_abandoned
+        da = -1 if !now_abandoned && was_abandoned
+
+        # Use the latest real-world evidence date before the job ran,
+        # falling back to the job run date if no evidence dates exist
+        job_date = sc.changed_at.to_date
+        dates = all_evidence_dates[sc.commitment_id] || []
+        event_date = dates.select { |d| d <= job_date }.max || dates.min || job_date
+
+        events << { date: event_date, delta_scope: 0,
+                    delta_started: ds, delta_completed: dc, delta_abandoned: da }
+
+        effective_state[sc.commitment_id] = sc.new_status
       end
 
       events.sort_by! { |e| e[:date] }
@@ -86,24 +110,26 @@ def show
       scope = 0
       started = 0
       completed = 0
+      abandoned = 0
       series = []
       current_date = nil
 
       events.each do |e|
         if current_date && e[:date] != current_date
-          series << { date: current_date.iso8601, scope: scope, started: started, completed: completed }
+          series << { date: current_date.iso8601, scope: scope, started: started, completed: completed, abandoned: abandoned }
         end
         current_date = e[:date]
         scope += e[:delta_scope]
         started += e[:delta_started]
         completed += e[:delta_completed]
+        abandoned += e[:delta_abandoned]
       end
 
-      series << { date: current_date.iso8601, scope: scope, started: started, completed: completed } if current_date
+      series << { date: current_date.iso8601, scope: scope, started: started, completed: completed, abandoned: abandoned } if current_date
 
       # Also emit today if last event was before today
       if current_date && current_date < Date.current
-        series << { date: Date.current.iso8601, scope: scope, started: started, completed: completed }
+        series << { date: Date.current.iso8601, scope: scope, started: started, completed: completed, abandoned: abandoned }
       end
 
       render json: {
@@ -119,21 +145,28 @@ def show
 
     private
 
-    def earliest_evidence_dates_for(commitment_ids)
-      # Find the earliest real-world date from each commitment's matched evidence.
-      # For Bills: use the earliest milestone date (first reading, latest_activity, etc.)
-      # For StatcanDatasets: use last_synced_at
-      matches = CommitmentMatch.where(commitment_id: commitment_ids).includes(:matchable)
+    def evidence_dates_for(commitment_ids)
+      result = Hash.new { |h, k| h[k] = [] }
 
-      result = {}
-      matches.each do |cm|
+      # Real-world dates from matched evidence (Bills, StatcanDatasets)
+      CommitmentMatch.where(commitment_id: commitment_ids).includes(:matchable).each do |cm|
         date = evidence_date_for(cm.matchable)
-        next unless date
-
-        existing = result[cm.commitment_id]
-        result[cm.commitment_id] = date if existing.nil? || date < existing
+        result[cm.commitment_id] << date if date
       end
 
+      # Real-world dates from criterion assessment sources
+      CriterionAssessment
+        .joins(criterion: :commitment)
+        .joins("LEFT JOIN sources ON sources.id = criterion_assessments.source_id")
+        .where(criteria: { commitment_id: commitment_ids })
+        .where.not(sources: { date: nil })
+        .pluck("criteria.commitment_id", "sources.date")
+        .each do |cid, date|
+          result[cid] << date
+        end
+
+      # Deduplicate
+      result.each_value(&:uniq!)
       result
     end
 
@@ -146,6 +179,8 @@ def evidence_date_for(matchable)
         ].compact.min&.to_date
       when StatcanDataset
         matchable.last_synced_at&.to_date
+      when Entry
+        matchable.published_at&.to_date
       end
     end
   end

app/controllers/api/dashboard_controller.rb

@@ -3,10 +3,17 @@ class DashboardController < ApplicationController
     def at_a_glance
       government = Government.find(params[:government_id])
 
-      policy_areas = PolicyArea.ordered.includes(commitments: :lead_department).where(commitments: { government_id: government.id }).distinct
+      base_scope = government.commitments
+      if params[:source_type].present?
+        base_scope = base_scope.joins(:sources).where(sources: { source_type: params[:source_type] }).distinct
+      end
+
+      base_commitment_ids = base_scope.pluck(:id)
+
+      policy_areas = PolicyArea.ordered.includes(commitments: :lead_department).where(commitments: { id: base_commitment_ids }).distinct
 
       data = policy_areas.map do |pa|
-        area_commitments = pa.commitments.where(government_id: government.id)
+        area_commitments = pa.commitments.where(id: base_commitment_ids)
         {
           id: pa.id,
           name: pa.name,
@@ -25,7 +32,7 @@ def at_a_glance
         }
       end
 
-      unassigned = government.commitments.where(policy_area_id: nil)
+      unassigned = Commitment.where(id: base_commitment_ids, policy_area_id: nil)
       if unassigned.any?
         data << {
           id: nil,
@@ -45,23 +52,21 @@ def at_a_glance
         }
       end
 
-      all_commitments = government.commitments
+      all_commitments = Commitment.where(id: base_commitment_ids)
       total = all_commitments.count
       status_counts = all_commitments.group(:status).count
 
       not_started = status_counts.fetch("not_started", 0)
-      implemented = status_counts.fetch("implemented", 0)
-      in_progress = status_counts.fetch("in_progress", 0) +
-                    status_counts.fetch("partially_implemented", 0)
-      successful = all_commitments.joins(:success_criteria).merge(Criterion.where(status: :met)).distinct.count
+      completed = status_counts.fetch("completed", 0)
+      in_progress = status_counts.fetch("in_progress", 0)
 
       render json: {
         government: { id: government.id, name: government.name },
         total_commitments: total,
         summary: {
           not_started: { count: not_started, label: "Not Started", subtitle: "no action taken" },
-          completed: { count: implemented, label: "Completed", subtitle: "of #{total} commitments" },
-          successful: { count: successful, label: "Successful", subtitle: "meeting success criteria" }
+          completed: { count: completed, label: "Completed", subtitle: "of #{total} commitments" },
+          in_progress: { count: in_progress, label: "In Progress", subtitle: "action taken" }
         },
         status_counts: status_counts,
         policy_areas: data

app/controllers/commitments_controller.rb

@@ -22,6 +22,10 @@ def index
       @commitments = @commitments.joins(:lead_commitment_department).joins("INNER JOIN departments AS lead_depts ON lead_depts.id = commitment_departments.department_id").where(lead_depts: { slug: params[:lead_department] })
     end
 
+    if params[:source_type].present?
+      @commitments = @commitments.joins(:sources).where(sources: { source_type: params[:source_type] }).distinct
+    end
+
     @commitments = apply_sorting(@commitments)
     @total_count = @commitments.count
     @commitments = @commitments.limit(page_size).offset(page_offset)
@@ -48,7 +52,7 @@ def sort_direction
   end
 
   def page_size
-    [(params[:per_page] || 50).to_i, 100].min
+    [(params[:per_page] || 50).to_i, 1000].min
   end
 
   def page_offset

app/jobs/commitment_assessment_job.rb

@@ -10,6 +10,7 @@ def perform(commitment)
     return if unassessed_matches.empty?
 
     evidence_items = unassessed_matches.map(&:matchable).compact
+    latest_evidence_date = evidence_items.filter_map { |m| evidence_date_for(m) }.max || Time.current
 
     commitment.criteria.find_each do |criterion|
       assessor = CriterionAssessor.create!(record: criterion)
@@ -23,18 +24,28 @@ def perform(commitment)
         previous_status: criterion.status,
         new_status: new_status,
         evidence_notes: assessor.assessment["evidence_notes"],
-        assessed_at: Time.current
+        assessed_at: latest_evidence_date
       )
 
       criterion.update!(
         status: new_status,
         evidence_notes: assessor.assessment["evidence_notes"],
-        assessed_at: Time.current
+        assessed_at: latest_evidence_date
       )
     end
 
-    unassessed_matches.update_all(assessed: true, assessed_at: Time.current)
-    commitment.update!(last_assessed_at: Time.current)
+    unassessed_matches.update_all(assessed: true, assessed_at: latest_evidence_date)
+    commitment.update!(last_assessed_at: latest_evidence_date)
     CommitmentStatusDerivationJob.perform_later(commitment)
   end
+
+  private
+
+  def evidence_date_for(matchable)
+    case matchable
+    when Entry then matchable.published_at
+    when Bill then [matchable.passed_house_first_reading_at, matchable.latest_activity_at].compact.max
+    when StatcanDataset then matchable.last_synced_at
+    end
+  end
 end

app/jobs/commitment_dedup_job.rb

@@ -2,12 +2,16 @@ class CommitmentDedupJob < ApplicationJob
   queue_as :default
 
   def perform(government)
-    commitments = government.commitments.where.not(status: :abandoned).order(:id)
+    commitments = government.commitments.includes(:lead_department).where.not(status: :abandoned).order(:id)
     return if commitments.size < 2
 
     Rails.logger.info("CommitmentDedupJob: Checking #{commitments.size} commitments for duplicates")
 
-    commitments.each_slice(50) do |batch|
+    commitments.group_by(&:lead_department).each do |department, batch|
+      next if batch.size < 2
+
+      Rails.logger.info("CommitmentDedupJob: Checking #{batch.size} commitments for #{department&.display_name || 'unassigned'}")
+
       finder = CommitmentDedupFinder.create!(record: government)
       finder.extract!(finder.prompt(batch))
 
@@ -37,11 +41,6 @@ def merge_commitment(keep, duplicate, reason)
     duplicate.commitment_departments.where.not(department_id: keep.commitment_departments.select(:department_id)).update_all(commitment_id: keep.id)
     duplicate.commitment_departments.where(department_id: keep.commitment_departments.select(:department_id)).delete_all
 
-    if duplicate.respond_to?(:superseded_by_id)
-      duplicate.update_columns(superseded_by_id: keep.id)
-    end
-
-    duplicate.abandonment_reason = "Duplicate of commitment ##{keep.id}: #{reason}"
-    duplicate.update!(status: :abandoned)
+    duplicate.destroy!
   end
 end

app/jobs/commitment_relevance_filter_job.rb

@@ -22,7 +22,7 @@ def perform(matchable)
         ) do |cm|
           cm.relevance_score = match_data["relevance_score"]
           cm.relevance_reasoning = match_data["relevance_reasoning"]
-          cm.matched_at = Time.current
+          cm.matched_at = evidence_date_for(matchable) || Time.current
         end
       end
     end
@@ -49,6 +49,7 @@ def evidence_date_for(matchable)
     case matchable
     when Entry then matchable.published_at&.to_date
     when Bill then matchable.passed_house_first_reading_at&.to_date
+    when StatcanDataset then matchable.last_synced_at&.to_date
     end
   end
 

app/jobs/commitment_status_derivation_job.rb

@@ -22,13 +22,8 @@ def perform(commitment)
       "from #{commitment.status} to #{recommended} (confidence: #{confidence}) — #{reasoning}"
     )
 
+    evidence_date = commitment.criteria.maximum(:assessed_at)
+    commitment.status_changed_at = evidence_date
     commitment.update!(status: recommended)
-
-    commitment.status_changes.create!(
-      previous_status: commitment.status_before_last_save,
-      new_status: recommended,
-      changed_at: Time.current,
-      reason: "AI-derived: #{reasoning}"
-    )
   end
 end