WIP: PG sequence bug repro + Postgres/MySQL CI matrix (#51)

.github/workflows/ci.yml

@@ -34,6 +34,68 @@ jobs:
       - name: Run tests
         run: bundle exec rspec
 
+  test_db:
+    runs-on: ubuntu-latest
+    name: test (${{ matrix.appraisal }}, ${{ matrix.database }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["4.0"]
+        appraisal: ["rails_8.1"]
+        database: ["postgresql", "mysql2"]
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mysql:
+        image: mysql:8
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd "mysqladmin ping -h localhost"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      BUNDLE_GEMFILE: gemfiles/${{ matrix.appraisal }}.gemfile
+      FIXTURE_KIT_INTEGRATION_FRAMEWORK: rspec
+      FIXTURE_KIT_DB: ${{ matrix.database }}
+      FIXTURE_KIT_DB_HOST: ${{ matrix.database == 'mysql2' && '127.0.0.1' || 'localhost' }}
+      FIXTURE_KIT_DB_USERNAME: ${{ matrix.database == 'mysql2' && 'root' || 'postgres' }}
+      FIXTURE_KIT_DB_PASSWORD: ${{ matrix.database == 'mysql2' && 'root' || 'postgres' }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Install dummy app dependencies
+        working-directory: spec/dummy
+        env:
+          BUNDLE_GEMFILE: ""
+        run: bundle install
+
+      - name: Run tests
+        run: bundle exec rspec
+
   lint:
     runs-on: ubuntu-latest
 

Appraisals

@@ -2,10 +2,26 @@ appraise "rails-8.0" do
   gem "activesupport", "~> 8.0.0"
   gem "activerecord", "~> 8.0.0"
   gem "railties", "~> 8.0.0"
+
+  group :postgres do
+    gem "pg"
+  end
+
+  group :mysql do
+    gem "mysql2"
+  end
 end
 
 appraise "rails-8.1" do
   gem "activesupport", "~> 8.1.0"
   gem "activerecord", "~> 8.1.0"
   gem "railties", "~> 8.1.0"
+
+  group :postgres do
+    gem "pg"
+  end
+
+  group :mysql do
+    gem "mysql2"
+  end
 end

Gemfile

@@ -3,3 +3,11 @@
 source "https://rubygems.org"
 
 gemspec
+
+group :postgres do
+  gem "pg"
+end
+
+group :mysql do
+  gem "mysql2"
+end

gemfiles/rails_8.0.gemfile

@@ -6,4 +6,12 @@ gem "activesupport", "~> 8.0.0"
 gem "activerecord", "~> 8.0.0"
 gem "railties", "~> 8.0.0"
 
+group :postgres do
+  gem "pg"
+end
+
+group :mysql do
+  gem "mysql2"
+end
+
 gemspec path: "../"

gemfiles/rails_8.0.gemfile.lock

@@ -74,6 +74,8 @@ GEM
       nokogiri (>= 1.12.0)
     minitest (6.0.1)
       prism (~> 1.5)
+    mysql2 (0.5.7)
+      bigdecimal
     nokogiri (1.19.1-aarch64-linux-gnu)
       racc (~> 1.4)
     nokogiri (1.19.1-aarch64-linux-musl)
@@ -90,6 +92,13 @@ GEM
       racc (~> 1.4)
     nokogiri (1.19.1-x86_64-linux-musl)
       racc (~> 1.4)
+    pg (1.6.3)
+    pg (1.6.3-aarch64-linux)
+    pg (1.6.3-aarch64-linux-musl)
+    pg (1.6.3-arm64-darwin)
+    pg (1.6.3-x86_64-darwin)
+    pg (1.6.3-x86_64-linux)
+    pg (1.6.3-x86_64-linux-musl)
     pp (0.6.3)
       prettyprint
     prettyprint (0.2.0)
@@ -181,6 +190,8 @@ DEPENDENCIES
   appraisal
   fixture_kit!
   irb
+  mysql2
+  pg
   railties (~> 8.0.0)
   rake
   rspec-rails
@@ -212,6 +223,7 @@ CHECKSUMS
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   loofah (2.25.0) sha256=df5ed7ac3bac6a4ec802df3877ee5cc86d027299f8952e6243b3dac446b060e6
   minitest (6.0.1) sha256=7854c74f48e2e975969062833adc4013f249a4b212f5e7b9d5c040bf838d54bb
+  mysql2 (0.5.7) sha256=ba09ede515a0ae8a7192040a1b778c0fb0f025fa5877e9be895cd325fa5e9d7b
   nokogiri (1.19.1-aarch64-linux-gnu) sha256=cfdb0eafd9a554a88f12ebcc688d2b9005f9fce42b00b970e3dc199587b27f32
   nokogiri (1.19.1-aarch64-linux-musl) sha256=1e2150ab43c3b373aba76cd1190af7b9e92103564063e48c474f7600923620b5
   nokogiri (1.19.1-arm-linux-gnu) sha256=0a39ed59abe3bf279fab9dd4c6db6fe8af01af0608f6e1f08b8ffa4e5d407fa3
@@ -220,6 +232,13 @@ CHECKSUMS
   nokogiri (1.19.1-x86_64-darwin) sha256=7093896778cc03efb74b85f915a775862730e887f2e58d6921e3fa3d981e68bf
   nokogiri (1.19.1-x86_64-linux-gnu) sha256=1a4902842a186b4f901078e692d12257678e6133858d0566152fe29cdb98456a
   nokogiri (1.19.1-x86_64-linux-musl) sha256=4267f38ad4fc7e52a2e7ee28ed494e8f9d8eb4f4b3320901d55981c7b995fc23
+  pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99
+  pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea
+  pg (1.6.3-aarch64-linux-musl) sha256=06a75f4ea04b05140146f2a10550b8e0d9f006a79cdaf8b5b130cde40e3ecc2c
+  pg (1.6.3-arm64-darwin) sha256=7240330b572e6355d7c75a7de535edb5dfcbd6295d9c7777df4d9dddfb8c0e5f
+  pg (1.6.3-x86_64-darwin) sha256=ee2e04a17c0627225054ffeb43e31a95be9d7e93abda2737ea3ce4a62f2729d6
+  pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d
+  pg (1.6.3-x86_64-linux-musl) sha256=9c9c90d98c72f78eb04c0f55e9618fe55d1512128e411035fe229ff427864009
   pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6
   prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85

gemfiles/rails_8.1.gemfile

@@ -6,4 +6,12 @@ gem "activesupport", "~> 8.1.0"
 gem "activerecord", "~> 8.1.0"
 gem "railties", "~> 8.1.0"
 
+group :postgres do
+  gem "pg"
+end
+
+group :mysql do
+  gem "mysql2"
+end
+
 gemspec path: "../"

gemfiles/rails_8.1.gemfile.lock

@@ -75,6 +75,8 @@ GEM
       nokogiri (>= 1.12.0)
     minitest (6.0.1)
       prism (~> 1.5)
+    mysql2 (0.5.7)
+      bigdecimal
     nokogiri (1.19.1-aarch64-linux-gnu)
       racc (~> 1.4)
     nokogiri (1.19.1-aarch64-linux-musl)
@@ -91,6 +93,13 @@ GEM
       racc (~> 1.4)
     nokogiri (1.19.1-x86_64-linux-musl)
       racc (~> 1.4)
+    pg (1.6.3)
+    pg (1.6.3-aarch64-linux)
+    pg (1.6.3-aarch64-linux-musl)
+    pg (1.6.3-arm64-darwin)
+    pg (1.6.3-x86_64-darwin)
+    pg (1.6.3-x86_64-linux)
+    pg (1.6.3-x86_64-linux-musl)
     pp (0.6.3)
       prettyprint
     prettyprint (0.2.0)
@@ -182,6 +191,8 @@ DEPENDENCIES
   appraisal
   fixture_kit!
   irb
+  mysql2
+  pg
   railties (~> 8.1.0)
   rake
   rspec-rails
@@ -214,6 +225,7 @@ CHECKSUMS
   logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
   loofah (2.25.0) sha256=df5ed7ac3bac6a4ec802df3877ee5cc86d027299f8952e6243b3dac446b060e6
   minitest (6.0.1) sha256=7854c74f48e2e975969062833adc4013f249a4b212f5e7b9d5c040bf838d54bb
+  mysql2 (0.5.7) sha256=ba09ede515a0ae8a7192040a1b778c0fb0f025fa5877e9be895cd325fa5e9d7b
   nokogiri (1.19.1-aarch64-linux-gnu) sha256=cfdb0eafd9a554a88f12ebcc688d2b9005f9fce42b00b970e3dc199587b27f32
   nokogiri (1.19.1-aarch64-linux-musl) sha256=1e2150ab43c3b373aba76cd1190af7b9e92103564063e48c474f7600923620b5
   nokogiri (1.19.1-arm-linux-gnu) sha256=0a39ed59abe3bf279fab9dd4c6db6fe8af01af0608f6e1f08b8ffa4e5d407fa3
@@ -222,6 +234,13 @@ CHECKSUMS
   nokogiri (1.19.1-x86_64-darwin) sha256=7093896778cc03efb74b85f915a775862730e887f2e58d6921e3fa3d981e68bf
   nokogiri (1.19.1-x86_64-linux-gnu) sha256=1a4902842a186b4f901078e692d12257678e6133858d0566152fe29cdb98456a
   nokogiri (1.19.1-x86_64-linux-musl) sha256=4267f38ad4fc7e52a2e7ee28ed494e8f9d8eb4f4b3320901d55981c7b995fc23
+  pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99
+  pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea
+  pg (1.6.3-aarch64-linux-musl) sha256=06a75f4ea04b05140146f2a10550b8e0d9f006a79cdaf8b5b130cde40e3ecc2c
+  pg (1.6.3-arm64-darwin) sha256=7240330b572e6355d7c75a7de535edb5dfcbd6295d9c7777df4d9dddfb8c0e5f
+  pg (1.6.3-x86_64-darwin) sha256=ee2e04a17c0627225054ffeb43e31a95be9d7e93abda2737ea3ce4a62f2729d6
+  pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d
+  pg (1.6.3-x86_64-linux-musl) sha256=9c9c90d98c72f78eb04c0f55e9618fe55d1512128e411035fe229ff427864009
   pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6
   prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
   prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85

lib/fixture_kit/coders/active_record_coder.rb

@@ -15,7 +15,10 @@ def generate(parent_data: nil, &block)
         model_name = name[NAME_PATTERN, :model_name]
         next unless model_name
 
-        captured_models.add(ActiveSupport::Inflector.constantize(model_name))
+        klass = ActiveSupport::Inflector.safe_constantize(model_name)
+        next unless klass.is_a?(Class) && klass < ActiveRecord::Base
+
+        captured_models.add(klass)
       end
 
       ActiveSupport::Notifications.subscribed(subscriber, EVENT, monotonic: true, &block)
@@ -27,11 +30,25 @@ def generate(parent_data: nil, &block)
     end
 
     def mount(data)
-      statements_by_connection(data).each do |connection, statements|
-        connection.disable_referential_integrity do
-          # execute_batch is private in current supported Rails versions.
-          # This should be revisited when Rails 8.2 makes it public.
-          connection.__send__(:execute_batch, statements, "FixtureKit Load")
+      models_by_pool(data).each do |pool, models|
+        pool.with_connection do |connection|
+          statements = models.flat_map do |model|
+            [build_delete_sql(connection, model.table_name), data[model]].compact
+          end
+
+          connection.disable_referential_integrity do
+            # execute_batch is private in current supported Rails versions.
+            # This should be revisited when Rails 8.2 makes it public.
+            connection.__send__(:execute_batch, statements, "FixtureKit Insert")
+          end
+
+          verify_foreign_keys!(connection)
+
+          # Replayed INSERTs use explicit PKs, which Postgres sequences do not
+          # observe. Re-sync the sequence so subsequent Model.create calls don't
+          # collide with an id we just inserted. No-op on adapters whose PK
+          # generators advance from explicit-id INSERTs (MySQL, SQLite).
+          reset_primary_key_sequences(connection, models.map(&:table_name))
         end
       end
     end
@@ -51,24 +68,30 @@ def base_table_model(model)
 
     def generate_statements(models)
       models.each_with_object({}) do |model, statements|
-        columns = model.column_names
+        columns = insertable_columns(model)
+        column_names = columns.map(&:name)
 
         rows = []
         model.unscoped.order(:id).find_each do |record|
-          row_values = columns.map do |col|
+          row_values = column_names.map do |col|
             value = record.read_attribute_before_type_cast(col)
             model.connection.quote(value)
           end
           rows << "(#{row_values.join(", ")})"
         end
 
-        sql = rows.empty? ? nil : build_insert_sql(model.table_name, columns, rows, model.connection)
+        sql = rows.empty? ? nil : build_insert_sql(model.table_name, column_names, rows, model.connection)
         statements[model] = sql
       end
     end
 
-    def build_delete_sql(model)
-      "DELETE FROM #{model.quoted_table_name}"
+    def insertable_columns(model)
+      supports_virtual = model.connection.supports_virtual_columns?
+      model.columns.reject { |c| supports_virtual && c.virtual? }
+    end
+
+    def build_delete_sql(connection, table_name)
+      "DELETE FROM #{connection.quote_table_name(table_name)}"
     end
 
     def build_insert_sql(table_name, columns, rows, connection)
@@ -78,19 +101,38 @@ def build_insert_sql(table_name, columns, rows, connection)
       "INSERT INTO #{quoted_table} (#{quoted_columns.join(", ")}) VALUES #{rows.join(", ")}"
     end
 
-    def statements_by_connection(records)
-      deleted_tables = Set.new
+    def verify_foreign_keys!(connection)
+      return unless ActiveRecord.verify_foreign_keys_for_fixtures
 
-      records.each_with_object({}) do |(model, sql), grouped|
-        connection = model.connection
-        grouped[connection] ||= []
+      begin
+        connection.check_all_foreign_keys_valid!
+      rescue ActiveRecord::StatementInvalid => e
+        raise FixtureKit::Error,
+          "Foreign key violations found in cached fixture data. The cache may be " \
+          "stale relative to your current schema or fixture definitions. " \
+          "Original error:\n\n#{e.message}"
+      end
+    end
 
-        table_key = [connection, model.table_name]
-        if deleted_tables.add?(table_key)
-          grouped[connection] << build_delete_sql(model)
-        end
+    def reset_primary_key_sequences(connection, tables)
+      # Rails main (>= 8.2) batches the reset in one round-trip per connection.
+      # Older versions fall back to one query per table.
+      if connection.respond_to?(:reset_column_sequences!)
+        connection.reset_column_sequences!(tables.map { |t| [t] })
+      elsif connection.respond_to?(:reset_pk_sequence!)
+        tables.each { |t| connection.reset_pk_sequence!(t) }
+      end
+    end
+
+    def models_by_pool(data)
+      seen = Set.new
+
+      data.each_with_object({}) do |(model, _), grouped|
+        pool = model.connection_pool
+        next unless seen.add?([pool, model.table_name])
 
-        grouped[connection] << sql if sql
+        grouped[pool] ||= []
+        grouped[pool] << model
       end
     end
   end

spec/dummy/Gemfile

@@ -3,3 +3,11 @@
 source "https://rubygems.org"
 
 gemspec path: "../.."
+
+group :postgres do
+  gem "pg"
+end
+
+group :mysql do
+  gem "mysql2"
+end