Improve ergonomics for regenerating snapshots

Project.toml

@@ -1,7 +1,7 @@
 name = "SnapshotTesting"
 uuid = "cf213a48-8697-4c15-82cb-081f2086cf9e"
 authors = ["Nathan Daly <nhdaly@gmail.com> and contributors"]
-version = "0.1.1"
+version = "0.2.0"
 
 [deps]
 DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6"

src/snapshots.jl

@@ -16,9 +16,17 @@ function create_expectation_snapshot(func, expected_dir, subpath)
     func(snapshot_dir)
 end
 
-function test_snapshot(func, expected_dir, subpath; allow_additions = true, regenerate = false)
-    if regenerate
-        create_expectation_snapshot(func, expected_dir, subpath)
+function test_snapshot(func, expected_dir, subpath; allow_additions = true)
+    # We're testing against the expected files
+    expected_path = joinpath(expected_dir, subpath)
+
+    if !isdir(expected_path)
+        mkpath(expected_path)
+        func(expected_path)
+        @info """Snapshot for \"$subpath\" did not exist. It has been created at:
+        $expected_path
+        """
+        @info "Please run the tests again for any changes to take effect"
         return nothing
     end
 
@@ -29,14 +37,34 @@ function test_snapshot(func, expected_dir, subpath; allow_additions = true, rege
     mkpath(snapshot_dir)
     func(snapshot_dir)
 
-    # Test against the expected files
-    expected_path = joinpath(expected_dir, subpath)
     @testset "$subpath" begin
-        _recursive_diff_dirs(expected_path, snapshot_dir; allow_additions)
+        has_failures = _recursive_diff_dirs(expected_path, snapshot_dir; allow_additions)
+
+        if has_failures
+            if isinteractive() || force_update()
+                if force_update() || input_bool("Replace snapshot with actual result (in $subpath)?")
+                    rm(expected_path; recursive=true, force=true)
+                    cp(snapshot_dir, expected_path; force=true)
+                    @info "Snapshot updated at $expected_path"
+                    @info "Please run the tests again for any changes to take effect"
+                end
+            else
+                @error """
+                Snapshot test failed for \"$subpath\".
+                To update the snapshots either run the tests interactively with 'include(\"test/runtests.jl\")',
+                or to force-update all failing snapshots set the environment variable `JULIA_SNAPSHOTTESTS_UPDATE`
+                to "true" and re-run the tests via Pkg.
+                """
+            end
+        end
     end
 end
-# Diff all the files in the output directory against the expected directory
+
+# Diff all the files in the output directory against the expected directory, returning
+# whether or not we found any failures
 function _recursive_diff_dirs(expected_dir, new_dir; allow_additions)
+    has_failures = false
+
     # Collect new files
     new_files = Set(String[])
     for (root, _, files) in walkdir(new_dir)
@@ -54,6 +82,7 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions)
             subpath = _chopprefix(_chopprefix(expected_path, expected_dir), "/")
             @test subpath in new_files
             if !(subpath in new_files)
+                has_failures = true
                 @error("New snapshot is missing file `$subpath`. Expected contents:\n",
                         expected_content)
             else
@@ -62,6 +91,7 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions)
                 new_content = read(new_path, String)
                 @test new_content == expected_content
                 if new_content != expected_content
+                    has_failures = true
                     println("Found non-matching content in `$file`.")
                     display(DeepDiffs.deepdiff(expected_content, new_content))
                 end
@@ -74,6 +104,7 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions)
     if !allow_additions
         # Report test failures for the new files if requested
         if !isempty(new_files)
+            has_failures = true
             @error("New snapshot contains unexpected files. If this is not an error in your
                 case, pass `allow_additions = true`.")
             for path in new_files
@@ -86,6 +117,8 @@ function _recursive_diff_dirs(expected_dir, new_dir; allow_additions)
             end
         end
     end
+
+    return has_failures
 end
 
 
@@ -102,3 +135,32 @@ function _chopprefix(s::AbstractString, prefix::AbstractString)
         i, j = iterate(s, k), iterate(prefix, j[2])
     end
 end
+
+"""
+    force_update()
+
+Check if the environment variable `JULIA_SNAPSHOTTESTS_UPDATE` is set to "true".
+When true, all failing snapshot tests will automatically update their references.
+"""
+force_update() = tryparse(Bool, get(ENV, "JULIA_SNAPSHOTTESTS_UPDATE", "false")) === true
+
+"""
+    input_bool(prompt)
+
+Display an interactive y/n prompt and return true for 'y', false for 'n'.
+Loops until a valid response is given.
+"""
+function input_bool(prompt)
+    while true
+        println(prompt, " [y/n]")
+        response = readline()
+        length(response) == 0 && continue
+        reply = lowercase(first(strip(response)))
+        if reply == 'y'
+            return true
+        elseif reply == 'n'
+            return false
+        end
+        # Otherwise loop and repeat the prompt
+    end
+end

test/runtests.jl

@@ -5,4 +5,7 @@ using Test
     @testset "snapshots" begin
         include("snapshots.jl")
     end
+    @testset "update modes" begin
+        include("test_update_modes.jl")
+    end
 end

test/test_update_modes.jl

@@ -0,0 +1,113 @@
+using SnapshotTesting
+using Test
+
+@testset "Snapshot Update Modes" begin
+
+    @testset "Environment variable force_update" begin
+        # Test that force_update returns false by default (clear env var first)
+        withenv("JULIA_SNAPSHOTTESTS_UPDATE" => nothing) do
+            @test SnapshotTesting.force_update() == false
+        end
+
+        # Test that force_update returns true when env var is set
+        withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "true") do
+            @test SnapshotTesting.force_update() == true
+        end
+
+        # Test that it returns false for other values
+        withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "false") do
+            @test SnapshotTesting.force_update() == false
+        end
+
+        withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "1") do
+            @test SnapshotTesting.force_update() == true  # "1" parses as true
+        end
+
+        withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "0") do
+            @test SnapshotTesting.force_update() == false  # "0" parses as false
+        end
+
+        withenv("JULIA_SNAPSHOTTESTS_UPDATE" => "invalid") do
+            @test SnapshotTesting.force_update() == false  # invalid string returns false
+        end
+    end
+
+    @testset "Interactive input_bool" begin
+        # Helper to test input_bool with simulated input
+        function test_input(input_string)
+            # Create a temporary file with the input
+            mktempdir() do tmpdir
+                input_file = joinpath(tmpdir, "input.txt")
+                write(input_file, input_string)
+
+                open(input_file, "r") do input_io
+                    redirect_stdin(input_io) do
+                        redirect_stdout(devnull) do
+                            SnapshotTesting.input_bool("Test prompt")
+                        end
+                    end
+                end
+            end
+        end
+
+        # Test 'y' response
+        @test test_input("y\n") == true
+
+        # Test 'n' response
+        @test test_input("n\n") == false
+
+        # Test case insensitivity with uppercase Y
+        @test test_input("Y\n") == true
+
+        # Test case insensitivity with uppercase N
+        @test test_input("N\n") == false
+
+        # Test that invalid input is retried
+        @test test_input("invalid\ny\n") == true
+
+        # Test empty input is retried
+        @test test_input("\n\nn\n") == false
+    end
+
+    @testset "Auto-creation of missing snapshots" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+
+            test_path = joinpath(expected, "autocreate")
+            @test !isdir(test_path)
+
+            # Run test - should auto-create snapshot
+            redirect_stdout(devnull) do
+                SnapshotTesting.test_snapshot(expected, "autocreate") do dir
+                    write(joinpath(dir, "newfile.txt"), "auto-created content")
+                end
+            end
+
+            # Verify snapshot was created
+            @test isdir(test_path)
+            @test isfile(joinpath(test_path, "newfile.txt"))
+            @test read(joinpath(test_path, "newfile.txt"), String) == "auto-created content"
+        end
+    end
+
+    @testset "Successful snapshot test (no changes needed)" begin
+        mktempdir() do tmpdir
+            expected = joinpath(tmpdir, "expected")
+            mkpath(expected)
+
+            # Create initial snapshot
+            test_path = joinpath(expected, "nochange")
+            mkpath(test_path)
+            write(joinpath(test_path, "file.txt"), "same content")
+
+            # Run test with same content - should pass without prompts
+            @testset "matching content" begin
+                SnapshotTesting.test_snapshot(expected, "nochange") do dir
+                    write(joinpath(dir, "file.txt"), "same content")
+                end
+            end
+        end
+    end
+
+end