test: adds system tests for performing tardiff operations using the cli tools on test tar files

Makefile

@@ -61,10 +61,15 @@ tools: .install.golangci-lint
 clean:
 	rm -f tar-diff tar-patch
 	rm -rf $(PROJECT)_$(VERSION) $(PROJ_TARBALL)
-	rm -rf $(GOCOVERDIR)
+	rm -rf $(dir $(GOCOVERDIR))
 
 integration-test: build
 	GOCOVERDIR=$(GOCOVERDIR) tests/test.sh
+	GOCOVERDIR=$(GOCOVERDIR) tests/test-multi-old.sh
+	GOCOVERDIR=$(GOCOVERDIR) tests/test-source-prefix.sh
+	GOCOVERDIR=$(GOCOVERDIR) tests/test-delta-paths.sh
+	GOCOVERDIR=$(GOCOVERDIR) tests/test-tar-errors.sh
+	GOCOVERDIR=$(GOCOVERDIR) tests/test-fuzzy-abs.sh
 	go tool covdata percent -i=$(GOCOVERDIR) -o=$(GOCOVERDIR)/integration.out
 
 

tests/test-delta-paths.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -e
+
+source tests/utils.sh
+
+use_gnu_tar_if_available
+
+WORKDIR=$(mktemp -d /tmp/test-tardiff-delta-paths-XXXXXX)
+
+# Part A: rollsum + copyRest
+TEST_DIR="$WORKDIR/rollsum"
+BIG_BYTES=$((3 * 1024 * 1024))
+OTHER_BYTES=$((2 * 1024 * 1024))
+
+mkdir -p "$TEST_DIR/orig/data" "$TEST_DIR/modified/data"
+
+head -c "$BIG_BYTES" /dev/zero >"$TEST_DIR/orig/data/big.bin"
+cp -a "$TEST_DIR/orig/data/big.bin" "$TEST_DIR/modified/data/big.bin"
+printf 'ROLLSUM_INTEGRATION' | dd of="$TEST_DIR/modified/data/big.bin" bs=1 seek=1500000 conv=notrunc status=none 2>/dev/null || \
+	printf 'ROLLSUM_INTEGRATION' | dd of="$TEST_DIR/modified/data/big.bin" bs=1 seek=1500000 conv=notrunc 2>/dev/null
+
+head -c "$OTHER_BYTES" /dev/zero >"$TEST_DIR/orig/data/other.bin"
+head -c "$OTHER_BYTES" /dev/zero | LC_ALL=C tr '\000' '\377' >"$TEST_DIR/modified/data/other.bin"
+
+create_tar "$TEST_DIR/old.tar" "$TEST_DIR/orig"
+create_tar "$TEST_DIR/new.tar" "$TEST_DIR/modified"
+
+echo "Generating tardiff A (-max-bsdiff-size 1: rollsum + copyRest)"
+./tar-diff -max-bsdiff-size 1 "$TEST_DIR/old.tar.gz" "$TEST_DIR/new.tar.bz2" "$TEST_DIR/changes.tardiff"
+
+echo "Applying tardiff A"
+./tar-patch "$TEST_DIR/changes.tardiff" "$TEST_DIR/orig" "$TEST_DIR/reconstructed.tar"
+
+echo "Verifying A"
+cmp "$TEST_DIR/reconstructed.tar" "$TEST_DIR/new.tar"
+
+echo OK rollsum-and-copyRest
+
+# Part B: bsdiff + qsufsort/split (no low max-bsdiff-size)
+BS="$WORKDIR/bsdiff"
+BSDIFF_BYTES=$((512 * 1024))
+
+mkdir -p "$BS/orig/data" "$BS/modified/data"
+if ! head -c "$BSDIFF_BYTES" /dev/urandom >"$BS/orig/data/large.bin" 2>/dev/null; then
+	head -c "$BSDIFF_BYTES" /dev/zero >"$BS/orig/data/large.bin"
+fi
+cp -a "$BS/orig/data/large.bin" "$BS/modified/data/large.bin"
+printf 'PATCH_BSDIFF_INTEGRATION_0123456789' | dd of="$BS/modified/data/large.bin" bs=1 seek=200000 conv=notrunc status=none 2>/dev/null || \
+	printf 'PATCH_BSDIFF_INTEGRATION_0123456789' | dd of="$BS/modified/data/large.bin" bs=1 seek=200000 conv=notrunc 2>/dev/null
+
+create_tar "$BS/old.tar" "$BS/orig"
+create_tar "$BS/new.tar" "$BS/modified"
+
+echo "Generating tardiff B (default max-bsdiff: medium file → bsdiff / qsufsort)"
+./tar-diff "$BS/old.tar.gz" "$BS/new.tar.bz2" "$BS/changes.tardiff"
+
+echo "Applying tardiff B"
+./tar-patch "$BS/changes.tardiff" "$BS/orig" "$BS/reconstructed.tar"
+
+echo "Verifying B"
+cmp "$BS/reconstructed.tar" "$BS/new.tar"
+
+echo OK bsdiff-large-old
+
+cleanup () {
+	rm -rf "$WORKDIR"
+}
+trap cleanup EXIT

tests/test-fuzzy-abs.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+source tests/utils.sh
+
+use_gnu_tar_if_available
+
+TEST_DIR=$(mktemp -d /tmp/test-tardiff-fuzzy-abs-XXXXXX)
+
+mkdir -p "$TEST_DIR/orig/data"
+printf 'aaaaaaaaaa' >"$TEST_DIR/orig/data/foo.a"
+head -c 50 /dev/zero | tr '\0' 'b' >"$TEST_DIR/orig/data/foo.b"
+
+mkdir -p "$TEST_DIR/modified/data"
+printf 'cccccccccccccccccccc' >"$TEST_DIR/modified/data/foo.c"
+
+create_tar "$TEST_DIR/old.tar" "$TEST_DIR/orig"
+create_tar "$TEST_DIR/new.tar" "$TEST_DIR/modified"
+
+echo "Generating tardiff (fuzzy rename: foo.a + foo.b -> foo.c)"
+./tar-diff "$TEST_DIR/old.tar.gz" "$TEST_DIR/new.tar.bz2" "$TEST_DIR/changes.tardiff"
+
+echo "Applying tardiff"
+./tar-patch "$TEST_DIR/changes.tardiff" "$TEST_DIR/orig" "$TEST_DIR/reconstructed.tar"
+
+echo "Verifying reconstruction"
+cmp "$TEST_DIR/reconstructed.tar" "$TEST_DIR/new.tar"
+
+echo OK fuzzy-abs
+
+cleanup () {
+	rm -rf "$TEST_DIR"
+}
+trap cleanup EXIT

tests/test-multi-old.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -e
+
+source tests/utils.sh
+
+use_gnu_tar_if_available
+
+# Usage: tests/test-multi-old.sh [N]
+#        N defaults to MULTI_OLD_N or 3. Example: MULTI_OLD_N=6 tests/test-multi-old.sh
+N="${1:-${MULTI_OLD_N:-3}}"
+
+if [[ "$N" =~ ^[0-9]+$ ]] && ((N >= 2)); then
+	:
+else
+	echo "usage: $0 [N]  (integer N >= 2, or set MULTI_OLD_N)" >&2
+	exit 1
+fi
+
+TEST_DIR=$(mktemp -d /tmp/test-tardiff-multi-old-XXXXXX)
+PEEL_COUNT=$((N - 1))
+
+# Relative paths under data/ to peel into separate old layers (order matches older N=2 case: bar first).
+declare -a order=( "dir1/bar.txt" "dir1/foo.txt" "dir1/move.txt" )
+declare -a PEEL=()
+
+create_orig "$TEST_DIR/orig"
+
+for ((i = 0; i < PEEL_COUNT; i++)); do
+	if ((i < ${#order[@]})); then
+		PEEL+=("${order[i]}")
+	else
+		k=$((i - ${#order[@]} + 1))
+		mkdir -p "$TEST_DIR/orig/data/dir1"
+		echo "multi-pad-$k" >"$TEST_DIR/orig/data/dir1/_multi_$k.txt"
+		PEEL+=("dir1/_multi_$k.txt")
+	fi
+done
+
+create_tar "$TEST_DIR/orig.tar" "$TEST_DIR/orig"
+modify_orig "$TEST_DIR/modified" "$TEST_DIR/orig.tar"
+create_tar "$TEST_DIR/modified.tar" "$TEST_DIR/modified"
+
+# old1: full tree minus every peeled file
+cp -dR --no-preserve=context "$TEST_DIR/orig" "$TEST_DIR/mo-base" 2>/dev/null \
+	|| cp -dR "$TEST_DIR/orig" "$TEST_DIR/mo-base"
+for rel in "${PEEL[@]}"; do
+	rm -f "$TEST_DIR/mo-base/data/$rel"
+done
+
+# old2..oldN: one peeled file each
+for ((i = 0; i < PEEL_COUNT; i++)); do
+	rel="${PEEL[i]}"
+	layer=$((i + 2))
+	sdir="$TEST_DIR/mo-slice-$layer"
+	mkdir -p "$sdir/data/$(dirname "$rel")"
+	cp "$TEST_DIR/orig/data/$rel" "$sdir/data/$rel"
+	create_tar "$TEST_DIR/mo-old${layer}.tar" "$sdir"
+done
+
+create_tar "$TEST_DIR/mo-old1.tar" "$TEST_DIR/mo-base"
+
+declare -a DIFF_ARGS=()
+for ((i = 1; i <= N; i++)); do
+	DIFF_ARGS+=("$TEST_DIR/mo-old${i}.tar.gz")
+done
+DIFF_ARGS+=("$TEST_DIR/modified.tar.bz2" "$TEST_DIR/mo-changes.tardiff")
+
+echo "Generating tardiff (N=$N multi-old)"
+./tar-diff "${DIFF_ARGS[@]}"
+
+echo "Applying tardiff (N=$N)"
+./tar-patch "$TEST_DIR/mo-changes.tardiff" "$TEST_DIR/orig" "$TEST_DIR/mo-reconstructed.tar"
+
+echo "Verifying reconstruction (N=$N)"
+cmp "$TEST_DIR/mo-reconstructed.tar" "$TEST_DIR/modified.tar"
+
+echo "OK multi-old (N=$N)"
+
+cleanup () {
+	rm -rf "$TEST_DIR"
+}
+trap cleanup EXIT

tests/test-source-prefix.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+
+set -e
+
+source tests/utils.sh
+
+use_gnu_tar_if_available
+
+TEST_DIR=$(mktemp -d /tmp/test-tardiff-srcprefix-XXXXXX)
+TREE="$TEST_DIR/tree"
+
+create_v1 () {
+	rm -rf "$TREE"
+	mkdir -p "$TREE/blobs/sha256" "$TREE/config" "$TREE/data"
+	printf '%s' 'blob-content'   > "$TREE/blobs/sha256/abc123"
+	printf '%s' 'config-v1'     > "$TREE/config/app.conf"
+	printf '%s' 'data-v1'        > "$TREE/data/file.txt"
+}
+
+create_v2 () {
+	rm -rf "$TREE"
+	mkdir -p "$TREE/blobs/sha256" "$TREE/config" "$TREE/data"
+	printf '%s' 'blob-content-modified' > "$TREE/blobs/sha256/abc123"
+	printf '%s' 'config-v2'              > "$TREE/config/app.conf"
+	printf '%s' 'data-v2'                > "$TREE/data/file.txt"
+}
+
+create_tar () {
+	local file=$1
+	local root=$2
+	tar cf "$file" -C "$root" blobs config data
+    compress_tar "$file"
+}
+
+unpack_old_tree () {
+	local dest=$1
+	rm -rf "$dest"
+	mkdir -p "$dest"
+	tar xf "$TEST_DIR/old.tar" -C "$dest"
+}
+
+create_v1
+create_tar "$TEST_DIR/old.tar" "$TREE"
+
+create_v2
+create_tar "$TEST_DIR/new.tar" "$TREE"
+
+unpack_old_tree "$TEST_DIR/unpack1"
+
+echo "Generating tardiff (--source-prefix=blobs/)"
+./tar-diff --source-prefix=blobs/ "$TEST_DIR/old.tar.gz" "$TEST_DIR/new.tar.bz2" "$TEST_DIR/delta1.tardiff"
+
+echo "Applying tardiff (single prefix)"
+./tar-patch "$TEST_DIR/delta1.tardiff" "$TEST_DIR/unpack1" "$TEST_DIR/out1.tar"
+
+echo "Verifying (single prefix)"
+cmp "$TEST_DIR/out1.tar" "$TEST_DIR/new.tar"
+
+unpack_old_tree "$TEST_DIR/unpack2"
+
+echo "Generating tardiff (--source-prefix=blobs/ --source-prefix=config/)"
+./tar-diff --source-prefix=blobs/ --source-prefix=config/ "$TEST_DIR/old.tar.gz" "$TEST_DIR/new.tar.bz2" "$TEST_DIR/delta2.tardiff"
+
+echo "Applying tardiff (dual prefix)"
+./tar-patch "$TEST_DIR/delta2.tardiff" "$TEST_DIR/unpack2" "$TEST_DIR/out2.tar"
+
+echo "Verifying (dual prefix)"
+cmp "$TEST_DIR/out2.tar" "$TEST_DIR/new.tar"
+
+echo OK source-prefix
+
+cleanup () {
+	rm -rf "$TEST_DIR"
+}
+trap cleanup EXIT

tests/test-tar-errors.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Integration: expected failures for tar-diff and tar-patch CLIs (cover binaries).
+
+set -e
+
+source tests/utils.sh
+
+use_gnu_tar_if_available
+
+TEST_DIR=$(mktemp -d /tmp/test-tardiff-cli-err-XXXXXX)
+
+expect_fail () {
+	local desc=$1
+	shift
+	set +e
+	"$@" &>/dev/null
+	local code=$?
+	set -e
+	if [[ "$code" -eq 0 ]]; then
+		echo "expected failure for: $desc" >&2
+		exit 1
+	fi
+}
+
+mkdir -p "$TEST_DIR/orig/data"
+
+# tar-diff: CLI / I/O errors
+expect_fail "tar-diff too few args" ./tar-diff
+expect_fail "tar-diff two args only" ./tar-diff one two
+
+dd if=/dev/urandom of="$TEST_DIR/junk.gz" bs=200 count=1 status=none 2>/dev/null || \
+	dd if=/dev/urandom of="$TEST_DIR/junk.gz" bs=200 count=1 2>/dev/null
+expect_fail "tar-diff missing old archive" ./tar-diff "$TEST_DIR/does-not-exist.gz" "$TEST_DIR/junk.gz" "$TEST_DIR/delta-x.tardiff"
+
+mkdir -p "$TEST_DIR/td-a/data" "$TEST_DIR/td-b/data"
+echo x >"$TEST_DIR/td-a/data/f.txt"
+echo y >"$TEST_DIR/td-b/data/f.txt"
+create_tar "$TEST_DIR/td-old.tar" "$TEST_DIR/td-a"
+create_tar "$TEST_DIR/td-new.tar" "$TEST_DIR/td-b"
+expect_fail "tar-diff missing new archive" ./tar-diff "$TEST_DIR/td-old.tar.gz" "$TEST_DIR/missing-new.tar.gz" "$TEST_DIR/td-out.tardiff"
+
+expect_fail "tar-diff corrupt archive inputs" ./tar-diff "$TEST_DIR/junk.gz" "$TEST_DIR/junk.gz" "$TEST_DIR/td-badio.tardiff"
+
+# tar-patch: invalid / corrupt delta
+printf 'not-a-tardiff' >"$TEST_DIR/bad-magic.tardiff"
+expect_fail "tar-patch bad magic" ./tar-patch "$TEST_DIR/bad-magic.tardiff" "$TEST_DIR/orig" "$TEST_DIR/out1.tar"
+
+printf '\x74\x61\x72\x64\x66\x31\x0a\x00' >"$TEST_DIR/bad-zstd.tardiff"
+printf '\x00\x01\x02\x03' >>"$TEST_DIR/bad-zstd.tardiff"
+expect_fail "tar-patch invalid zstd" ./tar-patch "$TEST_DIR/bad-zstd.tardiff" "$TEST_DIR/orig" "$TEST_DIR/out2.tar"
+
+expect_fail "tar-patch missing base dir" ./tar-patch "$TEST_DIR/bad-magic.tardiff" "$TEST_DIR/does-not-exist" "$TEST_DIR/out3.tar"
+
+# tar-patch: valid delta, missing source file
+mkdir -p "$TEST_DIR/solo/data" "$TEST_DIR/solom/data"
+echo hello >"$TEST_DIR/solo/data/only.txt"
+echo 'hello!' >"$TEST_DIR/solom/data/only.txt"
+create_tar "$TEST_DIR/solo-old.tar" "$TEST_DIR/solo"
+create_tar "$TEST_DIR/solo-new.tar" "$TEST_DIR/solom"
+./tar-diff "$TEST_DIR/solo-old.tar.gz" "$TEST_DIR/solo-new.tar.bz2" "$TEST_DIR/solo.tardiff"
+rm -f "$TEST_DIR/solo/data/only.txt"
+expect_fail "tar-patch missing source member" ./tar-patch "$TEST_DIR/solo.tardiff" "$TEST_DIR/solo" "$TEST_DIR/solo-out.tar"
+
+# tar-patch: stdout destination (happy path)
+mkdir -p "$TEST_DIR/st/data" "$TEST_DIR/stm/data"
+echo hello >"$TEST_DIR/st/data/only.txt"
+echo 'hello!' >"$TEST_DIR/stm/data/only.txt"
+create_tar "$TEST_DIR/st-old.tar" "$TEST_DIR/st"
+create_tar "$TEST_DIR/st-new.tar" "$TEST_DIR/stm"
+./tar-diff "$TEST_DIR/st-old.tar.gz" "$TEST_DIR/st-new.tar.bz2" "$TEST_DIR/st.tardiff"
+./tar-patch "$TEST_DIR/st.tardiff" "$TEST_DIR/st" - | cmp - "$TEST_DIR/st-new.tar"
+
+echo OK cli-errors-tar-diff-tar-patch
+
+cleanup () {
+	rm -rf "$TEST_DIR"
+}
+trap cleanup EXIT