benchmarks: convergence suite v1 (Ipopt + MadNLP on X gate)

.github/workflows/benchmark-convergence.yml

@@ -0,0 +1,52 @@
+name: Convergence Benchmarks
+on:
+  push:
+    tags: ['v*']
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'benchmark/convergence/**'
+      - '.github/workflows/benchmark-convergence.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  convergence:
+    name: Convergence suite (Ipopt + MadNLP)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1.12'
+          arch: x64
+
+      - uses: julia-actions/cache@v2
+
+      - name: Instantiate convergence environment
+        run: julia --project=benchmark/convergence -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+      - name: Run convergence benchmarks
+        env:
+          BENCHMARK_RUNNER: github-actions
+        run: |
+          julia --project=benchmark/convergence -t auto -e '
+            using TestItemRunner
+            TestItemRunner.run_tests("benchmark/convergence/")
+          '
+
+      - name: Upload convergence artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: convergence-${{ github.event.pull_request.number || github.ref_name }}-${{ github.sha }}
+          path: benchmark/convergence/results/
+          retention-days: 90

benchmark/convergence/.gitignore

@@ -0,0 +1,2 @@
+results/
+Manifest.toml

benchmark/convergence/Project.toml

@@ -0,0 +1,22 @@
+[deps]
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+DirectTrajOpt = "c823fa1f-8872-4af5-b810-2b9b72bbbf56"
+ExponentialAction = "e24c0720-ea99-47e8-929e-571b494574d3"
+HarmoniqsBenchmarks = "f45d0b76-2d23-4568-9599-481e0da131db"
+Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MadNLP = "2621e9c9-9eb4-46b1-8089-e8c72242dfb6"
+NamedTrajectories = "538bc3a1-5ab9-4fc3-b776-35ca1e893e08"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+TestItems = "1c621080-faea-4a02-84b6-bbd5e436b8fe"
+
+[sources]
+DirectTrajOpt = {path = "../.."}
+# HBJ v0.2.0 not yet registered in General; pin to a specific commit so
+# convergence results are reproducible across CI re-runs. Bump this SHA
+# (and the local Manifest) when HBJ ships a new feature we want to use.
+# Drop in favor of `[compat]` once HBJ registers in General.
+HarmoniqsBenchmarks = {url = "https://github.com/harmoniqs/HarmoniqsBenchmarks.jl", rev = "c38418cb7f932f2ff9a9c6c6eacf9a11ff1018c1"}

benchmark/convergence/README.md

@@ -0,0 +1,29 @@
+# DirectTrajOpt — Convergence Benchmarks
+
+Convergence-quality benchmarks built on HarmoniqsBenchmarks.jl v0.2.0's new
+convergence API (`InfidelityConvergence`, `ipopt_capture`,
+`compare_convergence`). Each `@testitem` runs a small X-gate state-transfer
+on Ipopt or MadNLP and asserts the result meets a problem-specific success
+bar before saving a JLD2 artifact under `benchmark/convergence/results/`.
+
+## Running locally
+
+```bash
+# from DirectTrajOpt.jl root
+julia --project=benchmark/convergence -e 'using Pkg; Pkg.instantiate()'
+julia --project=benchmark/convergence -e '
+    using TestItemRunner
+    @run_package_tests filter = ti -> occursin("convergence", ti.name)
+'
+```
+
+## What's covered
+
+- **X gate convergence: Ipopt** — uses `ipopt_capture()` to grab final
+  `iter_count` + `inf_pr`, builds an `InfidelityConvergence`, passes it
+  through `benchmark_solve!` to populate `BenchmarkResult.convergence`.
+- **X gate convergence: MadNLP** — same problem, MadNLP solver. No capture
+  hook yet, so `primal_infeasibility` is taken from the post-solve
+  evaluator's `constraint_violation`.
+
+Atoms / spin-qubit / bosonic demo problems land in separate follow-up PRs.

benchmark/convergence/convergence.jl

@@ -0,0 +1,118 @@
+using TestItems
+
+@testitem "X gate convergence: Ipopt" begin
+    using HarmoniqsBenchmarks
+    using DirectTrajOpt
+    using NamedTrajectories
+    using SparseArrays, ExponentialAction, Random, Dates, Printf, LinearAlgebra
+
+    include(joinpath(@__DIR__, "problem_utils.jl"))
+
+    runner = get(ENV, "BENCHMARK_RUNNER", "local")
+
+    prob = _make_xgate_prob(; N = 51, seed = 42)
+
+    # Wire Ipopt callback that captures final iter_count + inf_pr.
+    state, cb = ipopt_capture()
+    ipopt_opts = IpoptOptions(max_iter = 500, print_level = 0)
+
+    # benchmark_solve! forwards extra kwargs to DirectTrajOpt.solve!, so we
+    # can inject the capture callback through the same call.
+    result = benchmark_solve!(
+        prob,
+        ipopt_opts;
+        benchmark_name = "xgate_convergence_ipopt_N51",
+        runner = runner,
+        callback = cb,
+    )
+
+    final_inf = _xgate_infidelity(prob)
+    primal_inf = ipopt_primal_infeasibility(state)
+    iters = ipopt_iterations(state)
+
+    crit = InfidelityConvergence(
+        target_infidelity = 1e-3,
+        final_infidelity = final_inf,
+        primal_infeasibility = primal_inf,
+        feas_tol = 1e-6,
+    )
+
+    result_with_conv = _build_convergence_result(result, crit; iterations = iters)
+
+    @printf(
+        "\n=== X gate convergence (Ipopt) ===\n  iters=%d  final_inf=%.3e  inf_pr=%.3e  wall=%.3fs  converged=%s\n",
+        iters,
+        final_inf,
+        primal_inf,
+        result_with_conv.wall_time_s,
+        converged(crit),
+    )
+
+    @test converged(result_with_conv.convergence) == true
+
+    results_dir = joinpath(@__DIR__, "results")
+    saved = save_results(results_dir, "xgate_convergence_ipopt_N51", [result_with_conv])
+    println("  Saved $(saved)")
+
+    # Exercise the reporting path.
+    rows = compare_convergence([result_with_conv])
+    @test length(rows) == 1
+    @test rows[1].converged == true
+end
+
+
+@testitem "X gate convergence: MadNLP" begin
+    using HarmoniqsBenchmarks
+    using DirectTrajOpt
+    using NamedTrajectories
+    using SparseArrays, ExponentialAction, Random, Dates, Printf, LinearAlgebra
+    import MadNLP
+
+    include(joinpath(@__DIR__, "problem_utils.jl"))
+
+    runner = get(ENV, "BENCHMARK_RUNNER", "local")
+
+    prob = _make_xgate_prob(; N = 51, seed = 42)
+
+    madnlp_opts = MadNLPOptions(max_iter = 500, print_level = 6)
+
+    # MadNLP doesn't have an ipopt_capture analogue yet — use the post-solve
+    # constraint_violation that benchmark_solve! already extracted as the
+    # primal-infeasibility proxy.
+    result = benchmark_solve!(
+        prob,
+        madnlp_opts;
+        benchmark_name = "xgate_convergence_madnlp_N51",
+        runner = runner,
+    )
+
+    final_inf = _xgate_infidelity(prob)
+    primal_inf = result.constraint_violation
+
+    crit = InfidelityConvergence(
+        target_infidelity = 1e-3,
+        final_infidelity = final_inf,
+        primal_infeasibility = primal_inf,
+        feas_tol = 1e-6,
+    )
+
+    result_with_conv = _build_convergence_result(result, crit)
+
+    @printf(
+        "\n=== X gate convergence (MadNLP) ===\n  final_inf=%.3e  cviol=%.3e  wall=%.3fs  converged=%s\n",
+        final_inf,
+        primal_inf,
+        result_with_conv.wall_time_s,
+        converged(crit),
+    )
+
+    @test converged(result_with_conv.convergence) == true
+
+    results_dir = joinpath(@__DIR__, "results")
+    saved = save_results(results_dir, "xgate_convergence_madnlp_N51", [result_with_conv])
+    println("  Saved $(saved)")
+
+    rows = compare_convergence([result_with_conv])
+    @test length(rows) == 1
+    @test rows[1].converged == true
+end

benchmark/convergence/problem_utils.jl

@@ -0,0 +1,111 @@
+# Shared helpers for the convergence test items.
+# Included by each @testitem via
+#     include(joinpath(@__DIR__, "problem_utils.jl"))
+
+"""
+    _make_xgate_prob(; N=51, seed=42)
+
+X-gate-style bilinear state-transfer problem: 4D real Bloch-like rep with
+`x_init = [1,0,0,0]`, `x_goal = [0,1,0,0]`. Mirrors the shape of
+`get_seeded_prob` in `test/solver_test_utils.jl` — terminal cost pulls `x`
+toward the goal, and bounds/regularizers are sized so both Ipopt and MadNLP
+actually drive infidelity below the convergence target.
+"""
+function _make_xgate_prob(; N::Int = 51, seed::Int = 42)
+    Random.seed!(seed)
+    Δt = 0.1
+    u_bound = 1.0
+    ω = 0.1
+    Gx = sparse(Float64[0 0 0 1; 0 0 1 0; 0 -1 0 0; -1 0 0 0])
+    Gy = sparse(Float64[0 -1 0 0; 1 0 0 0; 0 0 0 -1; 0 0 1 0])
+    Gz = sparse(Float64[0 0 1 0; 0 0 0 -1; -1 0 0 0; 0 1 0 0])
+    G(u) = ω * Gz + u[1] * Gx + u[2] * Gy
+
+    x_init = [1.0, 0.0, 0.0, 0.0]
+    x_goal = [0.0, 1.0, 0.0, 0.0]
+
+    traj = NamedTrajectory(
+        (
+            x = 2rand(4, N) .- 1,
+            u = u_bound * (2rand(2, N) .- 1),
+            du = randn(2, N),
+            ddu = randn(2, N),
+            Δt = fill(Δt, N),
+        );
+        controls = (:ddu, :Δt),
+        timestep = :Δt,
+        bounds = (u = (-u_bound, u_bound), Δt = (Δt, Δt)),
+        initial = (x = x_init, u = zeros(2)),
+        final = (u = zeros(2),),
+        goal = (x = x_goal,),
+    )
+    integrators = [
+        BilinearIntegrator(G, :x, :u, traj),
+        DerivativeIntegrator(:u, :du, traj),
+        DerivativeIntegrator(:du, :ddu, traj),
+    ]
+    J = TerminalObjective(x -> 1e3 * sum(abs2, x - x_goal), :x, traj)
+    J += QuadraticRegularizer(:u, traj, 1e-2)
+    J += QuadraticRegularizer(:du, traj, 1e-2)
+    return DirectTrajOptProblem(traj, J, integrators)
+end
+
+"""
+    _xgate_infidelity(prob) -> Float64
+
+Infidelity = 1 - <x_final, x_goal>, clamped to [0, 1]. Cheap because both
+vectors are unit-norm in this representation.
+"""
+_xgate_infidelity(prob) = clamp(
+    1.0 - LinearAlgebra.dot(prob.trajectory.x[:, end], [0.0, 1.0, 0.0, 0.0]),
+    0.0,
+    1.0,
+)
+
+"""
+    _build_convergence_result(result::BenchmarkResult, crit::ConvergenceCriterion;
+                              iterations::Union{Nothing,Int}=nothing)
+
+Return a copy of `result` with `crit` attached as its `convergence` field
+(optionally overriding `iterations`). `BenchmarkResult` is immutable, so we
+rebuild it positionally; pulling this out of the testitems keeps them
+readable.
+"""
+function _build_convergence_result(
+    result::HarmoniqsBenchmarks.BenchmarkResult,
+    crit::HarmoniqsBenchmarks.ConvergenceCriterion;
+    iterations::Union{Nothing,Int} = nothing,
+)
+    iters = iterations === nothing ? result.iterations : iterations
+    return HarmoniqsBenchmarks.BenchmarkResult(
+        package = result.package,
+        package_version = result.package_version,
+        commit = result.commit,
+        benchmark_name = result.benchmark_name,
+        N = result.N,
+        state_dim = result.state_dim,
+        control_dim = result.control_dim,
+        n_constraints = result.n_constraints,
+        n_variables = result.n_variables,
+        wall_time_s = result.wall_time_s,
+        iterations = iters,
+        objective_value = result.objective_value,
+        constraint_violation = result.constraint_violation,
+        solver_status = result.solver_status,
+        solver = result.solver,
+        total_allocations_bytes = result.total_allocations_bytes,
+        total_allocs_count = result.total_allocs_count,
+        gc_time_ns = result.gc_time_ns,
+        gc_count = result.gc_count,
+        gc_full_count = result.gc_full_count,
+        peak_rss_delta_bytes = result.peak_rss_delta_bytes,
+        live_heap_delta_bytes = result.live_heap_delta_bytes,
+        oom_margin_bytes = result.oom_margin_bytes,
+        solver_options = result.solver_options,
+        convergence = crit,
+        julia_version = result.julia_version,
+        timestamp = result.timestamp,
+        runner = result.runner,
+        n_threads = result.n_threads,
+    )
+end

test/runtests.jl

@@ -3,5 +3,8 @@ using TestItemRunner
 
 include("test_snippets.jl")
 
-# Run all testitem tests in package
-@run_package_tests
+# Exclude benchmark/ testitems — those run in a separate project environment
+# with HarmoniqsBenchmarks.jl as a dependency. Match the "benchmark" path
+# component exactly so a future test file like foo_benchmark.jl isn't
+# accidentally skipped.
+@run_package_tests filter = ti -> !("benchmark" in splitpath(ti.filename))