[1.14] Advertise wait ack bit in free page hinting

docs/ballooning.md

@@ -465,6 +465,26 @@ your scenario.
 >
 > This will prevent ranges which have been reclaimed from being freed.
 
+#### `VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK`
+
+Whenever `free_page_hinting` is enabled, Firecracker also advertises
+`VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK` (bit 6). When negotiated, the guest
+driver waits for the device to signal-used each hint buffer before
+pushing the corresponding page onto its internal free list — closing
+the data-loss race described in the warning above without any host-side
+protocol change.
+
+The bit only takes effect on guests whose kernel carries the supporting
+patch (Jack Thomson's `virtio_balloon: Support wait on ACK for hinting`,
+not yet upstream as of this writing). On unsupported guests the driver
+self-clears the bit during `validate`, so the advertise is ignored and
+hinting falls back to the unsynchronised behaviour. There is no separate
+configuration knob — opting into `free_page_hinting` is sufficient.
+
+Note that the per-buffer round trip introduces extra wait time per hint
+cycle on supported guests; the safety/perf trade-off is intentional and
+documented at the kernel-patch level.
+
 ## Balloon Caveats
 
 - Firecracker has no control over the speed of inflation or deflation; this is

src/vmm/src/devices/virtio/balloon/device.rs

@@ -20,7 +20,8 @@ use super::{
     FREE_PAGE_HINT_STOP, INFLATE_INDEX, MAX_PAGE_COMPACT_BUFFER, MAX_PAGES_IN_DESC,
     MIB_TO_4K_PAGES, STATS_INDEX, VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
     VIRTIO_BALLOON_F_FREE_PAGE_HINTING, VIRTIO_BALLOON_F_FREE_PAGE_REPORTING,
-    VIRTIO_BALLOON_F_STATS_VQ, VIRTIO_BALLOON_PFN_SHIFT, VIRTIO_BALLOON_S_ALLOC_STALL,
+    VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK, VIRTIO_BALLOON_F_STATS_VQ, VIRTIO_BALLOON_PFN_SHIFT,
+    VIRTIO_BALLOON_S_ALLOC_STALL,
     VIRTIO_BALLOON_S_ASYNC_RECLAIM, VIRTIO_BALLOON_S_ASYNC_SCAN, VIRTIO_BALLOON_S_AVAIL,
     VIRTIO_BALLOON_S_CACHES, VIRTIO_BALLOON_S_DIRECT_RECLAIM, VIRTIO_BALLOON_S_DIRECT_SCAN,
     VIRTIO_BALLOON_S_HTLB_PGALLOC, VIRTIO_BALLOON_S_HTLB_PGFAIL, VIRTIO_BALLOON_S_MAJFLT,
@@ -299,6 +300,7 @@ impl Balloon {
         if free_page_hinting {
             log_dev_preview_warning("Free Page Hinting", None);
             avail_features |= 1u64 << VIRTIO_BALLOON_F_FREE_PAGE_HINTING;
+            avail_features |= 1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK;
             queue_count += 1;
         }
 
@@ -741,6 +743,10 @@ impl Balloon {
         self.avail_features & (1u64 << VIRTIO_BALLOON_F_FREE_PAGE_HINTING) != 0
     }
 
+    pub fn free_page_hinting_wait_ack(&self) -> bool {
+        self.avail_features & (1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK) != 0
+    }
+
     pub fn free_page_hinting_idx(&self) -> usize {
         let mut idx = BALLOON_MIN_NUM_QUEUES;
 
@@ -1169,6 +1175,7 @@ pub(crate) mod tests {
                 | (u64::from(*deflate_on_oom) << VIRTIO_BALLOON_F_DEFLATE_ON_OOM)
                 | ((u64::from(*reporting)) << VIRTIO_BALLOON_F_FREE_PAGE_REPORTING)
                 | ((u64::from(*hinting)) << VIRTIO_BALLOON_F_FREE_PAGE_HINTING)
+                | ((u64::from(*hinting)) << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK)
                 | ((u64::from(*stats_interval)) << VIRTIO_BALLOON_F_STATS_VQ);
 
             assert_eq!(
@@ -1188,6 +1195,31 @@ pub(crate) mod tests {
         }
     }
 
+    #[test]
+    fn test_wait_on_ack_advertised_with_fph() {
+        let balloon = Balloon::new(0, false, 0, true, false).unwrap();
+        assert!(balloon.free_page_hinting());
+        assert!(balloon.free_page_hinting_wait_ack());
+        assert_ne!(
+            balloon.avail_features & (1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK),
+            0
+        );
+    }
+
+    #[test]
+    fn test_wait_on_ack_not_advertised_without_fph() {
+        for (deflate, stats_interval, reporting) in
+            iproduct!(&[true, false], &[0u16, 1], &[true, false])
+        {
+            let balloon = Balloon::new(0, *deflate, *stats_interval, false, *reporting).unwrap();
+            assert!(!balloon.free_page_hinting_wait_ack());
+            assert_eq!(
+                balloon.avail_features & (1u64 << VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK),
+                0
+            );
+        }
+    }
+
     #[test]
     fn test_virtio_read_config() {
         let balloon = Balloon::new(0x10, true, 0, false, false).unwrap();

src/vmm/src/devices/virtio/balloon/mod.rs

@@ -54,6 +54,7 @@ const VIRTIO_BALLOON_F_STATS_VQ: u32 = 1; // Enable statistics.
 const VIRTIO_BALLOON_F_DEFLATE_ON_OOM: u32 = 2; // Deflate balloon on OOM.
 const VIRTIO_BALLOON_F_FREE_PAGE_HINTING: u32 = 3; // Enable free page hinting
 const VIRTIO_BALLOON_F_FREE_PAGE_REPORTING: u32 = 5; // Enable free page reporting
+pub(crate) const VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK: u32 = 6; // Hinting waits on device ack
 
 // The statistics tags. defined in linux "include/uapi/linux/virtio_balloon.h".
 const VIRTIO_BALLOON_S_SWAP_IN: u16 = 0;

src/vmm/src/devices/virtio/balloon/persist.rs

@@ -263,4 +263,29 @@ mod tests {
         assert_eq!(restored_balloon.stats_desc_index, balloon.stats_desc_index);
         assert_eq!(restored_balloon.latest_stats, balloon.latest_stats);
     }
+
+    #[test]
+    fn test_wait_on_ack_round_trips_snapshot() {
+        let guest_mem = default_mem();
+        let mut mem = vec![0; 4096];
+
+        let balloon = Balloon::new(0, false, 0, true, false).unwrap();
+        assert!(balloon.free_page_hinting_wait_ack());
+
+        Snapshot::new(balloon.save())
+            .save(&mut mem.as_mut_slice())
+            .unwrap();
+
+        let restored_balloon = Balloon::restore(
+            BalloonConstructorArgs { mem: guest_mem },
+            &Snapshot::load_without_crc_check(mem.as_slice())
+                .unwrap()
+                .data,
+        )
+        .unwrap();
+
+        assert!(restored_balloon.free_page_hinting());
+        assert!(restored_balloon.free_page_hinting_wait_ack());
+        assert_eq!(restored_balloon.avail_features, balloon.avail_features);
+    }
 }

tests/integration_tests/functional/test_balloon_wait_on_ack.py

@@ -0,0 +1,77 @@
+# Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Verify VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK negotiation with a patched guest.
+
+This test is gated on `@pytest.mark.requires_patched_kernel` and is excluded
+from every CI run (regular and nightly) by `tests/pytest.ini`. It assumes the
+6.1 artifact `vmlinux` has been replaced in place with a build that carries
+Jack Thomson's `virtio_balloon: Support wait on ACK for hinting` patch (see
+fc-kernels/patches/6.1.158/0001-virtio_balloon-Support-wait-on-ACK-for-hinting.patch).
+
+To run it after swapping in the patched kernel:
+
+    tools/devtool -y test -- -m requires_patched_kernel \\
+        tests/integration_tests/functional/test_balloon_wait_on_ack.py
+
+If the kernel is *not* patched, the bit-6 assertion fails with a clear
+"did you replace the kernel?" message — that's the expected signal that
+the prerequisite is missing.
+"""
+
+import pytest
+
+VIRTIO_ID_BALLOON = 5
+VIRTIO_BALLOON_F_FREE_PAGE_HINT = 3
+VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK = 6
+
+
+def _read_balloon_features(vm):
+    """Return the negotiated features bitstring of the balloon virtio device.
+
+    Linux exposes negotiated features via
+    `/sys/bus/virtio/devices/virtio*/features` as one ASCII char per bit,
+    LSB-first, plus a trailing newline. See `features_show` in upstream
+    `drivers/virtio/virtio.c`.
+    """
+    # /sys/bus/virtio/devices/virtio*/device is a 0x%04x string with
+    # trailing newline, e.g. "0x0005\n" for balloon. Match by hex value.
+    cmd = (
+        "for d in /sys/bus/virtio/devices/virtio*; do "
+        '  if [ "$(cat "$d/device")" = "0x0005" ]; then '
+        '    cat "$d/features"; '
+        "    exit 0; "
+        "  fi; "
+        "done; "
+        "exit 1"
+    )
+    rc, stdout, stderr = vm.ssh.run(cmd)
+    assert rc == 0, f"balloon virtio device not found in guest sysfs: {stderr}"
+    return stdout.strip()
+
+
+@pytest.mark.requires_patched_kernel
+def test_fph_wait_on_ack_negotiated(uvm_plain_6_1):
+    """The guest negotiates bit 6 (WAIT_ON_ACK) when FPH is enabled."""
+    vm = uvm_plain_6_1
+    vm.spawn()
+    vm.basic_config(vcpu_count=1, mem_size_mib=256)
+    vm.add_net_iface()
+    vm.api.balloon.put(
+        amount_mib=0,
+        deflate_on_oom=False,
+        free_page_hinting=True,
+    )
+    vm.start()
+
+    features = _read_balloon_features(vm)
+
+    # Format: LSB-first '0'/'1' string.
+    assert features[VIRTIO_BALLOON_F_FREE_PAGE_HINT] == "1", (
+        f"FREE_PAGE_HINT (bit 3) not negotiated; features={features!r}"
+    )
+    assert features[VIRTIO_BALLOON_F_HINT_WAIT_ON_ACK] == "1", (
+        f"HINT_WAIT_ON_ACK (bit 6) not negotiated; features={features!r}. "
+        "The guest kernel likely lacks the wait-on-ACK patch — did you "
+        "forget to replace the 6.1 artifact vmlinux with a patched build? "
+        "See the module docstring."
+    )

tests/pytest.ini

@@ -5,12 +5,13 @@ addopts =
     -vv
     --durations=10
     --showlocals
-    -m 'not nonci and not no_block_pr'
+    -m 'not nonci and not no_block_pr and not requires_patched_kernel'
     --json-report --json-report-file=../test_results/test-report.json
 
 markers =
     no_block_pr: tests whose failure does not block PR merging.
     nonci: mark test as nonci.
+    requires_patched_kernel: tests that require a kernel binary with out-of-tree patches; never run by CI, run manually with -m requires_patched_kernel.
 
 ; Overwrite the default norecursedirs, which includes 'build'.
 norecursedirs = .*