Skip to content

Commit cff4a3a

Browse files
committed
work: various changes to enable benefits of cow in wasm
1 parent 1ce7ef0 commit cff4a3a

14 files changed

Lines changed: 637 additions & 195 deletions

File tree

Cargo.lock

Lines changed: 155 additions & 123 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@ repository = "https://github.com/hyperlight-dev/hyperlight-wasm"
1313
readme = "README.md"
1414

1515
[workspace.dependencies]
16-
hyperlight-host = { version = "0.12.0", default-features = false, features = ["executable_heap", "init-paging"] }
16+
hyperlight-host = { git = "https://github.com/hyperlight-dev/hyperlight", branch = "lm/cow-wasm-work", default-features = false, features = ["executable_heap", "init-paging"] }

flake.lock

Lines changed: 44 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

flake.nix

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
{
2+
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
3+
inputs.nixpkgs-mozilla.url = "github:mozilla/nixpkgs-mozilla/master";
4+
outputs = { self, nixpkgs, nixpkgs-mozilla, ... } @ inputs:
5+
rec {
6+
overlays.fix-rust = self: super: {
7+
# Work around the nixpkgs-mozilla equivalent of
8+
# https://github.com/NixOS/nixpkgs/issues/278508 and an
9+
# incompatibility between nixpkgs-mozilla and makeRustPlatform
10+
rustChannelOf = args: let
11+
orig = super.rustChannelOf args;
12+
patchRustPkg = pkg: (pkg.overrideAttrs (oA: {
13+
buildCommand = (builtins.replaceStrings
14+
[ "rustc,rustdoc" ]
15+
[ "rustc,rustdoc,clippy-driver,cargo-clippy,miri,cargo-miri" ]
16+
oA.buildCommand) + (let
17+
wrapperPath = self.path + "/pkgs/build-support/bintools-wrapper/ld-wrapper.sh";
18+
baseOut = self.clangStdenv.cc.bintools.out;
19+
getStdenvAttrs = drv: (drv.overrideAttrs (oA: {
20+
passthru.origAttrs = oA;
21+
})).origAttrs;
22+
baseEnv = (getStdenvAttrs self.clangStdenv.cc.bintools).env;
23+
baseSubstitutedWrapper = self.replaceVars wrapperPath
24+
{
25+
inherit (baseEnv)
26+
shell coreutils_bin suffixSalt mktemp rm;
27+
use_response_file_by_default = "0";
28+
prog = null;
29+
out = null;
30+
};
31+
in ''
32+
# work around a bug in the overlay
33+
${oA.postInstall}
34+
35+
# copy over helper scripts that the wrapper needs
36+
(cd "${baseOut}"; find . -type f \( -name '*.sh' -or -name '*.bash' \) -print0) | while read -d $'\0' script; do
37+
mkdir -p "$out/$(dirname "$script")"
38+
substitute "${baseOut}/$script" "$out/$script" --replace-quiet "${baseOut}" "$out"
39+
done
40+
41+
# TODO: Work out how to make this work with cross builds
42+
ldlld="$out/lib/rustlib/${self.clangStdenv.targetPlatform.config}/bin/gcc-ld/ld.lld";
43+
if [ -e "$ldlld" ]; then
44+
export prog="$(readlink -f "$ldlld")"
45+
rm "$ldlld"
46+
substitute ${baseSubstitutedWrapper} "$ldlld" --subst-var "out" --subst-var "prog"
47+
chmod +x "$ldlld"
48+
fi
49+
'');
50+
})) // {
51+
targetPlatforms = [ "x86_64-linux" ];
52+
badTargetPlatforms = [ ];
53+
};
54+
overrideRustPkg = pkg: self.lib.makeOverridable (origArgs:
55+
patchRustPkg (pkg.override origArgs)
56+
) {};
57+
in builtins.mapAttrs (_: overrideRustPkg) orig;
58+
};
59+
gcroots =
60+
let gcrootForShell = pkg: pkg // derivation (pkg.drvAttrs // {
61+
origArgs = pkg.drvAttrs.args;
62+
# assume the builder is bash for now (it always is for
63+
# stdenv, which is the only thing that we will encounter
64+
# in this flake).
65+
args = [ "-c" "declare > $out" ];
66+
});
67+
in {
68+
shells.default = gcrootForShell devShells.x86_64-linux.default;
69+
};
70+
devShells.x86_64-linux.default =
71+
let pkgs = import nixpkgs {
72+
system = "x86_64-linux";
73+
overlays = [ (import (nixpkgs-mozilla + "/rust-overlay.nix")) overlays.fix-rust ];
74+
};
75+
in with pkgs; let
76+
customisedRustChannelOf = args:
77+
lib.flip builtins.mapAttrs (rustChannelOf args) (_: pkg: pkg.override {
78+
targets = [
79+
"x86_64-unknown-linux-gnu"
80+
"x86_64-pc-windows-msvc" "x86_64-unknown-none"
81+
"wasm32-wasip1" "wasm32-wasip2" "wasm32-unknown-unknown"
82+
"i686-unknown-linux-gnu"
83+
];
84+
extensions = [ "rust-src" ] ++ (if args.channel == "nightly" then [ "miri-preview" ] else []);
85+
});
86+
87+
# Hyperlight needs a variety of toolchains, since we use Nightly
88+
# for rustfmt and old toolchains to verify MSRV
89+
toolchains = lib.mapAttrs (_: customisedRustChannelOf) {
90+
stable = {
91+
date = "2025-12-11";
92+
channel = "stable";
93+
sha256 = "sha256-sqSWJDUxc+zaz1nBWMAJKTAGBuGWP25GCftIOlCEAtA=";
94+
};
95+
nightly = {
96+
date = "2026-01-19";
97+
channel = "nightly";
98+
sha256 = "sha256-Ye65U/qzilPLte800N5oxFOY96shgG8bST8dbrF6Qh0=";
99+
};
100+
"1.89" = {
101+
date = "2025-08-07";
102+
channel = "stable";
103+
sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE=";
104+
};
105+
};
106+
107+
rust-platform = makeRustPlatform {
108+
cargo = toolchains.stable.rust;
109+
rustc = toolchains.stable.rust;
110+
};
111+
112+
manifests = [
113+
"Cargo.toml"
114+
"src/hyperlight_wasm_macro/Cargo.toml"
115+
"src/rust_wasm_samples/Cargo.toml"
116+
"src/wasm_runtime/Cargo.toml"
117+
"src/component_sample/Cargo.toml"
118+
];
119+
manifestDeps = builtins.map (manifest:
120+
let lockPath = builtins.replaceStrings [ "toml" ] [ "lock" ] manifest; in
121+
let lockFile = ./${lockPath}; in
122+
rust-platform.importCargoLock { inherit lockFile; }) manifests;
123+
# when building a guest with cargo-hyperlight, or when
124+
# building a miri sysroot for the main workspace, we need to
125+
# include any crates.io dependencies of the standard library
126+
# (e.g. rustc-literal-escaper)
127+
stdlibLocks = lib.mapAttrsToList (_: toolchain:
128+
"${toolchain.rust}/lib/rustlib/src/rust/library/Cargo.lock"
129+
) toolchains;
130+
stdlibDeps = builtins.map (lockFile:
131+
rust-platform.importCargoLock { inherit lockFile; }) stdlibLocks;
132+
deps = pkgs.symlinkJoin {
133+
name = "cargo-deps";
134+
paths = stdlibDeps ++ manifestDeps;
135+
};
136+
137+
# Script snippet, used in the cargo/rustc wrappers below,
138+
# which creates a number of .cargo/config.toml files in
139+
# order to allow using Nix-fetched dependencies (this must
140+
# be done for the guests, as well as for the main
141+
# workspace). Ideally, we would just use environment
142+
# variables or the --config option to Cargo, but
143+
# unfortunately that tends not to play well with subcommands
144+
# like `cargo clippy` and `cargo hyperlight` (see
145+
# https://github.com/rust-lang/cargo/issues/11031).
146+
materialiseDeps = let
147+
sortedManifests = lib.lists.sort (p: q: p > q) manifests;
148+
matchClause = path: '' */${path}) root="''${manifest%${path}}" ;;'';
149+
matchClauses = lib.strings.concatStringsSep "\n"
150+
(builtins.map matchClause sortedManifests);
151+
in ''
152+
base_cargo() {
153+
PATH="$base/bin:$PATH" "$base/bin/cargo" "$@"
154+
}
155+
156+
manifest=$(base_cargo locate-project --message-format plain --workspace)
157+
case "$manifest" in
158+
${matchClauses}
159+
esac
160+
if [ -f ''${root}/flake.nix ]; then
161+
162+
mkdir -p $root/$.cargo
163+
cat >$root/.cargo/config.toml2 <<EOF
164+
[source.crates-io]
165+
replace-with = "vendored-sources"
166+
167+
[source.vendored-sources]
168+
directory = "${deps}"
169+
EOF
170+
171+
sed -i '/# vendor dependency configuration generated by nix/{N;d;}' $root/.git/info/exclude
172+
printf "# vendor dependency configuration generated by nix\n%s\n" "/.cargo" >> $root/.git/info/exclude
173+
fi
174+
175+
# libgit2-sys copies a vendored git2 into the target/
176+
# directory somewhere. In certain, rare, cases,
177+
# libgit2-sys is rebuilt in the same incremental dep
178+
# directory as it was before, and then this copy fails,
179+
# because the files, copied from the nix store, already
180+
# exist and do not have w permission. Hack around this
181+
# issue by making any existing libgit2-sys vendored git2
182+
# files writable before a build can be run
183+
find "$(base_cargo metadata --format-version 1 | jq -r '.target_directory')" -path '*/build/libgit2-sys-*/out/include' -print0 | xargs -r -0 chmod u+w -R
184+
'';
185+
186+
# Hyperlight scripts use cargo in a bunch of ways that don't
187+
# make sense for Nix cargo, including the `rustup +toolchain`
188+
# syntax to use a specific toolchain and `cargo install`, so we
189+
# build wrappers for rustc and cargo that enable this. The
190+
# scripts also use `rustup toolchain install` in some cases, in
191+
# order to work in CI, so we provide a fake rustup that does
192+
# nothing as well.
193+
rustup-like-wrapper = name: pkgs.writeShellScriptBin name
194+
(let
195+
clause = name: toolchain:
196+
"+${name}) base=\"${toolchain.rust}\"; shift 1; ;;";
197+
clauses = lib.strings.concatStringsSep "\n"
198+
(lib.mapAttrsToList clause toolchains);
199+
in ''
200+
base="${toolchains.stable.rust}"
201+
${materialiseDeps}
202+
case "$1" in
203+
${clauses}
204+
install) exit 0; ;;
205+
esac
206+
export PATH="$base/bin:$PATH"
207+
exec "$base/bin/${name}" "$@"
208+
'');
209+
fake-rustup = pkgs.symlinkJoin {
210+
name = "fake-rustup";
211+
paths = [
212+
(pkgs.writeShellScriptBin "rustup" "")
213+
(rustup-like-wrapper "rustc")
214+
(rustup-like-wrapper "cargo")
215+
];
216+
};
217+
218+
buildRustPackageClang = rust-platform.buildRustPackage.override { stdenv = clangStdenv; };
219+
220+
cargo-hyperlight = buildRustPackageClang rec {
221+
pname = "cargo-hyperlight";
222+
version = "0.1.5";
223+
src = fetchFromGitHub {
224+
owner = "hyperlight-dev";
225+
repo = "cargo-hyperlight";
226+
tag = "v${version}";
227+
hash = "sha256-xq4/c69N0wG/I8WOYVloo0J0JqoSIKiWWtECdSKrsxo=";
228+
};
229+
cargoHash = "sha256-muiMVrK1TydQiMitihfo7xYidqUIIQ+Hw3BIeo5rLFw=";
230+
};
231+
in (buildRustPackageClang (mkDerivationAttrs: {
232+
pname = "hyperlight";
233+
version = "0.0.0";
234+
src = lib.cleanSource ./.;
235+
cargoDeps = deps;
236+
237+
nativeBuildInputs = [
238+
azure-cli
239+
just
240+
dotnet-sdk_9
241+
llvmPackages_18.llvm
242+
gh
243+
lld
244+
valgrind
245+
pkg-config
246+
ffmpeg
247+
mkvtoolnix
248+
wasm-tools
249+
jq
250+
jaq
251+
gdb
252+
zlib
253+
cargo-hyperlight
254+
typos
255+
flatbuffers
256+
cargo-fuzz
257+
wit-bindgen
258+
cargo-component
259+
];
260+
buildInputs = [
261+
pango
262+
cairo
263+
openssl
264+
];
265+
266+
auditable = false;
267+
268+
LIBCLANG_PATH = "${pkgs.llvmPackages_18.libclang.lib}/lib";
269+
# Use unwrapped clang for compiling guests
270+
HYPERLIGHT_GUEST_clang = "${clang.cc}/bin/clang";
271+
272+
RUST_NIGHTLY = "${toolchains.nightly.rust}";
273+
# Set this through shellHook rather than nativeBuildInputs to be
274+
# really sure that it overrides the real cargo.
275+
postHook = ''
276+
export PATH="${fake-rustup}/bin:$PATH"
277+
'';
278+
})).overrideAttrs(oA: {
279+
hardeningDisable = [ "all" ];
280+
});
281+
};
282+
}

src/hyperlight_wasm/examples/component_example/main.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ fn main() {
4747
let state = State::new();
4848
let mut sb: hyperlight_wasm::ProtoWasmSandbox = hyperlight_wasm::SandboxBuilder::new()
4949
.with_guest_input_buffer_size(70000000)
50+
.with_guest_scratch_size(70995968)
5051
.with_guest_heap_size(200000000)
5152
.with_guest_stack_size(100000000)
5253
.build()

src/hyperlight_wasm/examples/helloworld/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ fn main() -> Result<()> {
112112
"RoundToInt test case {idx} failed: got {}, expected {}",
113113
result, expected_result
114114
);
115-
loaded_wasm_sandbox.restore(&snapshot)?
115+
loaded_wasm_sandbox.restore(snapshot.clone())?
116116
}
117117
Ok(())
118118
}

src/hyperlight_wasm/examples/interruption/main.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ fn main() -> Result<()> {
9797

9898
// Recovery option 1: Use restore() to recover the sandbox
9999
println!("\n7. Recovering sandbox using restore()...");
100-
loaded.restore(&snapshot)?;
100+
loaded.restore(snapshot)?;
101101
assert!(!loaded.is_poisoned()?);
102102
println!(" is_poisoned after restore: {}", loaded.is_poisoned()?);
103103

src/hyperlight_wasm/src/sandbox/loaded_wasm_sandbox.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ pub struct LoadedWasmSandbox {
4242
// because of this we cannot implement drop without making inner an Option (alternatively we could make MultiUseSandbox Copy but that would introduce other issues)
4343
inner: Option<MultiUseSandbox>,
4444
// The state the sandbox was in before loading a wasm module. Used for transitioning back to a `WasmSandbox` (unloading the wasm module).
45-
runtime_snapshot: Option<Snapshot>,
45+
runtime_snapshot: Option<Arc<Snapshot>>,
4646
}
4747

4848
impl LoadedWasmSandbox {
@@ -86,9 +86,9 @@ impl LoadedWasmSandbox {
8686
/// Returns `Err(HyperlightError::PoisonedSandbox)` if the sandbox is in a
8787
/// poisoned state. Use [`restore()`](Self::restore) with a previously
8888
/// taken snapshot to recover before taking a new snapshot.
89-
pub fn snapshot(&mut self) -> Result<Snapshot> {
89+
pub fn snapshot(&mut self) -> Result<Arc<Snapshot>> {
9090
match &mut self.inner {
91-
Some(inner) => inner.snapshot(),
91+
Some(inner) => inner.snapshot(None),
9292
None => log_then_return!("No inner MultiUseSandbox to snapshot"),
9393
}
9494
}
@@ -105,7 +105,8 @@ impl LoadedWasmSandbox {
105105
/// 1. Clear the poisoned state
106106
/// 2. Reset memory to the snapshot state
107107
/// 3. Allow subsequent [`call_guest_function()`](Self::call_guest_function) calls to succeed
108-
pub fn restore(&mut self, snapshot: &Snapshot) -> Result<()> {
108+
pub fn restore(&mut self, snapshot: Arc<Snapshot>) -> Result<()> {
109+
eprintln!("did a restore (2)");
109110
match &mut self.inner {
110111
Some(inner) => inner.restore(snapshot),
111112
None => log_then_return!("No inner MultiUseSandbox to restore"),
@@ -135,7 +136,7 @@ impl LoadedWasmSandbox {
135136

136137
pub(super) fn new(
137138
inner: MultiUseSandbox,
138-
runtime_snapshot: Snapshot,
139+
runtime_snapshot: Arc<Snapshot>,
139140
) -> Result<LoadedWasmSandbox> {
140141
metrics::gauge!(METRIC_ACTIVE_LOADED_WASM_SANDBOXES).increment(1);
141142
metrics::counter!(METRIC_TOTAL_LOADED_WASM_SANDBOXES).increment(1);

0 commit comments

Comments
 (0)