Minor updates/improvements to wast subcommand

ci/miri-wast.sh

@@ -25,4 +25,5 @@ cargo run -- wast --target pulley64 --precompile-save ./miri-wast "$@" \
 MIRIFLAGS="$MIRIFLAGS -Zmiri-disable-isolation -Zmiri-permissive-provenance" \
   cargo miri run -- wast -Ccache=n --target pulley64 --precompile-load ./miri-wast "$@" \
   -O memory-init-cow=n \
-  -W function-references,gc
+  -W function-references,gc \
+  --ignore-error-messages

crates/fuzzing/src/oracles.rs

@@ -765,6 +765,7 @@ pub fn wast_test(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<()> {
     let mut wast_context = WastContext::new(&engine, async_, move |store| {
         fuzz_config.configure_store_epoch_and_fuel(store);
     });
+    wast_context.ignore_error_messages(test.config.spec_test.unwrap_or(false));
     wast_context
         .register_spectest(&wasmtime_wast::SpectestConfig {
             use_shared_memory: true,

crates/test-util/src/wast.rs

@@ -472,10 +472,6 @@ impl WastTest {
             // changes
             "test/async/same-component-stream-future.wast",
             "test/async/trap-if-block-and-sync.wast",
-            // These tests assert different errors and aren't updated for
-            // memory64.
-            "test/wasm-tools/memory64.wast",
-            "test/wasm-tools/resources.wast",
         ];
         if unsupported.iter().any(|part| self.path.ends_with(part)) {
             return true;

crates/wast/src/wast.rs

@@ -35,6 +35,7 @@ pub struct WastContext {
 
     modules_by_filename: Arc<HashMap<String, Vec<u8>>>,
     configure_store: Arc<dyn Fn(&mut Store<()>) + Send + Sync>,
+    ignore_error_messages: bool,
 }
 
 enum Outcome<T = Results> {
@@ -145,13 +146,21 @@ impl WastContext {
             precompile_load: None,
             modules_by_filename: Arc::default(),
             configure_store: Arc::new(configure),
+            ignore_error_messages: false,
         }
     }
 
     fn engine(&self) -> &Engine {
         self.core_linker.engine()
     }
 
+    /// Configures whether or not error messages are ignored in directives like
+    /// `assert_invalid`.
+    pub fn ignore_error_messages(&mut self, ignore: bool) -> &mut Self {
+        self.ignore_error_messages = ignore;
+        self
+    }
+
     /// Saves precompiled modules/components into `path` instead of executing
     /// test directives.
     pub fn precompile_save(&mut self, path: impl AsRef<Path>) -> &mut Self {
@@ -733,10 +742,7 @@ impl WastContext {
                     Ok(_) => bail!("expected module to fail to build"),
                     Err(e) => e,
                 };
-                let error_message = format!("{err:?}");
-                if !is_matching_assert_invalid_error_message(filename, &text, &error_message) {
-                    bail!("assert_invalid: expected \"{text}\", got \"{error_message}\"",)
-                }
+                self.match_error_message(&text, err)?;
             }
             AssertMalformed {
                 file,
@@ -757,10 +763,7 @@ impl WastContext {
                     Ok(_) => bail!("expected module to fail to link"),
                     Err(e) => e,
                 };
-                let error_message = format!("{err:?}");
-                if !is_matching_assert_invalid_error_message(filename, &text, &error_message) {
-                    bail!("assert_unlinkable: expected {text}, got {error_message}",)
-                }
+                self.match_error_message(&text, err)?;
             }
             AssertException { line: _, action } => {
                 let result = self.perform_action(&action)?;
@@ -805,6 +808,7 @@ impl WastContext {
                     precompile_load: self.precompile_load.clone(),
                     precompile_save: self.precompile_save.clone(),
                     configure_store: self.configure_store.clone(),
+                    ignore_error_messages: self.ignore_error_messages,
                 };
                 let child = scope.spawn(move || child_cx.run_directives(commands, filename));
                 threads.insert(name.to_string(), child);
@@ -862,25 +866,15 @@ impl WastContext {
         self.generate_dwarf = enable;
         self
     }
-}
 
-fn is_matching_assert_invalid_error_message(test: &str, expected: &str, actual: &str) -> bool {
-    if actual.contains(expected) {
-        return true;
-    }
-
-    // Historically wasmtime/wasm-tools tried to match the upstream error
-    // message. This generally led to a large sequence of matches here which is
-    // not easy to maintain and is particularly difficult when test suites and
-    // proposals conflict with each other (e.g. one asserts one error message
-    // and another asserts a different error message). Overall we didn't benefit
-    // a whole lot from trying to match errors so just assume the error is
-    // roughly the same and otherwise don't try to match it.
-    if test.contains("spec_testsuite") {
-        return true;
+    fn match_error_message(&self, expected: &str, err: wasmtime::Error) -> Result<()> {
+        if self.ignore_error_messages {
+            return Ok(());
+        }
+        let actual = format!("{err:?}");
+        if actual.contains(expected) {
+            return Ok(());
+        }
+        bail!("assert_invalid: expected \"{expected}\", got \"{actual}\"",)
     }
-
-    // we are in control over all non-spec tests so all the error messages
-    // there should exactly match the `assert_invalid` or such
-    false
 }

src/commands/wast.rs

@@ -42,6 +42,11 @@ pub struct WastCommand {
     /// to import.
     #[arg(long = "wasmtime-builtins")]
     wasmtime_builtins: bool,
+
+    /// Whether or not to ignore error messages in directives like
+    /// `assert_invalid`.
+    #[arg(long)]
+    ignore_error_messages: bool,
 }
 
 impl WastCommand {
@@ -52,6 +57,13 @@ impl WastCommand {
         let async_ = optional_flag_with_default(self.async_, true);
         let mut config = self.common.config(None)?;
         config.shared_memory(true);
+
+        let generate_dwarf = optional_flag_with_default(self.generate_dwarf, true);
+        // When DWARF is being generated go ahead and enable backtrace details
+        // unconditionally as that's generally what's intended.
+        if generate_dwarf {
+            config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
+        }
         let engine = Engine::new(&config)?;
         let mut wast_context = WastContext::new(
             &engine,
@@ -71,13 +83,14 @@ impl WastCommand {
             },
         );
 
-        wast_context.generate_dwarf(optional_flag_with_default(self.generate_dwarf, true));
+        wast_context.generate_dwarf(generate_dwarf);
         wast_context
             .register_spectest(&SpectestConfig {
                 use_shared_memory: true,
                 suppress_prints: false,
             })
             .expect("error instantiating \"spectest\"");
+        wast_context.ignore_error_messages(self.ignore_error_messages);
 
         if let Some(path) = &self.precompile_save {
             wast_context.precompile_save(path);

tests/wast.rs

@@ -276,6 +276,12 @@ fn run_wast(test: &WastTest, config: WastConfig) -> wasmtime::Result<()> {
                 use_shared_memory: true,
                 suppress_prints: true,
             })?;
+
+            // Ignore error messages for spec tests because Wasmtime will often
+            // differ in exact wording from the upstream spec interpreter.
+            if test.config.spec_test.unwrap_or(false) {
+                wast_context.ignore_error_messages(true);
+            }
             if test
                 .path
                 .to_str()