add canonical CMR

Author: stringhandler

cli/Cargo.lock

@@ -17,3 +17,4 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 toml_edit = { version = "0.22", features = ["serde"] }
 dotenvy = "0.15"
+simplicity-lang = { version = "0.7.0", features = ["base64"] }

cli/Cargo.toml

@@ -0,0 +1,88 @@
+use anyhow::{Context, Result};
+use simplicity::{
+    dag::{DagLike, MaxSharing},
+    jet::Elements,
+    node::{Commit, CommitNode, Inner},
+    BitIter, Cmr, Word,
+};
+
+/// Decode a Simplicity program from a base64 string and return its canonical CMR as hex.
+///
+/// The compiled `.simb` format and `simc` output are base64-encoded CommitNodes.
+pub fn canonical_cmr_from_base64(b64: &str) -> Result<String> {
+    let node = CommitNode::<Elements>::from_str(b64)
+        .context("failed to decode Simplicity CommitNode from base64")?;
+    Ok(compute_canonical_cmr(&node).to_string())
+}
+
+/// Decode a Simplicity program from raw bytes and return its canonical CMR as hex.
+#[allow(dead_code)]
+pub fn canonical_cmr_from_bytes(bytes: &[u8]) -> Result<String> {
+    let bits = BitIter::from(bytes);
+    let node = CommitNode::<Elements>::decode(bits)
+        .context("failed to decode Simplicity CommitNode")?;
+    Ok(compute_canonical_cmr(&node).to_string())
+}
+
+/// Compute the canonical CMR of an already-decoded [`CommitNode`].
+///
+/// Walks the DAG in post-order and recomputes the CMR with:
+/// - Every `Word` node replaced by an all-zeros word of the same bit-width
+/// - Every hidden-branch CMR in `AssertL`/`AssertR` replaced with `Cmr::unit()`
+///
+/// Programs that share the same template but differ only in baked-in constants
+/// (e.g. a public key parameter) will produce the same canonical CMR.
+pub fn compute_canonical_cmr(node: &CommitNode<Elements>) -> Cmr {
+    let mut cmrs: Vec<Cmr> = Vec::new();
+
+    for data in node.post_order_iter::<MaxSharing<Commit<Elements>>>() {
+        let lc = data.left_index.map(|i| cmrs[i]);
+        let rc = data.right_index.map(|i| cmrs[i]);
+
+        let canonical = match data.node.inner() {
+            Inner::Iden => Cmr::iden(),
+            Inner::Unit => Cmr::unit(),
+            Inner::InjL(_) => Cmr::injl(lc.unwrap()),
+            Inner::InjR(_) => Cmr::injr(lc.unwrap()),
+            Inner::Take(_) => Cmr::take(lc.unwrap()),
+            Inner::Drop(_) => Cmr::drop(lc.unwrap()),
+            Inner::Comp(_, _) => Cmr::comp(lc.unwrap(), rc.unwrap()),
+            Inner::Case(_, _) => Cmr::case(lc.unwrap(), rc.unwrap()),
+            Inner::Pair(_, _) => Cmr::pair(lc.unwrap(), rc.unwrap()),
+            Inner::Disconnect(_, _) => Cmr::disconnect(lc.unwrap()),
+            // AssertL(child, hidden_right_cmr): replace hidden CMR with unit.
+            Inner::AssertL(_, _) => Cmr::case(lc.unwrap(), Cmr::unit()),
+            // AssertR(hidden_left_cmr, child): sole DAG child is treated as left in iteration.
+            Inner::AssertR(_, _) => Cmr::case(Cmr::unit(), lc.unwrap()),
+            Inner::Witness(_) => Cmr::witness(),
+            Inner::Fail(entropy) => Cmr::fail(*entropy),
+            Inner::Jet(jet) => Cmr::jet(*jet),
+            // Replace word value with all-zeros of the same bit-width.
+            Inner::Word(w) => Cmr::const_word(&zero_word(w.n())),
+        };
+        cmrs.push(canonical);
+    }
+
+    cmrs.pop().expect("CommitNode is non-empty")
+}
+
+/// Create an all-zeros [`Word`] of type `2^(2^n)`.
+fn zero_word(n: u32) -> Word {
+    match n {
+        0 => Word::u1(0),
+        1 => Word::u2(0),
+        2 => Word::u4(0),
+        3 => Word::u8(0),
+        4 => Word::u16(0),
+        5 => Word::u32(0),
+        6 => Word::u64(0),
+        7 => Word::u128(0),
+        8 => Word::u256([0u8; 32]),
+        9 => Word::u512([0u8; 64]),
+        _ => {
+            let half = zero_word(n - 1);
+            half.product(zero_word(n - 1))
+                .expect("same-sized zero words can always be combined")
+        }
+    }
+}

cli/src/canonicalize.rs

@@ -1,3 +1,5 @@
+mod canonicalize;
+
 use std::collections::BTreeMap;
 use std::fs;
 use std::io::{BufRead, BufReader};
@@ -150,6 +152,17 @@ enum Commands {
         #[arg(long)]
         rebuild_docker: bool,
     },
+
+    /// Backfill canonical_cmr into all TOML entries without re-running Docker.
+    ///
+    /// Reads the compiled .simb files already sitting next to each TOML,
+    /// decodes them, computes the canonical CMR (all Word values and pruned-branch
+    /// CMRs zeroed out), and writes canonical_cmr into the TOML.
+    ///
+    /// Programs that share the same template but differ only in baked-in
+    /// constants (e.g. different public keys) will get the same canonical_cmr,
+    /// making it possible to match on-chain spends back to a known template.
+    Backfill,
 }
 
 // ---------------------------------------------------------------------------
@@ -604,6 +617,24 @@ fn apply_simc_to_toml(doc: &mut DocumentMut, s: &SimcOutput, auto_tags: &[String
     if let Some(cmr) = &s.cmr {
         doc["cmr"] = sv(cmr);
     }
+
+    // Canonical CMR: CMR recomputed after zeroing all Word values and hidden-branch
+    // CMRs. Programs sharing the same template but different baked-in constants
+    // (e.g. public keys) will have the same canonical_cmr, enabling template matching.
+    doc.remove("canonical_cmr");
+    if let Some(program_b64) = &s.program {
+        if s._error.as_deref().unwrap_or("").is_empty() {
+            match canonicalize::canonical_cmr_from_base64(program_b64) {
+                Ok(c) => {
+                    doc["canonical_cmr"] = sv(&c);
+                }
+                Err(e) => {
+                    eprintln!("  warning: canonical CMR computation failed: {e:#}");
+                }
+            }
+        }
+    }
+
     if let Some(ta) = &s.type_arrow {
         doc["type_arrow"] = sv(ta);
     }
@@ -1056,6 +1087,87 @@ fn cmd_debug(url: &str) -> Result<()> {
     Ok(())
 }
 
+fn cmd_backfill() -> Result<()> {
+    let dir = data_dir();
+    let tomls = collect_tomls(&dir);
+    let total = tomls.len();
+    let mut updated = 0usize;
+    let mut skipped = 0usize;
+
+    for toml_path in &tomls {
+        let stem = toml_path.file_stem().and_then(|s| s.to_str()).unwrap_or("?");
+
+        // Find the best available .simb: prefer the base compile, then any
+        // args/witness variant (they all share the same canonical CMR).
+        let base_simb = toml_path.with_extension("simb");
+        let simb_path = if base_simb.exists() {
+            Some(base_simb)
+        } else {
+            // Walk siblings for <stem>.*.simb
+            let dir = toml_path.parent().unwrap_or(Path::new("."));
+            fs::read_dir(dir)
+                .ok()
+                .and_then(|entries| {
+                    entries
+                        .filter_map(|e| e.ok())
+                        .map(|e| e.path())
+                        .find(|p| {
+                            p.extension().and_then(|e| e.to_str()) == Some("simb")
+                                && p.file_stem()
+                                    .and_then(|s| s.to_str())
+                                    .map(|s| s.starts_with(stem))
+                                    .unwrap_or(false)
+                        })
+                })
+        };
+
+        let Some(simb_path) = simb_path else {
+            skipped += 1;
+            continue;
+        };
+
+        let b64 = match fs::read_to_string(&simb_path) {
+            Ok(s) => s.trim().to_string(),
+            Err(e) => {
+                eprintln!("  warning: cannot read {}: {e}", simb_path.display());
+                skipped += 1;
+                continue;
+            }
+        };
+
+        let canonical_cmr = match canonicalize::canonical_cmr_from_base64(&b64) {
+            Ok(c) => c,
+            Err(e) => {
+                eprintln!("  warning: {stem}: {e:#}");
+                skipped += 1;
+                continue;
+            }
+        };
+
+        let raw = fs::read_to_string(toml_path)
+            .with_context(|| format!("cannot read {}", toml_path.display()))?;
+        let mut doc: DocumentMut = raw
+            .parse()
+            .with_context(|| format!("cannot parse {}", toml_path.display()))?;
+
+        // Skip if canonical_cmr is already correct.
+        if doc.get("canonical_cmr").and_then(|v| v.as_str()) == Some(&canonical_cmr) {
+            continue;
+        }
+
+        doc["canonical_cmr"] = Item::Value(Value::from(canonical_cmr.as_str()));
+        fs::write(toml_path, doc.to_string())
+            .with_context(|| format!("cannot write {}", toml_path.display()))?;
+        updated += 1;
+    }
+
+    println!(
+        "Done: {updated} TOMLs updated, {skipped} skipped (no .simb), {} unchanged",
+        total - updated - skipped
+    );
+    Ok(())
+}
+
 fn main() {
     let _ = dotenvy::dotenv();
 
@@ -1100,6 +1212,7 @@ fn main() {
         Commands::Debug { url } => cmd_debug(&url),
         Commands::Preprocess { url } => cmd_preprocess(&url),
         Commands::Compile { all, tag, slugs, rebuild_docker } => cmd_compile(all, tag.as_deref(), &slugs, rebuild_docker),
+        Commands::Backfill => cmd_backfill(),
     };
     if let Err(e) = result {
         eprintln!("error: {e:#}");

cli/src/main.rs

@@ -12,6 +12,7 @@ jet_set = "core"
 liquid_address = "ex1p6mnj7hpa5x9e7vgs82jyu33e79f0kqshfmf0uxx7ey4a0am740pq0clqml"
 liquid_testnet_address = "tex1p6mnj7hpa5x9e7vgs82jyu33e79f0kqshfmf0uxx7ey4a0am740pqwpd82s"
 is_redeem = false
+canonical_cmr = "6c25da58ce743ff77ea9bc9ea6df50fc51a5f7d6877d88b0798d455cb7f4eae7"
 autodetected_tags = []
 
 [jets]

data/programs/BlockstreamResearch/SimplicityHL/array_fold.toml

@@ -12,6 +12,7 @@ jet_set = "core"
 liquid_address = "ex1pcw25hlh6479se9xwjfeup597x9ujzcnv5mnjw08ehwf0qg6v989s4thx70"
 liquid_testnet_address = "tex1pcw25hlh6479se9xwjfeup597x9ujzcnv5mnjw08ehwf0qg6v989s5j9p0q"
 is_redeem = false
+canonical_cmr = "01bc069fa67262554f9db7dc87646f96b7c779573b9f5a53470366126d0bedf6"
 comments = ["From https://github.com/BlockstreamResearch/SimplicityHL/issues/153"]
 autodetected_tags = []
 

data/programs/BlockstreamResearch/SimplicityHL/array_fold_2n.toml

@@ -12,6 +12,7 @@ jet_set = "core"
 liquid_address = "ex1p305439usq06f4maelan8txnxshktvayu9z5gnwu6zrrxm9vmlufqef9ldr"
 liquid_testnet_address = "tex1p305439usq06f4maelan8txnxshktvayu9z5gnwu6zrrxm9vmlufqcshcuv"
 is_redeem = false
+canonical_cmr = "0b809a8c6f73c06e13eef8a8b76bcd90909cb5d70cd13967e5785620e90518c1"
 autodetected_tags = []
 
 [jets]

data/programs/BlockstreamResearch/SimplicityHL/cat.toml

@@ -12,6 +12,7 @@ jet_set = "core"
 liquid_address = "ex1p0xyxkr84prhfaykcu88k6z2mu2k42xlmjydjwdtdpvtxjvuajegq9x8cf3"
 liquid_testnet_address = "tex1p0xyxkr84prhfaykcu88k6z2mu2k42xlmjydjwdtdpvtxjvuajegqyl4lc7"
 is_redeem = false
+canonical_cmr = "823d2b98c7d6423cc6d48f0473c5ac4d11f18360859fcf44836685499fef68f1"
 comments = ["This program is an emulation of CTV using simplicity", "Instead of specifying the template hash as in BIP CTV,", "we require the user to specify all the components of the sighash", "that they want to commit."]
 autodetected_tags = []
 

data/programs/BlockstreamResearch/SimplicityHL/ctv.toml

@@ -12,6 +12,7 @@ type_arrow = "1 → 1"
 jet_set = "core"
 liquid_address = "ex1pphcc2shlqhzvnavkmq9960umzx94k7jgr6x3gnpkcnc38xyjf9pqc9stus"
 liquid_testnet_address = "tex1pphcc2shlqhzvnavkmq9960umzx94k7jgr6x3gnpkcnc38xyjf9pqeuzvdl"
+canonical_cmr = "e90992e6bfce117cb75f5490e50ed1258c1f901f645c7fc44d9210f1a0c4ab33"
 witnesses = ["TRANSFER_OR_TIMEOUT"]
 comments = ["docs.ivylang.org/bitcoin/language/ExampleContracts.html#escrowwithdelay", "1 * G", "2 * G", "3 * G", "ESCROW WITH DELAY", "An escrow agent can approve the movement of coins in cooperation with the", "sender or the recipient. The escrow agent cannot steal the coins for himself.", "The sender can refund her coins after a timeout.", "https://docs.ivylang.org/bitcoin/language/ExampleContracts.html#escrowwithdelay"]
 autodetected_tags = []

data/programs/BlockstreamResearch/SimplicityHL/escrow_with_delay.toml

@@ -11,6 +11,7 @@ type_arrow = "1 → 1"
 jet_set = "core"
 liquid_address = "ex1pzp4xccn92zvhh44z9qwh3ap3jnv677ympuaafmyv4urgfrp2lafs6s7k32"
 liquid_testnet_address = "tex1pzp4xccn92zvhh44z9qwh3ap3jnv677ympuaafmyv4urgfrp2lafsmfv3q9"
+canonical_cmr = "dcb49abacef82397279d8425300af4335eeff069a7171779f49945a0a8fa6ba6"
 comments = ["Add counter to streaming hash and finalize when the loop exists", "Hash bytes 0x00 to 0xff", "Hash bytes 0x0000 to 0xffff", "This takes ~10 seconds on my computer", "let ctx: Ctx8 = jet::sha_256_ctx_8_init();", "let out: Either<u256, Ctx8> = for_while::<hash_counter_16>(ctx, ());", "let expected: u256 = 0x281f79f89f0121c31db2bea5d7151db246349b25f5901c114505c18bfaa50ba1;", "assert!(jet::eq_256(expected, unwrap_left::<Ctx8>(out)));"]
 autodetected_tags = []