Distiled cpu guesslang implementation for language detection

.github/workflows/main.yml

@@ -20,13 +20,13 @@ jobs:
     with:
       all-features: false
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
 
   test:
     uses: ealmloff/dioxus-ci/.github/workflows/test.yml@main
     with:
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
 
   fmt:
     uses: ealmloff/dioxus-ci/.github/workflows/fmt.yml@main
@@ -36,11 +36,34 @@ jobs:
     with:
       all-features: false
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
 
   clippy:
     uses: ealmloff/dioxus-ci/.github/workflows/clippy.yml@main
     with:
       all-features: false
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
+
+  wasm-smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install wasm target
+        run: rustup target add wasm32-unknown-unknown
+      - name: Build scalar wasm smoke
+        run: cargo build -p dioxus-code-language-detection --example wasm_smoke --target wasm32-unknown-unknown --release
+      - name: Run scalar wasm smoke
+        run: node scripts/run-wasm-smoke.mjs target/wasm32-unknown-unknown/release/examples/wasm_smoke.wasm
+      - name: Build SIMD wasm smoke
+        run: cargo build -p dioxus-code-language-detection --example wasm_smoke --target wasm32-unknown-unknown --release
+        env:
+          RUSTFLAGS: -Ctarget-feature=+simd128
+      - name: Run SIMD wasm smoke
+        run: node scripts/run-wasm-smoke.mjs target/wasm32-unknown-unknown/release/examples/wasm_smoke.wasm
+      - name: Build relaxed SIMD wasm smoke
+        run: cargo build -p dioxus-code-language-detection --example wasm_smoke --target wasm32-unknown-unknown --release
+        env:
+          RUSTFLAGS: -Ctarget-feature=+simd128,+relaxed-simd
+      - name: Run relaxed SIMD wasm smoke
+        run: node scripts/run-wasm-smoke.mjs target/wasm32-unknown-unknown/release/examples/wasm_smoke.wasm

.github/workflows/nightly.yml

@@ -19,22 +19,38 @@ jobs:
       toolchain: nightly
       all-features: false
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
 
   test:
     uses: ealmloff/dioxus-ci/.github/workflows/test.yml@main
     with:
       toolchain: nightly
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
 
   clippy:
     uses: ealmloff/dioxus-ci/.github/workflows/clippy.yml@main
     with:
       toolchain: nightly
       all-features: false
       no-default-features: true
-      features: dioxus-code/all-languages dioxus-code-docsite/web dioxus-code-live-input/web
+      features: dioxus-code/all-languages dioxus-code-editor/web dioxus-code-docsite/web dioxus-code-live-input/web
+
+  miri:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install nightly miri
+        run: rustup toolchain install nightly --profile minimal --component miri
+      - name: Test language detection with miri
+        run: cargo +nightly miri test -p dioxus-code-language-detection --lib
+
+  handrolled-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run language detection fuzz tests
+        run: cargo test -p dioxus-code-language-detection handrolled_fuzz -- --include-ignored
 
   web-demo:
     uses: ealmloff/dioxus-ci/.github/workflows/web-build.yml@main

.gitignore

@@ -1 +1,4 @@
 /target
+.codex
+__pycache__/
+*.pyc

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "code-editor",
     "dioxus-code-macro",
     "docsite",
+    "language-detection",
     "examples/basic",
     "examples/editor",
     "examples/live-input",
@@ -12,7 +13,7 @@ members = [
 resolver = "3"
 
 [workspace.package]
-version = "0.1.0"
+version = "0.0.1"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/ealmloff/dioxus-code"
@@ -21,8 +22,9 @@ keywords = ["dioxus", "code", "highlighting", "editor"]
 categories = ["gui", "web-programming"]
 
 [workspace.dependencies]
-dioxus-code = { version = "0.1.0", path = "." }
-dioxus-code-editor = { version = "0.1.0", path = "code-editor", default-features = false }
+dioxus-code = { version = "0.0.1", path = "." }
+dioxus-code-editor = { version = "0.0.1", path = "code-editor", default-features = false }
+dioxus-code-language-detection = { version = "0.0.1", path = "language-detection" }
 
 [package]
 name = "dioxus-code"
@@ -43,7 +45,11 @@ rustdoc-args = ["--cfg", "docsrs"]
 [features]
 default = ["macro"]
 macro = ["dep:dioxus-code-macro", "dioxus-code-macro/lang-rust"]
-runtime = ["arborium/lang-rust", "dep:arborium-tree-sitter"]
+runtime = [
+    "arborium/lang-rust",
+    "dep:arborium-tree-sitter",
+    "dep:dioxus-code-language-detection",
+]
 all-languages = [
     "runtime",
     "arborium/all-languages",
@@ -259,7 +265,8 @@ arborium = { version = "2.16.0", default-features = false }
 arborium-theme = "2.16.0"
 arborium-tree-sitter = { version = "2.16.0", optional = true }
 dioxus = { version = "0.7.0", default-features = false, features = ["lib"] }
-dioxus-code-macro = { version = "0.1.0", path = "dioxus-code-macro", default-features = false, optional = true }
+dioxus-code-macro = { version = "0.0.1", path = "dioxus-code-macro", default-features = false, optional = true }
+dioxus-code-language-detection = { workspace = true, optional = true }
 
 [build-dependencies]
 arborium = { version = "2.16.0", default-features = false }

README.md

@@ -21,8 +21,8 @@ A small Dioxus component for rendering source code with proper highlighting. Par
 
 Two ways to highlight:
 
-- **[`code!`] macro** — parses at compile time and embeds the highlighted spans. Default.
-- **[`SourceCode`]** — parses at runtime. Opt in with the `runtime` feature for dynamic source text.
+- **[`code!`] macro** — parses at compile time. The runtime ships only the spans, no parser. Default.
+- **[`SourceCode`]** — parses at runtime. Opt in with the `runtime` feature when the source isn't known until the user types it.
 
 ## Quick start
 
@@ -51,7 +51,7 @@ When the file extension is ambiguous, pass [`CodeOptions::builder`] with [`CodeO
 
 ## Runtime highlighting
 
-For editor-style use cases with dynamic source text:
+For editor-style use cases where the source isn't known at compile time:
 
 ```toml
 [dependencies]
@@ -60,23 +60,24 @@ dioxus-code = { version = "0.1", features = ["runtime"] }
 
 ```rust
 # use dioxus::prelude::*;
-use dioxus_code::{Code, Language, SourceCode, Theme};
+use dioxus_code::{Code, CodeOptions, Language, SourceCode, Theme};
 # let user_input = String::new();
 # let _ =
 rsx! {
     Code {
-        src: SourceCode::new(Language::Rust, user_input),
+        src: SourceCode::new(user_input)
+            .with_options(CodeOptions::builder().with_language(Language::Rust)),
         theme: Theme::GITHUB_LIGHT,
     }
 }
 # ;
 ```
 
-Pass a [`Language`] variant when building [`SourceCode`]. The `runtime` feature includes Rust; enable the matching `lang-*` feature, or `all-languages`, for additional grammars.
+Language can be set explicitly with the same [`CodeOptions`] builder used by [`code!`], or auto-detected from the source. The default `runtime` feature includes Rust; pass `lang-python`, `lang-toml`, or `all-languages` for the rest.
 
 ## Editor
 
-[`dioxus-code-editor`] is a sibling crate that pairs the highlighter with a textarea input layer:
+[`dioxus-code-editor`] is a sibling crate that pairs the highlighter with a `contenteditable` input layer:
 
 ```rust
 # use dioxus::prelude::*;
@@ -130,7 +131,7 @@ Code {
 
 ```sh
 dx serve --example dioxus-code-basic       # macro + runtime side by side
-dx serve --example dioxus-code-macro-only  # compile-time highlighted spans
+dx serve --example dioxus-code-macro-only  # compile-time only, no parser in the binary
 dx serve --example dioxus-code-live-input  # textarea bound to runtime highlighter
 ```
 

build.rs

@@ -97,25 +97,6 @@ fn main() {
         ));
     }
 
-    generated.push_str(
-        r#"    /// Every syntax theme, in declaration order.
-    ///
-    /// ```rust
-    /// use dioxus_code::Theme;
-    /// assert!(Theme::ALL.contains(&Theme::TOKYO_NIGHT));
-    /// ```
-    pub const ALL: &'static [Theme] = &[
-"#,
-    );
-    for theme in &themes {
-        generated.push_str(&format!("        Self::{},\n", theme.const_name));
-    }
-    generated.push_str(
-        r#"    ];
-
-"#,
-    );
-
     generated.push_str(
         r#"}
 "#,

code-editor/Cargo.toml

@@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
 
 [dependencies]
 dioxus = { version = "0.7.0", default-features = false, features = ["lib"] }
-dioxus-code = { workspace = true, features = ["runtime"] }
+dioxus-code = { workspace = true, features = ["lang-toml", "lang-python"] }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen = "0.2"
@@ -30,4 +30,6 @@ web-sys = { version = "0.3", features = [
 ] }
 
 [features]
-all-languages = ["dioxus-code/all-languages"]
+default = ["desktop"]
+desktop = ["dioxus/desktop", "dioxus/launch"]
+web = ["dioxus/web", "dioxus/launch"]

code-editor/README.md

@@ -23,7 +23,7 @@ A controlled, syntax-highlighted code editor for Dioxus. Pairs an editable input
 
 ```toml
 [dependencies]
-dioxus-code-editor = "0.1.0"
+dioxus-code-editor = "0.0.1"
 ```
 
 ```rust
@@ -53,7 +53,7 @@ The component is controlled — drive [`CodeEditorProps::value`] from your own s
 | prop | description |
 |---|---|
 | [`CodeEditorProps::value`] | Current editor contents. |
-| [`CodeEditorProps::language`] | Syntax grammar selection. Pass a [`Language`] variant (for example [`Language::Rust`]) or use [`Language::from_slug`] for runtime slugs. |
+| [`CodeEditorProps::language`] | Tree-sitter grammar selection. Pass a [`Language`] variant (for example [`Language::Rust`]) or use [`Language::from_slug`] for custom slugs. |
 | [`CodeEditorProps::theme`] | Syntax theme selection shared with [`dioxus-code`](https://crates.io/crates/dioxus-code); accepts [`Theme`] or [`CodeTheme`]. |
 | [`CodeEditorProps::line_numbers`] | Show a one-based line gutter. Defaults to `true`. |
 | [`CodeEditorProps::read_only`] | Disable editing while preserving highlighting. |