Adds testing for semantic lookups

.github/workflows/binaries.yml

@@ -46,10 +46,19 @@ jobs:
         shell: bash
         run: cargo build --release --verbose 2>&1 | tee -a ${{ inputs.log-dir }}/build.log
       - name: Test
+        id: test
         shell: bash
         run: |
           set -o pipefail
           cargo test --verbose  2>&1 | tee -a ${{ inputs.log-dir }}/test.log
+      - name: Fuzz Test Failures
+        if: ${{ failure() && steps.test.outcome == 'failure' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: fuzz-reproducers-${{ inputs.output_binary_name }}
+          path: target/test_output/*.fail
+          if-no-files-found: ignore
+          overwrite: true
       - name: Prepare Parity Test
         if: ${{ inputs.parity_test_path != '' }}
         uses: actions/checkout@v4

Cargo.toml

@@ -25,6 +25,19 @@ name = "dfa"
 test = false
 path = "src/dfa/main.rs"
 
+[[bin]]
+name = "dml-fuzz-runner"
+test = false
+bench = false
+doc = false
+path = "src/bin/dml-fuzz-runner.rs"
+required-features = ["fuzzing"]
+
+[[test]]
+name = "fuzzing"
+path = "tests/fuzzing.rs"
+required-features = ["fuzzing"]
+
 [dependencies]
 anyhow = "1.0"
 clap = { version = "4.2", features = ["cargo", "derive"] }
@@ -53,3 +66,11 @@ urlencoding = "2.1"
 utf8-read = "0.4"
 walkdir = "2"
 heck = "0.5"
+
+[features]
+# Internal-only: enables building the `dml-fuzz-runner` helper binary
+# used by the fuzzing tests. Not intended for end users.
+fuzzing = []
+
+[dev-dependencies]
+rand = "0.10"

src/actions/mod.rs

@@ -406,6 +406,41 @@ impl <O: Output> InitActionContext<O> {
         }
     }
 
+    /// Used in semantic lookup tests to hack together an inited context without going through an actual server
+    #[doc(hidden)]
+    pub fn new_for_testing(
+        analysis: Arc<Mutex<AnalysisStorage>>,
+        vfs: Arc<Vfs>,
+    ) -> InitActionContext<O> {
+        let shut_down = Arc::new(AtomicBool::new(false));
+        let config = Arc::new(Mutex::new(Config::default()));
+
+        InitActionContext {
+            analysis,
+            vfs,
+            analysis_queue: Arc::new(AnalysisQueue::init(Arc::clone(&shut_down))),
+            current_notifier: Arc::new(Mutex::new(None)),
+            quiescent: Arc::new(AtomicBool::new(false)),
+            workspace_roots: Arc::new(Mutex::new(Vec::new())),
+            cached_path_resolver: Arc::new(Mutex::new(None)),
+            direct_opens: Arc::new(Mutex::new(HashSet::new())),
+            compilation_info: Arc::new(Mutex::new(HashMap::new())),
+            device_active_contexts: Arc::new(Mutex::new(HashSet::new())),
+            previously_checked_contexts: Arc::new(Mutex::new(HashSet::new())),
+            prev_changes: Arc::new(Mutex::new(HashMap::new())),
+            active_waits: Arc::new(Mutex::new(Vec::new())),
+            outstanding_requests: Arc::new(Mutex::new(HashMap::new())),
+            config,
+            lint_config: Arc::new(Mutex::new(LintCfg::default())),
+            sent_warnings: Arc::new(Mutex::new(HashSet::new())),
+            jobs: Arc::new(Mutex::new(Jobs::default())),
+            client_capabilities: Arc::new(ClientCapabilities::default()),
+            has_notified_missing_builtins: false,
+            shut_down,
+            pid: std::process::id(),
+        }
+    }
+
     fn add_direct_open(&self, path: PathBuf) {
         // NOTE: from_path_buf already logs the error, no need to do it here
         let Some(canon_path) = CanonPath::from_path_buf(path) else { return };

src/analysis/mod.rs

@@ -695,6 +695,15 @@ pub enum AnalysisError {
     Cancelled,
 }
 
+impl std::fmt::Debug for AnalysisError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            AnalysisError::VFSError(v) => write!(f, "VFSError({:?})", v),
+            AnalysisError::Cancelled => write!(f, "Cancelled"),
+        }
+    }
+}
+
 pub type AnalysisProcessResult<T> = Result<T, AnalysisError>;
 
 impl From<VFSError> for AnalysisError {

src/analysis/parsing/expression.rs

@@ -1236,7 +1236,7 @@ pub fn ensure_string_concatenation(expr: &Expression) -> Vec<LocalDMLError> {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::test::*;
+    use crate::test_helpers::*;
 
     #[allow(clippy::ptr_arg)]
     fn test_expression_tree(source: &str,

src/analysis/parsing/misc.rs

@@ -665,7 +665,7 @@ pub fn objident_filter(token: TokenKind) -> bool {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::test::*;
+    use crate::test_helpers::*;
 
     #[test]
     fn initializer() {

src/analysis/parsing/statement.rs

@@ -2031,7 +2031,7 @@ pub fn dmlstatement_first_token_matcher(token: TokenKind) -> bool {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::test::*;
+    use crate::test_helpers::*;
 
     #[allow(clippy::ptr_arg)]
     fn test_statement_tree(source: &str,

src/analysis/parsing/structure.rs

@@ -2280,7 +2280,7 @@ pub fn post_parse_toplevel(toplevel: &TopAst,
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::test::*;
+    use crate::test_helpers::*;
 
     // discovered a particular case while running towards test cases
     #[test]

src/analysis/parsing/types.rs

@@ -694,7 +694,7 @@ impl CTypeDeclContent {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::test::*;
+    use crate::test_helpers::*;
 
     #[test]
     fn ctypedecl() {

src/analysis/scope.rs

@@ -112,6 +112,13 @@ impl ContextKey {
             Self::AllWithTemplate(_, _) => None,
         }
     }
+
+    pub fn as_simple_symbol(&self) -> Option<&SimpleSymbol> {
+        match self {
+            Self::Template(s) | Self::Method(s) | Self::Structure(s) => Some(s),
+            Self::AllWithTemplate(_, _) => None,
+        }
+    }
 }
 
 impl Named for ContextKey {
@@ -232,6 +239,11 @@ impl SymbolContext {
                              pos: &ZeroFilePosition,
                              acc: &mut Vec<&'t ContextKey>)
                              -> Option<&'t SimpleSymbol> {
+        if self.context.loc_span().contains_pos(pos) {
+            if let Some(simp) = self.context.as_simple_symbol() {
+                return Some(simp);
+            }
+        }
         acc.push(&self.context);
         self.subsymbols.iter()
             .find(|sub|{trace!("Considering {:?}, contains pos? {:?}",

src/analysis/templating/objects.rs

@@ -2267,6 +2267,11 @@ pub fn make_object(loc: ZeroSpan,
                    report: &mut Vec<DMLError>) -> StructureKey {
     debug!("Making object {}", identity.val);
 
+    // Capture the direct declarations before add_template_specs extends
+    // obj_specs with template-instantiated specs, to be used for decl
+    // locations later
+    let direct_decls: Vec<Arc<ObjectSpec>> = obj_specs.clone();
+
     let mut each_stmts = parent_each_stmts.clone();
     let used_ineach_locs = add_template_specs(&mut obj_specs, &each_stmts);
     add_template_ineachs(&obj_specs, &mut each_stmts);
@@ -2293,7 +2298,7 @@ pub fn make_object(loc: ZeroSpan,
            symbols.keys().map(|k|k.as_str()).collect::<Vec<&str>>());
 
     let new_obj_key = create_object_instance(Some(loc),
-                                             &obj_specs,
+                                             &direct_decls,
                                              identity, kind,
                                              &array_info, container);
 

src/bin/dml-fuzz-runner.rs

@@ -0,0 +1,83 @@
+//  © 2024 Intel Corporation
+//  SPDX-License-Identifier: Apache-2.0 and MIT
+
+//! Sub-process runner for the fuzz harness in `src/test/fuzzing_tests.rs`.
+//!
+//! Reads input bytes from stdin, runs the DML parser, and exits with one
+//! of three signal/code disciplines that the parent process classifies:
+//!
+//!   * exit 0           — parsed cleanly.
+//!   * exit 1           — Rust panic; message written to stderr as
+//!                        `PANIC: <message>`.
+//!   * killed by signal — uncaught crash (stack overflow → SIGSEGV,
+//!                        abort → SIGABRT, etc.). The parent classifies as
+//!                        `Panic` with a signal-number note.
+
+use std::io::Read;
+use std::str::FromStr;
+use std::thread;
+
+use logos::Logos;
+
+/// Stack size for the parse worker thread. Production LSP workers use
+/// Rust's default (~2 MiB on Linux), so we match that here
+const PARSE_THREAD_STACK: usize = 2 * 1024 * 1024;
+
+use dls::analysis::FileSpec;
+use dls::analysis::parsing::lexer::TokenKind;
+use dls::analysis::parsing::parser::{FileInfo, FileParser};
+use dls::analysis::parsing::structure::{parse_toplevel, post_parse_toplevel};
+use dls::analysis::parsing::tree::TreeElement;
+use dls::vfs::TextFile;
+
+fn parse_once(input: &str) {
+    let text = TextFile::from_str(input).expect("TextFile::from_str is infallible");
+    let path = std::path::PathBuf::from("fuzz.dml");
+    let file_spec = FileSpec { path: &path, file: &text };
+
+    let lexer = TokenKind::lexer(&text.text);
+    let mut parser = FileParser::new(lexer);
+    let mut file_info = FileInfo::default();
+    let ast = parse_toplevel(&mut parser, &mut file_info, file_spec);
+
+    let _ = parser.report_skips();
+    let _ = ast.report_missing();
+    let mut errors = Vec::new();
+    post_parse_toplevel(&ast, &text, &mut errors);
+}
+
+fn panic_payload_to_string(payload: Box<dyn std::any::Any + Send>) -> String {
+    if let Some(s) = payload.downcast_ref::<&str>() {
+        (*s).to_owned()
+    } else if let Some(s) = payload.downcast_ref::<String>() {
+        s.clone()
+    } else {
+        "non-string panic payload".to_owned()
+    }
+}
+
+fn main() {
+    // Suppress the default panic printer; we emit our own stderr message
+    // so the parent has a single, well-formed line to capture.
+    std::panic::set_hook(Box::new(|_| {}));
+
+    let mut input = String::new();
+    if std::io::stdin().read_to_string(&mut input).is_err() {
+        std::process::exit(2);
+    }
+
+    let result = thread::Builder::new()
+        .name("dml-fuzz-parse".into())
+        .stack_size(PARSE_THREAD_STACK)
+        .spawn(move || std::panic::catch_unwind(|| parse_once(&input)))
+        .expect("spawn parse worker")
+        .join()
+        .expect("parse worker thread join failed");
+    match result {
+        Ok(()) => std::process::exit(0),
+        Err(payload) => {
+            eprintln!("PANIC: {}", panic_payload_to_string(payload));
+            std::process::exit(1);
+        }
+    }
+}

src/file_tests.rs

@@ -19,6 +19,8 @@
 
 #[cfg(test)]
 mod test {
+    use regex::Regex;
+
     fn get_files_with_extension(extension: &'static str) ->  Vec<String> {
         let files = walkdir::WalkDir::new(".");
         files.into_iter()
@@ -37,23 +39,21 @@ mod test {
 
     fn compare_linewise(canon: &str, content: &str) -> bool {
         for (canon_line, content_line) in canon.lines().zip(content.lines()) {
-            if canon_line != content_line {
+            let re = Regex::new(&format!("^{}$", canon_line))
+                .expect("invalid regex in canon header");
+            if !re.is_match(content_line) {
                 return false;
             }
         }
         true
     }
 
-    #[test]
-    fn check_copyright_headers_rs() {
+    fn check_copyright_headers(extension: &'static str, canon: &str) {
         let mut missmatching_files = vec![];
-        for path in get_files_with_extension(".rs") {
+        for path in get_files_with_extension(extension) {
             let content = std::fs::read_to_string(&path)
-                .unwrap_or_else(|e|panic!("Could not read file {}, {}", path, e));
-            if !compare_linewise(
-"//  © 2024 Intel Corporation
-//  SPDX-License-Identifier: Apache-2.0 and MIT",
-                                 content.as_str()) {
+                .unwrap_or_else(|e| panic!("Could not read file {}, {}", path, e));
+            if !compare_linewise(canon, content.as_str()) {
                 missmatching_files.push(path.to_string());
             }
         }
@@ -63,24 +63,19 @@ mod test {
         }
     }
 
+    #[test]
+    fn check_copyright_headers_rs() {
+        check_copyright_headers(".rs",
+r"//  © \d{4}(-\d{4})? Intel Corporation
+//  SPDX-License-Identifier: Apache-2\.0 and MIT");
+    }
+
     #[test]
     fn check_copyright_headers_md() {
-        let mut missmatching_files = vec![];
-        for path in get_files_with_extension(".md") {
-            let content = std::fs::read_to_string(&path)
-                .unwrap_or_else(|e| panic!("Could not read file {}, {}", path, e));
-            if !compare_linewise(
-"<!--
-  © 2024 Intel Corporation
-  SPDX-License-Identifier: Apache-2.0 and MIT
--->",
-                                 content.as_str()) {
-                missmatching_files.push(path.to_string());
-            }
-        }
-        if !missmatching_files.is_empty() {
-            panic!("The following files do not have the correct copyright header:\n{}",
-                missmatching_files.join("\n"));
-        }
+        check_copyright_headers(".md",
+r"<!--
+  © \d{4}(-\d{4})? Intel Corporation
+  SPDX-License-Identifier: Apache-2\.0 and MIT
+-->");
     }
 }

src/lib.rs

@@ -43,7 +43,7 @@ pub mod span;
 pub mod utility;
 pub mod vfs;
 #[cfg(test)]
-pub mod test;
+pub mod test_helpers;
 pub mod logging;
 pub mod file_tests;