Merge pull request #244 from seddonym/read-data-map-file

seddonym · web-flow · commit fe2499723e2c · 2025-09-05T12:36:47.000+01:00
Read data map file
diff --git a/.importlinter b/.importlinter
@@ -17,6 +17,7 @@ ignore_imports =
     ; Grimp doesn't understand extension modules: it thinks grimp._rustgrimp
     ; is an object within grimp, rather than a module in its own right,
     ; so we ignore these imports here.
+    grimp.adaptors.caching -> grimp
     grimp.adaptors.graph -> grimp
     grimp.adaptors.filesystem -> grimp
     grimp.application.scanning -> grimp
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -25,6 +25,7 @@ ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.
 ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
 ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "v0.4.10" }
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0.137"
 serde_yaml = "0.9"
 unindent = "0.2.4"
 encoding_rs = "0.8.35"
@@ -38,4 +39,4 @@ default = ["extension-module"]
 
 [dev-dependencies]
 parameterized = "2.0.0"
-serde_json = "1.0.137"
+
diff --git a/rust/src/caching.rs b/rust/src/caching.rs
@@ -0,0 +1,55 @@
+use crate::errors::{GrimpError, GrimpResult};
+use crate::filesystem::get_file_system_boxed;
+use crate::import_scanning::{DirectImport, imports_by_module_to_py};
+use crate::module_finding::Module;
+use pyo3::types::PyDict;
+use pyo3::{Bound, PyAny, PyResult, Python, pyfunction};
+use std::collections::{HashMap, HashSet};
+
+/// Reads the cache file containing all the imports for a given package.
+/// Args:
+/// - filename: str
+/// - file_system: The file system interface to use. (A BasicFileSystem.)
+/// Returns Dict[Module, Set[DirectImport]]
+#[pyfunction]
+pub fn read_cache_data_map_file<'py>(
+    py: Python<'py>,
+    filename: &str,
+    file_system: Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyDict>> {
+    let file_system_boxed = get_file_system_boxed(&file_system)?;
+
+    let file_contents = file_system_boxed.read(filename)?;
+
+    let imports_by_module = parse_json_to_map(&file_contents, filename)?;
+
+    Ok(imports_by_module_to_py(py, imports_by_module))
+}
+
+pub fn parse_json_to_map(
+    json_str: &str,
+    filename: &str,
+) -> GrimpResult<HashMap<Module, HashSet<DirectImport>>> {
+    let raw_map: HashMap<String, Vec<(String, usize, String)>> = serde_json::from_str(json_str)
+        .map_err(|_| GrimpError::CorruptCache(filename.to_string()))?;
+
+    let mut parsed_map: HashMap<Module, HashSet<DirectImport>> = HashMap::new();
+
+    for (module_name, imports) in raw_map {
+        let module = Module {
+            name: module_name.clone(),
+        };
+        let import_set: HashSet<DirectImport> = imports
+            .into_iter()
+            .map(|(imported, line_number, line_contents)| DirectImport {
+                importer: module_name.clone(),
+                imported,
+                line_number,
+                line_contents,
+            })
+            .collect();
+        parsed_map.insert(module, import_set);
+    }
+
+    Ok(parsed_map)
+}
diff --git a/rust/src/errors.rs b/rust/src/errors.rs
@@ -1,4 +1,6 @@
-use crate::exceptions::{InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError};
+use crate::exceptions::{
+    CorruptCache, InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError,
+};
 use pyo3::PyErr;
 use pyo3::exceptions::PyValueError;
 use ruff_python_parser::ParseError as RuffParseError;
@@ -26,6 +28,9 @@ pub enum GrimpError {
         #[source]
         parse_error: RuffParseError,
     },
+
+    #[error("Could not use corrupt cache file {0}.")]
+    CorruptCache(String),
 }
 
 pub type GrimpResult<T> = Result<T, GrimpError>;
@@ -43,6 +48,7 @@ impl From<GrimpError> for PyErr {
             GrimpError::ParseError {
                 line_number, text, ..
             } => PyErr::new::<ParseError, _>((line_number, text)),
+            GrimpError::CorruptCache(_) => CorruptCache::new_err(value.to_string()),
         }
     }
 }
diff --git a/rust/src/exceptions.rs b/rust/src/exceptions.rs
@@ -5,6 +5,7 @@ use pyo3::prelude::*;
 create_exception!(_rustgrimp, ModuleNotPresent, PyException);
 create_exception!(_rustgrimp, NoSuchContainer, PyException);
 create_exception!(_rustgrimp, InvalidModuleExpression, PyException);
+create_exception!(_rustgrimp, CorruptCache, PyException);
 
 // We need to use here `pyclass(extends=PyException)` instead of `create_exception!`
 // since the exception contains custom data. See:
diff --git a/rust/src/filesystem.rs b/rust/src/filesystem.rs
@@ -1,5 +1,5 @@
 use itertools::Itertools;
-use pyo3::exceptions::{PyFileNotFoundError, PyUnicodeDecodeError};
+use pyo3::exceptions::{PyFileNotFoundError, PyTypeError, PyUnicodeDecodeError};
 use pyo3::prelude::*;
 use regex::Regex;
 use std::collections::HashMap;
@@ -375,3 +375,21 @@ pub fn parse_indented_file_system_string(file_system_string: &str) -> HashMap<St
 
     file_paths_map
 }
+
+#[allow(clippy::borrowed_box)]
+pub fn get_file_system_boxed<'py>(
+    file_system: &Bound<'py, PyAny>,
+) -> PyResult<Box<dyn FileSystem + Send + Sync>> {
+    let file_system_boxed: Box<dyn FileSystem + Send + Sync>;
+
+    if let Ok(py_real) = file_system.extract::<PyRef<PyRealBasicFileSystem>>() {
+        file_system_boxed = Box::new(py_real.inner.clone());
+    } else if let Ok(py_fake) = file_system.extract::<PyRef<PyFakeBasicFileSystem>>() {
+        file_system_boxed = Box::new(py_fake.inner.clone());
+    } else {
+        return Err(PyTypeError::new_err(
+            "file_system must be an instance of RealBasicFileSystem or FakeBasicFileSystem",
+        ));
+    }
+    Ok(file_system_boxed)
+}
diff --git a/rust/src/import_scanning.rs b/rust/src/import_scanning.rs
@@ -1,9 +1,8 @@
 use crate::errors::GrimpResult;
-use crate::filesystem::{FileSystem, PyFakeBasicFileSystem, PyRealBasicFileSystem};
-use crate::import_parsing;
+use crate::filesystem::FileSystem;
 use crate::module_finding::{FoundPackage, Module};
+use crate::{import_parsing, module_finding};
 use itertools::Itertools;
-use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
 use pyo3::types::{PyDict, PySet};
 /// Statically analyses some Python modules for import statements within their shared package.
@@ -13,10 +12,10 @@ use std::io::{self, ErrorKind};
 
 #[derive(Debug, Hash, Eq, PartialEq)]
 pub struct DirectImport {
-    importer: String,
-    imported: String,
-    line_number: usize,
-    line_contents: String,
+    pub importer: String,
+    pub imported: String,
+    pub line_number: usize,
+    pub line_contents: String,
 }
 
 pub fn py_found_packages_to_rust(py_found_packages: &Bound<'_, PyAny>) -> HashSet<FoundPackage> {
@@ -55,24 +54,6 @@ fn module_is_descendant(module_name: &str, potential_ancestor: &str) -> bool {
     module_name.starts_with(&format!("{potential_ancestor}."))
 }
 
-#[allow(clippy::borrowed_box)]
-pub fn get_file_system_boxed<'py>(
-    file_system: &Bound<'py, PyAny>,
-) -> PyResult<Box<dyn FileSystem + Send + Sync>> {
-    let file_system_boxed: Box<dyn FileSystem + Send + Sync>;
-
-    if let Ok(py_real) = file_system.extract::<PyRef<PyRealBasicFileSystem>>() {
-        file_system_boxed = Box::new(py_real.inner.clone());
-    } else if let Ok(py_fake) = file_system.extract::<PyRef<PyFakeBasicFileSystem>>() {
-        file_system_boxed = Box::new(py_fake.inner.clone());
-    } else {
-        return Err(PyTypeError::new_err(
-            "file_system must be an instance of RealBasicFileSystem or FakeBasicFileSystem",
-        ));
-    }
-    Ok(file_system_boxed)
-}
-
 /// Statically analyses the given module and returns a set of Modules that
 /// it imports.
 #[allow(clippy::borrowed_box)]
@@ -350,3 +331,22 @@ fn _distill_external_module(
         Some(module_name.split('.').next().unwrap().to_string())
     }
 }
+
+/// Convert the rust data structure into a Python dict[Module, set[DirectImport]].
+pub fn imports_by_module_to_py(
+    py: Python,
+    imports_by_module: HashMap<module_finding::Module, HashSet<DirectImport>>,
+) -> Bound<PyDict> {
+    let valueobjects_pymodule = PyModule::import(py, "grimp.domain.valueobjects").unwrap();
+    let py_module_class = valueobjects_pymodule.getattr("Module").unwrap();
+
+    let imports_by_module_py = PyDict::new(py);
+    for (module, imports) in imports_by_module.iter() {
+        let py_module_instance = py_module_class.call1((module.name.clone(),)).unwrap();
+        let py_imports = to_py_direct_imports(py, imports);
+        imports_by_module_py
+            .set_item(py_module_instance, py_imports)
+            .unwrap();
+    }
+    imports_by_module_py
+}
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
@@ -1,3 +1,4 @@
+mod caching;
 pub mod errors;
 pub mod exceptions;
 mod filesystem;
@@ -7,16 +8,18 @@ mod import_scanning;
 pub mod module_expressions;
 mod module_finding;
 
+use crate::caching::read_cache_data_map_file;
 use crate::errors::{GrimpError, GrimpResult};
-use crate::exceptions::{InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError};
+use crate::exceptions::{
+    CorruptCache, InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError,
+};
 use crate::filesystem::{PyFakeBasicFileSystem, PyRealBasicFileSystem};
 use crate::graph::higher_order_queries::Level;
 use crate::graph::{Graph, Module, ModuleIterator, ModuleTokenIterator};
-use crate::import_scanning::{
-    get_file_system_boxed, py_found_packages_to_rust, scan_for_imports_no_py, to_py_direct_imports,
-};
+use crate::import_scanning::{py_found_packages_to_rust, scan_for_imports_no_py};
 use crate::module_expressions::ModuleExpression;
 use derive_new::new;
+use filesystem::get_file_system_boxed;
 use itertools::Itertools;
 use pyo3::IntoPyObjectExt;
 use pyo3::exceptions::PyValueError;
@@ -29,6 +32,7 @@ use std::collections::HashSet;
 #[pymodule]
 fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(scan_for_imports))?;
+    m.add_wrapped(wrap_pyfunction!(read_cache_data_map_file))?;
     m.add_class::<GraphWrapper>()?;
     m.add_class::<PyRealBasicFileSystem>()?;
     m.add_class::<PyFakeBasicFileSystem>()?;
@@ -39,6 +43,7 @@ fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
         py.get_type::<InvalidModuleExpression>(),
     )?;
     m.add("ParseError", py.get_type::<ParseError>())?;
+    m.add("CorruptCache", py.get_type::<CorruptCache>())?;
     Ok(())
 }
 
@@ -65,8 +70,6 @@ fn scan_for_imports<'py>(
     exclude_type_checking_imports: bool,
     file_system: Bound<'py, PyAny>,
 ) -> PyResult<Bound<'py, PyDict>> {
-    let valueobjects_pymodule = PyModule::import(py, "grimp.domain.valueobjects").unwrap();
-    let py_module_class = valueobjects_pymodule.getattr("Module").unwrap();
     let file_system_boxed = get_file_system_boxed(&file_system)?;
     let found_packages_rust = py_found_packages_to_rust(&found_packages);
     let modules_rust: HashSet<module_finding::Module> = module_files
@@ -112,14 +115,7 @@ fn scan_for_imports<'py>(
     }
     let imports_by_module = imports_by_module_result.unwrap();
 
-    let imports_by_module_py = PyDict::new(py);
-    for (module, imports) in imports_by_module.iter() {
-        let py_module_instance = py_module_class.call1((module.name.clone(),)).unwrap();
-        let py_imports = to_py_direct_imports(py, imports);
-        imports_by_module_py
-            .set_item(py_module_instance, py_imports)
-            .unwrap();
-    }
+    let imports_by_module_py = import_scanning::imports_by_module_to_py(py, imports_by_module);
 
     Ok(imports_by_module_py)
 }
diff --git a/src/grimp/adaptors/caching.py b/src/grimp/adaptors/caching.py
@@ -1,4 +1,5 @@
 import hashlib
+
 import json
 import logging
 from typing import Dict, List, Optional, Set, Tuple, Type
@@ -9,6 +10,7 @@
 
 from ..application.ports.caching import Cache as AbstractCache
 from ..application.ports.caching import CacheMiss
+from grimp import _rustgrimp as rust  # type: ignore[attr-defined]
 
 logger = logging.getLogger(__name__)
 PrimitiveFormat = Dict[str, List[Tuple[str, Optional[int], str]]]
@@ -199,74 +201,22 @@ def _read_data_map_file(self) -> Dict[Module, Set[DirectImport]]:
             ),
         )
         try:
-            serialized = self.file_system.read(data_cache_filename)
+            imports_by_module = rust.read_cache_data_map_file(
+                data_cache_filename, self.file_system.convert_to_basic()
+            )
         except FileNotFoundError:
             logger.info(f"No cache file: {data_cache_filename}.")
             return {}
-
-        # Deserialize to primitives.
-        try:
-            deserialized_json = json.loads(serialized)
-            logger.info(f"Used cache data file {data_cache_filename}.")
-        except json.JSONDecodeError:
+        except rust.CorruptCache:
             logger.warning(f"Could not use corrupt cache file {data_cache_filename}.")
             return {}
 
-        primitives_map: PrimitiveFormat = self._to_primitives_data_map(deserialized_json)
-
-        return {
-            Module(name=name): {
-                DirectImport(
-                    importer=Module(name),
-                    imported=Module(import_data[0]),
-                    line_number=int(import_data[1]),  # type: ignore
-                    line_contents=import_data[2],
-                )
-                for import_data in imports_data
-            }
-            for name, imports_data in primitives_map.items()
-        }
+        logger.info(f"Used cache data file {data_cache_filename}.")
+        return imports_by_module
 
     def _build_data_cache_filename(self, found_package: FoundPackage) -> str:
         return self.file_system.join(self.cache_dir, f"{found_package.name}.data.json")
 
-    def _to_primitives_data_map(self, deserialized_json: object) -> PrimitiveFormat:
-        """
-        Convert the deserialized json from a data file to a narrower schema.
-
-        Anything that doesn't fit the schema will be removed.
-        """
-        if not isinstance(deserialized_json, dict):
-            return {}
-
-        primitives_map: PrimitiveFormat = {}
-
-        for key, value in deserialized_json.items():
-            if not isinstance(key, str):
-                continue
-            if not isinstance(value, list):
-                continue
-            primitive_imports = []
-            for deserialized_import in value:
-                try:
-                    [imported, line_number, line_contents] = deserialized_import
-                except ValueError:
-                    continue
-                try:
-                    primitive_imports.append(
-                        (
-                            str(imported),
-                            int(line_number) if line_number else None,
-                            str(line_contents),
-                        )
-                    )
-                except TypeError:
-                    continue
-
-            primitives_map[key] = primitive_imports
-
-        return primitives_map
-
     def _write_marker_files_if_not_already_there(self) -> None:
         marker_files_info = (
             (".gitignore", "# Automatically created by Grimp.\n*"),

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,6 @@`
`1`		`-use crate::exceptions::{InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError};`
	`1`	`+use crate::exceptions::{`
	`2`	`+ CorruptCache, InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError,`
	`3`	`+};`
`2`	`4`	`use pyo3::PyErr;`
`3`	`5`	`use pyo3::exceptions::PyValueError;`
`4`	`6`	`use ruff_python_parser::ParseError as RuffParseError;`
`@@ -26,6 +28,9 @@ pub enum GrimpError {`
`26`	`28`	`#[source]`
`27`	`29`	`parse_error: RuffParseError,`
`28`	`30`	`},`
	`31`	`+`
	`32`	`+ #[error("Could not use corrupt cache file {0}.")]`
	`33`	`+ CorruptCache(String),`
`29`	`34`	`}`
`30`	`35`
`31`	`36`	`pub type GrimpResult<T> = Result<T, GrimpError>;`
`@@ -43,6 +48,7 @@ impl From<GrimpError> for PyErr {`
`43`	`48`	`GrimpError::ParseError {`
`44`	`49`	`line_number, text, ..`
`45`	`50`	`} => PyErr::new::<ParseError, _>((line_number, text)),`
	`51`	`+ GrimpError::CorruptCache(_) => CorruptCache::new_err(value.to_string()),`
`46`	`52`	`}`
`47`	`53`	`}`
`48`	`54`	`}`