Adds import "path/to/file.flatdata"; support to the schema language.

docs/schema-language.md

@@ -318,3 +318,99 @@ Retrieving all edges is now as easy as this:
 ```cpp
 edges.slice(nodes[i].edges_range)
 ```
+
+## Imports
+
+Schemas can be split across multiple files using import statements.
+An import pulls in all definitions (structs, enums, constants, archives) from
+another file, making them available for use in the importing file.
+
+```cpp
+import "path/to/types.flatdata";
+```
+
+Import statements must appear at the top of the file, before any namespace or
+type definitions.
+
+### Path Resolution
+
+Import paths are resolved **relative to the file** containing the import
+statement:
+
+```cpp
+import "types.flatdata";            // same directory
+import "sub/geo_types.flatdata";    // subdirectory
+import "../shared/common.flatdata"; // parent directory
+```
+
+### Diamond and Cyclic Imports
+
+Diamond imports (the same file imported via multiple paths) are deduplicated
+automatically. Cyclic imports are also supported — a parent archive schema can
+import a child schema that imports the parent back.
+
+### Generated Code
+
+For **C++** and **Rust**, the generator uses separate compilation: only types
+from the root file are emitted, with include/import directives referencing
+the separately generated imported files. Each `.flatdata` file must be
+generated individually.
+
+For **Python**, **Dot**, and **Flatdata** output, all types are emitted
+monolithically.
+
+### Example
+
+```
+schema/
+├── types.flatdata
+└── main.flatdata
+```
+
+```cpp
+// types.flatdata
+namespace geo {
+    struct Point {
+        x : u32 : 32;
+        y : u32 : 32;
+    }
+}
+```
+
+```cpp
+// main.flatdata
+import "types.flatdata";
+namespace app {
+    archive Locations {
+        points : vector< .geo.Point >;
+    }
+}
+```
+
+Generate each file separately:
+
+```sh
+flatdata-generator -s schema/types.flatdata -g cpp -O schema/types.h
+flatdata-generator -s schema/main.flatdata -g cpp -O schema/main.h
+```
+
+The generated `main.h` will contain `#include "types.h"` and only define the
+`app::Locations` archive.
+
+### Rust Project Setup
+
+Each generated Rust file must live in its own module, with all imported schemas
+as siblings under a common parent module:
+
+```
+my_crate/
+├── build.rs
+└── src/
+    └── schema/
+        ├── mod.rs          // pub mod types; pub mod main_schema;
+        ├── types.rs        // include!(concat!(env!("OUT_DIR"), "/schema/types.rs"));
+        └── main_schema.rs  // include!(concat!(env!("OUT_DIR"), "/schema/main.rs"));
+```
+
+The generated code uses `pub use super::...::module::namespace::*;` re-exports
+to wire imported types through the module hierarchy.

flatdata-cpp/cmake/flatdata/GenerateSource.cmake

@@ -15,15 +15,23 @@ function(flatdata_generate_source TARGET_NAME SCHEMA_FILENAME OUTPUT_FILENAME)
     file(GLOB_RECURSE FLATDATA_GENERATOR_SOURCES ${FLATDATA_GENERATOR_PATH}/**/*.py)
     file(GLOB_RECURSE FLATDATA_GENERATOR_TEMPLATES ${FLATDATA_GENERATOR_PATH}/**/*.jinja2)
 
+    set(DEPFILE ${OUTPUT_FILENAME}.d)
+    set(DEPFILE_ARGS)
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.20")
+        set(DEPFILE_ARGS DEPFILE ${DEPFILE})
+    endif()
+
     add_custom_command(
         OUTPUT ${OUTPUT_FILENAME}
         COMMAND ${PYTHON3_EXECUTABLE} ${FLATDATA_GENERATOR_PATH}/generator.py
         --gen cpp
         --schema ${SCHEMA_FILENAME}
         --output-file ${OUTPUT_FILENAME}
+        --depfile ${DEPFILE}
         DEPENDS ${FLATDATA_GENERATOR_SOURCES}
         DEPENDS ${FLATDATA_GENERATOR_TEMPLATES}
         DEPENDS ${SCHEMA_FILENAME}
+        ${DEPFILE_ARGS}
         WORKING_DIRECTORY ${GENERATOR_PATH}
         COMMENT "Generating sources from flatdata schema"
     )

flatdata-cpp/test/CMakeLists.txt

@@ -12,8 +12,35 @@ flatdata_generate_source(generate_flatdata_test_case_ranges
     ${CMAKE_CURRENT_SOURCE_DIR}/../../test_cases/archives/ranges.flatdata
     ${CMAKE_CURRENT_BINARY_DIR}/generated/ranges.hpp)
 
+# Import feature test cases: generate both imported and root schemas
+# The root schema (main) produces #include "types.h", so both files must be
+# in the same generated directory.
+flatdata_generate_source(generate_import_simple_types
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../test_cases/imports/simple/types.flatdata
+    ${CMAKE_CURRENT_BINARY_DIR}/generated/imports/simple/types.h)
+
+flatdata_generate_source(generate_import_simple_main
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../test_cases/imports/simple/main.flatdata
+    ${CMAKE_CURRENT_BINARY_DIR}/generated/imports/simple/main.h)
+
+flatdata_generate_source(generate_import_cross_ns_other
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../test_cases/imports/cross_namespace/other.flatdata
+    ${CMAKE_CURRENT_BINARY_DIR}/generated/imports/cross_namespace/other.h)
+
+flatdata_generate_source(generate_import_cross_ns_main
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../test_cases/imports/cross_namespace/main.flatdata
+    ${CMAKE_CURRENT_BINARY_DIR}/generated/imports/cross_namespace/main.h)
+
 add_executable(flatdata_test ${TEST_FLATDATA_SOURCES})
-add_dependencies(flatdata_test generate_flatdata_test_code generate_flatdata_test_code2 generate_flatdata_test_case_ranges)
+add_dependencies(flatdata_test
+    generate_flatdata_test_code
+    generate_flatdata_test_code2
+    generate_flatdata_test_case_ranges
+    generate_import_simple_types
+    generate_import_simple_main
+    generate_import_cross_ns_other
+    generate_import_cross_ns_main
+)
 
 target_include_directories(flatdata_test
     PRIVATE ${Boost_INCLUDE_DIRS}

flatdata-cpp/test/ImportTest.cpp

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) 2025 HERE Europe B.V.
+ * See the LICENSE file in the root of this project for license details.
+ */
+
+// Test that code generated from schemas with imports compiles and works correctly.
+// The "simple" test case: main.flatdata imports types.flatdata
+// main.h is generated with #include "types.h" and only defines the local archive.
+// types.h defines the struct from the imported file.
+#include "imports/simple/main.h"
+
+// The "cross_namespace" test case: main.flatdata imports other.flatdata (different namespace)
+#include "imports/cross_namespace/main.h"
+
+#include <flatdata/MemoryResourceStorage.h>
+#include "catch_amalgamated.hpp"
+
+TEST_CASE( "imported_types_are_usable_in_archive", "[Import]" )
+{
+    std::shared_ptr< flatdata::ResourceStorage > storage
+        = flatdata::MemoryResourceStorage::create( );
+    auto builder = app::ABuilder::open( storage );
+    REQUIRE( builder.is_open( ) );
+
+    auto data = builder.start_data( );
+    data.grow( ).x = 42;
+    data.grow( ).y = 100;
+    data.close( );
+
+    auto archive = app::A::open( storage );
+    REQUIRE( archive.data( ).size( ) == 2 );
+    REQUIRE( archive.data( )[ 0 ].x == 42 );
+    REQUIRE( archive.data( )[ 1 ].y == 100 );
+}
+
+TEST_CASE( "cross_namespace_imported_enum_works", "[Import]" )
+{
+    std::shared_ptr< flatdata::ResourceStorage > storage
+        = flatdata::MemoryResourceStorage::create( );
+    auto builder = app::MainBuilder::open( storage );
+    REQUIRE( builder.is_open( ) );
+
+    auto entries = builder.start_entries( );
+    entries.grow( ).id = 7;
+    entries.grow( ).kind = ::defs::Kind::B;
+    entries.close( );
+
+    auto archive = app::Main::open( storage );
+    REQUIRE( archive.entries( ).size( ) == 2 );
+    REQUIRE( archive.entries( )[ 0 ].id == 7 );
+    REQUIRE( archive.entries( )[ 1 ].kind == ::defs::Kind::B );
+}

flatdata-generator/README.md

@@ -16,6 +16,26 @@ pip3 install flatdata-generator
 flatdata-generator -s locations.flatdata -g cpp -O locations.hpp
 ```
 
+### Multi-file Schemas
+
+When a schema uses `import` statements, each file should be generated
+separately. Imported types are referenced via include/import directives rather
+than being re-emitted:
+
+```sh
+# Generate shared types
+flatdata-generator -s schema/types.flatdata -g cpp -O schema/types.h
+
+# Generate main schema (will #include "types.h")
+flatdata-generator -s schema/main.flatdata -g cpp -O schema/main.h
+```
+
+For Rust, the same approach applies — each imported file becomes its own module
+with `pub use` re-exports connecting the namespaces.
+
+Python and Dot generators emit all types monolithically (no separate generation
+needed for the root file — all imported definitions are included in the output).
+
 Currently supported target languages:
 
 * C++
@@ -30,9 +50,14 @@ Currently supported target languages:
 
 The `flatdata` generator works in several stages which are clearly separated from one another and can be extended/tested in isolation:
 
-1. **Parse the source schema** file using `pyparsing` library. Grammar
+1. **Resolve imports** starting from the root schema file. The importer
+   (`importer.py`) performs a depth-first traversal of import statements,
+   deduplicating files and handling cyclic imports. The result is an ordered
+   list of resolved files with their parsed content.
+
+2. **Parse the source schema** file using `pyparsing` library. Grammar
    for the schema is defined in `grammar.py`
-2. **Construct a node tree** out of `pyparsing.ParseResults`. The node tree
+3. **Construct a node tree** out of `pyparsing.ParseResults`. The node tree
    contains entities for every construct of flatdata grammar, organized
    in hierarchical order, allowing non-tree references between nodes:
 
@@ -49,7 +74,7 @@ The `flatdata` generator works in several stages which are clearly separated fro
       -  `TypeReference` - model type dependencies, which are used during
          topological sorting at a later stage and for schema resolution.
 
-3. **Augment the tree** with structures and references that are not
+4. **Augment the tree** with structures and references that are not
    directly corresponding to `pyparsing.ParseResults` or needed to
    implement advanced features. Among these:
 
@@ -59,17 +84,17 @@ The `flatdata` generator works in several stages which are clearly separated fro
    -  **Add constant references** to all archives so that constants are
       available for schema resolution.
 
-4. **Resolve references** iterates through all references and tries to
+5. **Resolve references** iterates through all references and tries to
    find a node they refer to, either in:
 
    -  Parent scopes until (inclusive) innermost parent namespace.
    -  Root node if path is fully qualified.
 
-5. **Perform topological sorting** to detect cycles in between entities
+6. **Perform topological sorting** to detect cycles in between entities
    and to determine the order of serialization for targets that depend
    on one.
 
-6. **Generate the source code** using nodes in topological order *and/or*
+7. **Generate the source code** using nodes in topological order *and/or*
    the tree (depending on the generator architecture - recursive descent
    or iterative).
 
@@ -87,6 +112,11 @@ Node tree enforces several properties of the flatdata schema:
    participate in topological sorting of the DAG formed by the tree
    edges and edges between source and target of a `TypeReference`
 
+When building a tree from multiple files, each node is tagged with its
+`source_file` (the file it was defined in) and an `is_local` flag
+(whether it belongs to the root file being generated). This allows
+generators to filter nodes for separate compilation.
+
 ### References
 
 Reference names are mangled so they are not ambiguous with other paths

flatdata-generator/flatdata/generator/app.py

@@ -19,6 +19,7 @@
 
 from flatdata.generator.engine import Engine
 from flatdata.generator.tree.errors import FlatdataSyntaxError
+from flatdata.generator.tree.syntax_tree import SyntaxTree
 
 
 def _parse_command_line() -> argparse.Namespace:
@@ -32,6 +33,8 @@ def _parse_command_line() -> argparse.Namespace:
     parser.add_argument("-O", "--output-file", type=str, required=True,
                         default=None,
                         help="Destination file. Forces all output to be stored in one file")
+    parser.add_argument("-d", "--depfile", type=str, default=None,
+                        help="Write a Makefile-style dependency file listing all imported schemas")
     parser.add_argument("-v", "--verbose", action="store_true",
                         help="Enable verbose mode")
     parser.add_argument("--debug", action="store_true",
@@ -62,14 +65,12 @@ def _run(args: argparse.Namespace) -> None:
     _setup_logging(args)
     _check_args(args)
 
-    with open(args.schema, 'r') as input_file:
-        schema = input_file.read()
-        try:
-            engine = Engine(schema)
-            logging.debug("Tree: %s", engine.tree)
-        except FlatdataSyntaxError as ex:
-            logging.fatal("Error reading schema: %s ", ex)
-            sys.exit(1)
+    try:
+        engine = Engine.from_file(args.schema)
+        logging.debug("Tree: %s", engine.tree)
+    except FlatdataSyntaxError as ex:
+        logging.fatal("Error reading schema: %s ", ex)
+        sys.exit(1)
 
     try:
         logging.info("Generating %s...", args.gen)
@@ -85,6 +86,26 @@ def _run(args: argparse.Namespace) -> None:
         output.write(output_content)
         logging.info("Code for %s is written to %s", args.gen, args.output_file)
 
+    if args.depfile:
+        _write_depfile(args.depfile, args.output_file, args.schema, engine.tree)
+
+
+def _write_depfile(depfile_path: str, output_file: str, schema_file: str,
+                   tree: 'SyntaxTree') -> None:
+    """Write a Makefile-style depfile listing all schema dependencies."""
+    deps = [os.path.abspath(schema_file)]
+    # source_file_map keys are absolute paths of all imported files
+    deps.extend(sorted(tree.source_file_map.keys()))
+
+    # Escape spaces in paths for Make syntax
+    def escape(p: str) -> str:
+        return p.replace(" ", "\\ ")
+
+    dep_str = " ".join(escape(d) for d in deps)
+    with open(depfile_path, "w") as f:
+        f.write(f"{escape(output_file)}: {dep_str}\n")
+    logging.info("Depfile written to %s", depfile_path)
+
 
 def main() -> None:
     """Entrypoint"""

flatdata-generator/flatdata/generator/engine.py

@@ -6,7 +6,7 @@
 import types
 from typing import overload
 
-from flatdata.generator.tree.builder import build_ast
+from flatdata.generator.tree.builder import build_ast, build_ast_from_file
 from flatdata.generator.tree.nodes.trivial.namespace import Namespace
 from flatdata.generator.tree.nodes.node import Node
 from flatdata.generator.tree.syntax_tree import SyntaxTree
@@ -40,6 +40,17 @@ def available_generators(cls) -> list[str]:
         """
         return list(cls._GENERATORS.keys())
 
+    @classmethod
+    def from_file(cls, path: str) -> 'Engine':
+        """
+        Create Engine from a schema file, resolving imports.
+        :raises FlatdataSyntaxError
+        """
+        engine = cls.__new__(cls)
+        engine.tree = build_ast_from_file(path)
+        engine.schema = engine.tree.root_schema or ""
+        return engine
+
     def __init__(self, schema: str) -> None:
         """
         Instantiates generator engine for a given schema.