TEDEFO-4930 Add #include directive support to EFX translators

src/main/java/eu/europa/ted/efx/EfxTranslatorOptions.java

@@ -6,6 +6,7 @@
 import java.util.List;
 import java.util.Locale;
 
+import eu.europa.ted.efx.interfaces.IncludedFileResolver;
 import eu.europa.ted.efx.interfaces.TranslatorOptions;
 import eu.europa.ted.efx.model.DecimalFormat;
 
@@ -42,6 +43,7 @@ public class EfxTranslatorOptions implements TranslatorOptions {
     private final String userDefinedFunctionNamespace;
     private final boolean profilerEnabled;
     private final Path profilerOutputPath;
+    private final IncludedFileResolver includedFileResolver;
 
     public EfxTranslatorOptions(DecimalFormat symbols) {
         this(DEFAULT_PROFILER_ENABLED, DEFAULT_PROFILER_OUTPUT_PATH, DEFAULT_UDF_NAMESPACE, symbols, Locale.ENGLISH);
@@ -68,11 +70,16 @@ public EfxTranslatorOptions(String udfNamespace, DecimalFormat symbols, Locale p
     }
 
     public EfxTranslatorOptions(boolean profilerEnabled, Path profilerOutputPath, String udfNamespace, DecimalFormat symbols, Locale primaryLocale, Locale... otherLocales) {
+        this(profilerEnabled, profilerOutputPath, udfNamespace, symbols, null, primaryLocale, otherLocales);
+    }
+
+    public EfxTranslatorOptions(boolean profilerEnabled, Path profilerOutputPath, String udfNamespace, DecimalFormat symbols, IncludedFileResolver includedFileResolver, Locale primaryLocale, Locale... otherLocales) {
         this.userDefinedFunctionNamespace = udfNamespace;
         this.symbols = symbols;
         this.primaryLocale = primaryLocale;
         this.profilerEnabled = profilerEnabled;
         this.profilerOutputPath = profilerOutputPath;
+        this.includedFileResolver = includedFileResolver;
         this.otherLocales = new ArrayList<>(Arrays.asList(otherLocales));
     }
 
@@ -130,4 +137,9 @@ public boolean isProfilerEnabled() {
     public Path getProfilerOutputPath() {
         return this.profilerOutputPath;
     }
+
+    @Override
+    public IncludedFileResolver getIncludedFileResolver() {
+        return this.includedFileResolver;
+    }
 }

src/main/java/eu/europa/ted/efx/exceptions/InvalidUsageException.java

@@ -29,7 +29,8 @@ public enum ErrorCode {
         INVALID_NOTICE_SUBTYPE_TOKEN,
         FIELD_NOT_WITHHOLDABLE,
         TEMPLATE_ONLY_FUNCTION,
-        UNSUPPORTED_REGEX_CONSTRUCT
+        UNSUPPORTED_REGEX_CONSTRUCT,
+        CIRCULAR_INCLUDE
     }
 
     private static final String SHORTHAND_REQUIRES_CODE_OR_INDICATOR = "Indirect label reference shorthand #{%1$s}, requires a field of type 'code' or 'indicator'. Field %1$s is of type %2$s.";
@@ -39,6 +40,7 @@ public enum ErrorCode {
     private static final String FIELD_NOT_WITHHOLDABLE = "Field '%s' is always published and cannot be withheld from publication.";
     private static final String TEMPLATE_ONLY_FUNCTION = "Function '%s' can only be used in templates, not in expressions or validation rules.";
     private static final String UNSUPPORTED_REGEX_CONSTRUCT = "Invalid regex pattern %s at position %d: %s";
+    private static final String CIRCULAR_INCLUDE = "Circular #include detected: '%s'.";
 
     private final ErrorCode errorCode;
 
@@ -83,4 +85,8 @@ public static InvalidUsageException templateOnlyFunction(ParserRuleContext ctx,
     public static InvalidUsageException unsupportedRegexConstruct(String pattern, int position, String reason) {
         return new InvalidUsageException(ErrorCode.UNSUPPORTED_REGEX_CONSTRUCT, UNSUPPORTED_REGEX_CONSTRUCT, pattern, position, reason);
     }
+
+    public static InvalidUsageException circularInclude(String path) {
+        return new InvalidUsageException(ErrorCode.CIRCULAR_INCLUDE, CIRCULAR_INCLUDE, path);
+    }
 }

src/main/java/eu/europa/ted/efx/exceptions/TranslatorConfigurationException.java

@@ -29,7 +29,9 @@ public enum ErrorCode {
         UNHANDLED_VARIABLE_CONTEXT,
         UNHANDLED_PRIVACY_SETTING,
         UNHANDLED_LINKED_FIELD_PROPERTY,
-        UNHANDLED_PREDICATE_CONTEXT
+        UNHANDLED_PREDICATE_CONTEXT,
+        INCLUDE_RESOLVER_NOT_CONFIGURED,
+        UNRESOLVED_INCLUDE_DIRECTIVE
     }
 
     private static final String TYPE_NOT_REGISTERED =
@@ -78,6 +80,14 @@ public enum ErrorCode {
         "If the grammar was updated to allow predicates in new contexts, " +
         "add a handler for this case in enterPredicate().";
 
+    private static final String INCLUDE_RESOLVER_NOT_CONFIGURED =
+        "EFX rules contain #include directives but no IncludedFileResolver is configured. " +
+        "Pass an IncludedFileResolver via TranslatorOptions to enable include resolution.";
+
+    private static final String UNRESOLVED_INCLUDE_DIRECTIVE =
+        "Unresolved #include directive '%s' found during preprocessing. " +
+        "Include resolution may have been skipped or failed silently.";
+
     private final ErrorCode errorCode;
 
     private TranslatorConfigurationException(ErrorCode errorCode, String template, Object... args) {
@@ -124,4 +134,12 @@ public static TranslatorConfigurationException unhandledLinkedFieldProperty(Stri
     public static TranslatorConfigurationException unhandledPredicateContext(String contextClassName) {
         return new TranslatorConfigurationException(ErrorCode.UNHANDLED_PREDICATE_CONTEXT, UNHANDLED_PREDICATE_CONTEXT, contextClassName);
     }
+
+    public static TranslatorConfigurationException includeResolverNotConfigured() {
+        return new TranslatorConfigurationException(ErrorCode.INCLUDE_RESOLVER_NOT_CONFIGURED, INCLUDE_RESOLVER_NOT_CONFIGURED);
+    }
+
+    public static TranslatorConfigurationException unresolvedIncludeDirective(String path) {
+        return new TranslatorConfigurationException(ErrorCode.UNRESOLVED_INCLUDE_DIRECTIVE, UNRESOLVED_INCLUDE_DIRECTIVE, path);
+    }
 }

src/main/java/eu/europa/ted/efx/interfaces/IncludedFileResolver.java

@@ -0,0 +1,25 @@
+package eu.europa.ted.efx.interfaces;
+
+import java.io.IOException;
+
+/**
+ * Resolves {@code #include} directive paths to their text content.
+ *
+ * <p>
+ * Implementations determine how include paths are mapped to file contents. For example, a
+ * file-system resolver may resolve paths relative to a base directory, while a database-backed
+ * resolver may look up the content by name.
+ * </p>
+ */
+@FunctionalInterface
+public interface IncludedFileResolver {
+
+  /**
+   * Resolves the given include path and returns the text content of the included file.
+   *
+   * @param path The include path as specified in the {@code #include} directive (without quotes).
+   * @return The text content of the included file.
+   * @throws IOException If the path cannot be resolved or the content cannot be read.
+   */
+  String resolve(String path) throws IOException;
+}

src/main/java/eu/europa/ted/efx/interfaces/TranslatorOptions.java

@@ -26,8 +26,35 @@ public interface TranslatorOptions {
 
     /**
      * Returns the output path for EFX profiling results.
-     * 
+     *
      * @return Path where profiling results should be written, or null if no file output is desired
      */
     public Path getProfilerOutputPath();
+
+    /**
+     * Returns the include resolver for resolving {@code #include} directives in rules files.
+     *
+     * @return The include resolver, or null if include resolution is not configured.
+     */
+    default IncludedFileResolver getIncludedFileResolver() {
+        return null;
+    }
+
+    /**
+     * Returns a new {@link TranslatorOptions} that delegates all methods to this instance
+     * but overrides the {@link IncludedFileResolver}.
+     */
+    static TranslatorOptions withResolver(TranslatorOptions delegate, IncludedFileResolver resolver) {
+        return new TranslatorOptions() {
+            @Override public DecimalFormat getDecimalFormat() { return delegate.getDecimalFormat(); }
+            @Override public String getPrimaryLanguage2LetterCode() { return delegate.getPrimaryLanguage2LetterCode(); }
+            @Override public String getPrimaryLanguage3LetterCode() { return delegate.getPrimaryLanguage3LetterCode(); }
+            @Override public String[] getAllLanguage2LetterCodes() { return delegate.getAllLanguage2LetterCodes(); }
+            @Override public String[] getAllLanguage3LetterCodes() { return delegate.getAllLanguage3LetterCodes(); }
+            @Override public String getUserDefinedFunctionNamespace() { return delegate.getUserDefinedFunctionNamespace(); }
+            @Override public boolean isProfilerEnabled() { return delegate.isProfilerEnabled(); }
+            @Override public Path getProfilerOutputPath() { return delegate.getProfilerOutputPath(); }
+            @Override public IncludedFileResolver getIncludedFileResolver() { return resolver; }
+        };
+    }
 }

src/main/java/eu/europa/ted/efx/sdk2/EfxExpressionTranslatorV2.java

@@ -3697,6 +3697,16 @@ public void exitLateBoundSequenceFromConcatenatedIterations(LateBoundSequenceFro
 
     // #endregion Scope management --------------------------------------------
 
+    // #region Include directive guard -----------------------------------------
+
+    @Override
+    public void exitIncludeDirective(IncludeDirectiveContext ctx) {
+      String path = ctx.IncludePath() != null ? ctx.IncludePath().getText().trim() : "<unknown>";
+      throw TranslatorConfigurationException.unresolvedIncludeDirective(path);
+    }
+
+    // #endregion Include directive guard --------------------------------------
+
   }
 
   // #endregion Pre-processing ------------------------------------------------

src/main/java/eu/europa/ted/efx/sdk2/EfxRulesTranslatorV2.java

@@ -15,13 +15,15 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.Map;
 
 import org.antlr.v4.runtime.BaseErrorListener;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.antlr.v4.runtime.tree.ParseTreeWalker;
 import org.slf4j.Logger;
@@ -30,6 +32,7 @@
 import eu.europa.ted.eforms.sdk.component.SdkComponent;
 import eu.europa.ted.eforms.sdk.component.SdkComponentType;
 import eu.europa.ted.efx.interfaces.EfxRulesTranslator;
+import eu.europa.ted.efx.interfaces.IncludedFileResolver;
 import eu.europa.ted.efx.interfaces.ScriptGenerator;
 import eu.europa.ted.efx.interfaces.SymbolResolver;
 import eu.europa.ted.efx.interfaces.TranslatorOptions;
@@ -113,6 +116,13 @@ public Map<String, String> translateRules(Path pathname, TranslatorOptions optio
       throws IOException {
     logger.debug("Translating EFX rules from file: {}", pathname);
     CharStream input = CharStreams.fromPath(pathname);
+
+    // Default to filesystem-based include resolution relative to the input file
+    if (options.getIncludedFileResolver() == null) {
+      Path baseDir = pathname.toAbsolutePath().getParent();
+      options = TranslatorOptions.withResolver(options, new FileSystemIncludedFileResolver(baseDir));
+    }
+
     return translateRulesFromCharStream(input, options);
   }
 
@@ -123,8 +133,7 @@ public Map<String, String> translateRules(String rules, TranslatorOptions option
     try {
       return translateRulesFromCharStream(input, options);
     } catch (IOException e) {
-      // This should never happen when reading from a string
-      throw new RuntimeException("Unexpected IOException while translating rules from string", e);
+      throw new UncheckedIOException("Include resolution failed during rules translation", e);
     }
   }
 
@@ -149,7 +158,7 @@ private Map<String, String> translateRulesFromCharStream(CharStream input,
     logger.debug("Parsing EFX rules");
 
     // New in EFX-2: rules preprocessing
-    final RulesPreprocessor preprocessor = this.new RulesPreprocessor(input);
+    final RulesPreprocessor preprocessor = this.new RulesPreprocessor(input, options.getIncludedFileResolver());
     final String preprocessedRules = preprocessor.processRules();
 
     // Now parse the preprocessed rules
@@ -176,6 +185,11 @@ private Map<String, String> translateRulesFromCharStream(CharStream input,
     ParseTreeWalker walker = new ParseTreeWalker();
     walker.walk(this, tree);
 
+    if (this.completeValidation.getStages().isEmpty()) {
+      throw new ParseCancellationException(
+          "Rules file must contain at least one validation stage");
+    }
+
     // Generate output using the validator generator
     return this.validatorGenerator.generateOutput(this.completeValidation);
   }
@@ -649,8 +663,9 @@ public void exitFallbackRule(FallbackRuleContext ctx) {
    */
   class RulesPreprocessor extends ExpressionPreprocessor {
 
-    RulesPreprocessor(final CharStream charStream) {
-      super(charStream);
+    RulesPreprocessor(final CharStream charStream, IncludedFileResolver resolver)
+        throws IOException {
+      super(new IncludeProcessor(resolver).resolve(charStream));
     }
 
     String processRules() {

src/main/java/eu/europa/ted/efx/sdk2/EfxTemplateTranslatorV2.java

@@ -15,6 +15,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
@@ -42,6 +43,7 @@
 import eu.europa.ted.efx.exceptions.InvalidIndentationException;
 import eu.europa.ted.efx.interfaces.Argument;
 import eu.europa.ted.efx.interfaces.EfxTemplateTranslator;
+import eu.europa.ted.efx.interfaces.IncludedFileResolver;
 import eu.europa.ted.efx.interfaces.MarkupGenerator;
 import eu.europa.ted.efx.interfaces.ScriptGenerator;
 import eu.europa.ted.efx.interfaces.SymbolResolver;
@@ -165,6 +167,12 @@ public EfxTemplateTranslatorV2(final MarkupGenerator markupGenerator,
    */
   @Override
   public String renderTemplate(final Path pathname, TranslatorOptions options) throws IOException {
+    // Default to filesystem-based include resolution relative to the input file
+    if (options.getIncludedFileResolver() == null) {
+      Path baseDir = pathname.toAbsolutePath().getParent();
+      options = TranslatorOptions.withResolver(options, new FileSystemIncludedFileResolver(baseDir));
+    }
+
     return renderTemplate(CharStreams.fromPath(pathname), options);
   }
 
@@ -173,21 +181,26 @@ public String renderTemplate(final Path pathname, TranslatorOptions options) thr
    */
   @Override
   public String renderTemplate(final String template, TranslatorOptions options) {
-    return renderTemplate(CharStreams.fromString(template), options);
+    try {
+      return renderTemplate(CharStreams.fromString(template), options);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Include resolution failed during template rendering", e);
+    }
   }
 
   @Override
   public String renderTemplate(final InputStream stream, TranslatorOptions options) throws IOException {
     return renderTemplate(CharStreams.fromStream(stream), options);
   }
 
-  private String renderTemplate(final CharStream charStream, TranslatorOptions options) {
+  private String renderTemplate(final CharStream charStream, TranslatorOptions options)
+      throws IOException {
     logger.debug("Rendering template");
     final long startTime = System.currentTimeMillis();
 
     // New in EFX-2: template preprocessing
     final long preprocessingStartTime = System.currentTimeMillis();
-    final TemplatePreprocessor preprocessor = this.new TemplatePreprocessor(charStream);
+    final TemplatePreprocessor preprocessor = this.new TemplatePreprocessor(charStream, options.getIncludedFileResolver());
     final String preprocessedTemplate = preprocessor.processTemplate();
     final long preprocessingEndTime = System.currentTimeMillis();
     final long preprocessingDuration = preprocessingEndTime - preprocessingStartTime;
@@ -1518,8 +1531,8 @@ private int getIndentLevel(IndentationContext ctx) {
    */
   class TemplatePreprocessor extends ExpressionPreprocessor {
 
-    TemplatePreprocessor(CharStream template) {
-      super(template);
+    TemplatePreprocessor(CharStream template, IncludedFileResolver resolver) throws IOException {
+      super(new IncludeProcessor(resolver).resolve(template));
     }
 
     String processTemplate() {

src/main/java/eu/europa/ted/efx/sdk2/FileSystemIncludedFileResolver.java

@@ -0,0 +1,40 @@
+package eu.europa.ted.efx.sdk2;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import eu.europa.ted.efx.interfaces.IncludedFileResolver;
+
+/**
+ * Resolves include paths relative to a base directory on the file system.
+ *
+ * <p>
+ * This is the default resolver used when translating rules from a file path. Include paths
+ * specified in {@code #include} directives are resolved relative to the base directory.
+ * </p>
+ */
+public class FileSystemIncludedFileResolver implements IncludedFileResolver {
+
+  private final Path baseDir;
+
+  public FileSystemIncludedFileResolver(Path baseDir) throws IOException {
+    this.baseDir = baseDir.toRealPath();
+  }
+
+  @Override
+  public String resolve(String path) throws IOException {
+    Path normalized = this.baseDir.resolve(path).normalize();
+    if (!normalized.startsWith(this.baseDir)) {
+      throw new IOException(
+          "Include path '" + path + "' resolves outside the base directory");
+    }
+    // Resolve symlinks to prevent symlink-based traversal
+    Path real = normalized.toRealPath();
+    if (!real.startsWith(this.baseDir)) {
+      throw new IOException(
+          "Include path '" + path + "' resolves outside the base directory");
+    }
+    return Files.readString(real);
+  }
+}