feat: add support for YAML file includes and nested includes in context

src/context/yaml/index.ts

@@ -18,6 +18,65 @@ import { Assets, Config, Auth0APIClient, AssetTypes, KeywordMappings } from '../
 import { filterOnlyIncludedResourceTypes } from '..';
 import { preserveKeywords } from '../../keywordPreservation';
 
+// Custom YAML type for file includes
+const includeType = new yaml.Type('!include', {
+  kind: 'scalar',
+  resolve: (data) => typeof data === 'string',
+  construct: (data) => {
+    // This will be handled during the actual loading process
+    return { __include: data };
+  }
+});
+
+const schema = yaml.DEFAULT_SCHEMA.extend([includeType]);
+
+// Function to resolve includes with cycle detection
+function resolveIncludes(content: string, basePath: string, mappings?: KeywordMappings, disableKeywordReplacement?: boolean, visitedFiles = new Set<string>()): string {
+  const obj = yaml.load(content, { schema });
+  return resolveIncludesInObject(obj, basePath, mappings, disableKeywordReplacement, visitedFiles);
+}
+
+function resolveIncludesInObject(obj, basePath, mappings?: KeywordMappings, disableKeywordReplacement?: boolean, visitedFiles = new Set<string>()) {
+  if (Array.isArray(obj)) {
+    return obj.map(item => resolveIncludesInObject(item, basePath, mappings, disableKeywordReplacement, visitedFiles));
+  }
+
+  if (obj && typeof obj === 'object') {
+    if (obj.__include) {
+      const filePath = path.resolve(basePath, obj.__include);
+
+      if (visitedFiles.has(filePath)) {
+        throw new Error(`Circular include detected: ${filePath}`);
+      }
+
+      if (fs.existsSync(filePath)) {
+        visitedFiles.add(filePath);
+        let content = fs.readFileSync(filePath, 'utf8');
+
+        // Apply keyword replacement to included file content if mappings are provided
+        if (mappings && !disableKeywordReplacement) {
+          content = keywordReplace(content, mappings);
+        } else if (mappings && disableKeywordReplacement) {
+          content = wrapArrayReplaceMarkersInQuotes(content, mappings);
+        }
+
+        const result = resolveIncludesInObject(yaml.load(content, { schema }), path.dirname(filePath), mappings, disableKeywordReplacement, new Set(visitedFiles));
+        visitedFiles.delete(filePath);
+        return result;
+      }
+      throw new Error(`Include file not found: ${filePath}`);
+    }
+
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = resolveIncludesInObject(value, basePath, mappings, disableKeywordReplacement, visitedFiles);
+    }
+    return result;
+  }
+
+  return obj;
+}
+
 export default class YAMLContext {
   basePath: string;
   configFile: string;
@@ -62,6 +121,10 @@ export default class YAMLContext {
     if (!isFile(toLoad)) {
       // try load not relative to yaml file
       toLoad = f;
+      if (!isFile(toLoad)) {
+        // try absolute path resolution
+        toLoad = path.resolve(f);
+      }
     }
     return loadFileAndReplaceKeywords(path.resolve(toLoad), {
       mappings: this.mappings,
@@ -78,13 +141,13 @@ export default class YAMLContext {
       try {
         const fPath = path.resolve(this.configFile);
         log.debug(`Loading YAML from ${fPath}`);
+        const content = opts.disableKeywordReplacement
+          ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings)
+          : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings);
+
         Object.assign(
           this.assets,
-          yaml.load(
-            opts.disableKeywordReplacement
-              ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings)
-              : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings)
-          ) || {}
+          resolveIncludes(content, path.dirname(fPath), this.mappings, opts.disableKeywordReplacement)
         );
       } catch (err) {
         log.debug(err.stack);

test/context/yaml/actions.test.js

@@ -66,6 +66,165 @@ describe('#YAML context actions', () => {
     expect(context.assets.actions).to.deep.equal(target);
   });
 
+  it('should process YAML with includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'includes');
+    cleanThenMkdir(dir);
+
+    const clientsYaml = `
+- name: "Test Client"
+  app_type: "spa"
+- name: "Test M2M"
+  app_type: "non_interactive"
+    `;
+    const clientsFile = path.join(dir, 'clients.yaml');
+    fs.writeFileSync(clientsFile, clientsYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Test Tenant'
+
+clients: !include clients.yaml
+    `;
+    const mainFile = path.join(dir, 'tenant.yaml');
+    fs.writeFileSync(mainFile, mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: mainFile };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.tenant).to.deep.equal({
+      friendly_name: 'Test Tenant',
+    });
+    expect(context.assets.clients).to.deep.equal([
+      {
+        name: 'Test Client',
+        app_type: 'spa',
+      },
+      {
+        name: 'Test M2M',
+        app_type: 'non_interactive',
+      },
+    ]);
+  });
+
+  it('should handle nested includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'nested-includes');
+    cleanThenMkdir(dir);
+
+    const rolesYaml = `
+- name: Admin
+  description: Administrator
+- name: User
+  description: Regular User
+    `;
+    fs.writeFileSync(path.join(dir, 'roles.yaml'), rolesYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Main Tenant'
+
+roles: !include roles.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.roles).to.deep.equal([
+      { name: 'Admin', description: 'Administrator' },
+      { name: 'User', description: 'Regular User' },
+    ]);
+  });
+
+  it('should process logStreams with includes', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'logstreams-includes');
+    cleanThenMkdir(dir);
+
+    const logStreamsYaml = `
+- name: LoggingSAAS
+  isPriority: false
+  filters:
+  - type: category
+    name: auth.login.fail
+  - type: category
+    name: auth.login.notification
+  - type: category
+    name: auth.login.success
+  - type: category
+    name: auth.logout.fail
+  sink:
+    httpContentFormat: JSONLINES
+    httpContentType: application/json
+    httpEndpoint: "##LOGGING_WEBHOOK_URL##"
+  type: http
+- name: SIEM
+  isPriority: false
+  filters:
+  - type: category
+    name: auth.login.fail
+  - type: category
+    name: auth.login.notification
+  - type: category
+    name: auth.login.success
+  sink:
+    httpContentFormat: JSONLINES
+    httpContentType: application/json
+    httpEndpoint: "##SIEM_WEBHOOK_URL##"
+  type: http
+    `;
+    fs.writeFileSync(path.join(dir, 'logStreams.yaml'), logStreamsYaml);
+
+    const mainYaml = `
+tenant:
+  friendly_name: 'Test Tenant'
+
+logStreams: !include logStreams.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = {
+      AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml'),
+      AUTH0_KEYWORD_REPLACE_MAPPINGS: {
+        LOGGING_WEBHOOK_URL: 'https://logging.com/inputs/test',
+        SIEM_WEBHOOK_URL: 'https://siem.example.com/webhook'
+      }
+    };
+    const context = new Context(config, mockMgmtClient());
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.logStreams).to.have.length(2);
+    expect(context.assets.logStreams[0]).to.deep.include({
+      name: 'LoggingSAAS',
+      isPriority: false,
+      type: 'http'
+    });
+    expect(context.assets.logStreams[0].sink.httpEndpoint).to.equal('https://logging.com/inputs/test');
+    expect(context.assets.logStreams[1]).to.deep.include({
+      name: 'SIEM',
+      isPriority: false,
+      type: 'http'
+    });
+    expect(context.assets.logStreams[1].sink.httpEndpoint).to.equal('https://siem.example.com/webhook');
+  });
+
+  it('should error on missing include file', async () => {
+    const dir = path.join(testDataDir, 'yaml', 'missing-include');
+    cleanThenMkdir(dir);
+
+    const mainYaml = `
+clients: !include missing.yaml
+    `;
+    fs.writeFileSync(path.join(dir, 'tenant.yaml'), mainYaml);
+
+    const config = { AUTH0_INPUT_FILE: path.join(dir, 'tenant.yaml') };
+    const context = new Context(config, mockMgmtClient());
+
+    await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith(
+      Error,
+      /Include file not found/
+    );
+  });
   it('should dump actions', async () => {
     const dir = path.join(testDataDir, 'yaml', 'actionsDump');
     cleanThenMkdir(dir);

test/context/yaml/includes.test.js

@@ -0,0 +1,120 @@
+import path from 'path';
+import fs from 'fs-extra';
+import { expect } from 'chai';
+import Context from '../../../src/context/yaml';
+import { cleanThenMkdir, testDataDir, mockMgmtClient } from '../../utils';
+
+describe('#YAML includes cycle detection', () => {
+  it('should detect direct circular include', async () => {
+    const dir = path.resolve(testDataDir, 'yaml', 'circular-direct');
+    cleanThenMkdir(dir);
+
+    const mainFile = path.join(dir, 'main.yaml');
+    fs.writeFileSync(mainFile, 'data: !include self.yaml');
+
+    const selfFile = path.join(dir, 'self.yaml');
+    fs.writeFileSync(selfFile, 'value: !include self.yaml');
+
+    const config = { AUTH0_INPUT_FILE: mainFile };
+    const context = new Context(config, mockMgmtClient());
+
+    await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith(
+      Error,
+      /Circular include detected/
+    );
+  });
+
+  it('should detect indirect circular include (A → B → A)', async () => {
+    const dir = path.resolve(testDataDir, 'yaml', 'circular-indirect');
+    cleanThenMkdir(dir);
+
+    const fileA = path.join(dir, 'a.yaml');
+    fs.writeFileSync(fileA, 'data: !include b.yaml');
+
+    const fileB = path.join(dir, 'b.yaml');
+    fs.writeFileSync(fileB, 'value: !include a.yaml');
+
+    const config = { AUTH0_INPUT_FILE: fileA };
+    const context = new Context(config, mockMgmtClient());
+
+    await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith(
+      Error,
+      /Circular include detected/
+    );
+  });
+
+  it('should detect complex circular include (A → B → C → A)', async () => {
+    const dir = path.resolve(testDataDir, 'yaml', 'circular-complex');
+    cleanThenMkdir(dir);
+
+    const fileA = path.join(dir, 'a.yaml');
+    fs.writeFileSync(fileA, 'data: !include b.yaml');
+
+    const fileB = path.join(dir, 'b.yaml');
+    fs.writeFileSync(fileB, 'value: !include c.yaml');
+
+    const fileC = path.join(dir, 'c.yaml');
+    fs.writeFileSync(fileC, 'final: !include a.yaml');
+
+    const config = { AUTH0_INPUT_FILE: fileA };
+    const context = new Context(config, mockMgmtClient());
+
+    await expect(context.loadAssetsFromLocal()).to.be.eventually.rejectedWith(
+      Error,
+      /Circular include detected/
+    );
+  });
+
+  it('should allow valid includes without cycles', async () => {
+    const dir = path.resolve(testDataDir, 'yaml', 'valid-includes');
+    cleanThenMkdir(dir);
+
+    const mainFile = path.join(dir, 'main.yaml');
+    fs.writeFileSync(mainFile, `
+tenant:
+  friendly_name: Test
+  data: !include shared.yaml
+rules: !include rules.yaml
+`);
+
+    const sharedFile = path.join(dir, 'shared.yaml');
+    fs.writeFileSync(sharedFile, 'shared_value: 42');
+
+    const rulesFile = path.join(dir, 'rules.yaml');
+    fs.writeFileSync(rulesFile, '[]');
+
+    const config = { AUTH0_INPUT_FILE: mainFile };
+    const context = new Context(config, mockMgmtClient());
+
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.tenant.friendly_name).to.equal('Test');
+    expect(context.assets.tenant.data.shared_value).to.equal(42);
+    expect(context.assets.rules).to.deep.equal([]);
+  });
+
+  it('should allow same file included multiple times in different branches', async () => {
+    const dir = path.resolve(testDataDir, 'yaml', 'multiple-includes');
+    cleanThenMkdir(dir);
+
+    const mainFile = path.join(dir, 'main.yaml');
+    fs.writeFileSync(mainFile, `
+tenant:
+  friendly_name: Test
+  config1: !include shared.yaml
+  config2: !include shared.yaml
+`);
+
+    const sharedFile = path.join(dir, 'shared.yaml');
+    fs.writeFileSync(sharedFile, 'shared');
+
+    const config = { AUTH0_INPUT_FILE: mainFile };
+    const context = new Context(config, mockMgmtClient());
+
+    await context.loadAssetsFromLocal();
+
+    expect(context.assets.tenant.friendly_name).to.equal('Test');
+    expect(context.assets.tenant.config1).to.equal('shared');
+    expect(context.assets.tenant.config2).to.equal('shared');
+  });
+});