feat: process configuration in chunks

.github/workflows/scheduled-jobs.yaml

@@ -10,57 +10,127 @@ on:
       - schema/**
       - definitions/**
       - docs/COMPARISON.md
-  workflow_dispatch:
-    inputs:
-      reset:
-        description: Space-separated paths to remove from schema before running an update
-        default: ""
-        type: string
+  workflow_dispatch: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  update-schemas:
-    name: Update schemas
-    runs-on: ubuntu-latest
+  configuration-chunks:
+    name: Split configuration
     if: github.repository == 'CustomResourceDefinition/catalog' || github.event_name == 'workflow_dispatch'
-    permissions:
-      contents: write
-      packages: write
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v6
         with:
-          ssh-key: ${{ secrets.DEPLOY_KEY_SCHEDULED_JOBS }}
+          sparse-checkout: |
+            build
+            configuration.yaml
+            Makefile
+            make.d
+
+      - run: make config-chunks
+        env:
+          CHUNK_SIZE: 100
+
+      - run: mv build/configuration_*.yml .
+
+      - run: |
+          {
+            printf "matrix="
+            echo configuration*.yml | tr ' ' "\n" | jq -Rsc 'split("\n")[:-1]'
+          } | tee -a $GITHUB_OUTPUT
+        id: matrix
+
+      - uses: actions/upload-artifact@v5
+        with:
+          name: configurations
+          path: configuration_*.yml
+          retention-days: 1
+
+  update:
+    name: Create patch files
+    needs: configuration-chunks
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        file: ${{ fromJSON(needs.configuration-chunks.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v6
 
-      - name: Optionally remove paths
-        if: github.event_name == 'workflow_dispatch' && github.event.inputs.reset != ''
-        run: |
-          cd schema
-          rm -rf ${{ github.event.inputs.reset }} || true
-          cd -
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
 
-          cd definitions
-          rm -rf ${{ github.event.inputs.reset }} || true
-          cd -
+      - name: Setup Golang cache
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - uses: actions/download-artifact@v6
+        with:
+          name: configurations
 
       - run: make update
         env:
-          GITHUB_ACTOR: ${{ github.actor }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_REF: ${{ github.ref }}
+          CONFIGURATION: ${{ matrix.file }}
+
+      - run: git diff schema definitions | tee ${{ matrix.file }}.patch
+
+      - uses: actions/upload-artifact@v5
+        with:
+          name: ${{ matrix.file }}.patch
+          path: ${{ matrix.file }}.patch
+          retention-days: 1
+
+  commit:
+    name: Update files
+    needs: update
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ssh-key: ${{ secrets.DEPLOY_KEY_SCHEDULED_JOBS }}
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: go.mod
+
+      - name: Setup Golang cache
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
 
       - uses: actions/checkout@v6
         with:
           repository: datreeio/CRDs-catalog
           path: build/remote/datreeio
 
+      - uses: actions/download-artifact@v6
+        with:
+          merge-multiple: true
+
+      - run: git apply --allow-empty *.patch
+
       - run: make comparison
-        env:
-          GITHUB_ACTOR: ${{ github.actor }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_REF: ${{ github.ref }}
+
+      - if: github.ref != 'refs/heads/main'
+        run: git diff --stat
 
       - uses: EndBug/add-and-commit@v9
         name: Publish changes
@@ -76,9 +146,9 @@ jobs:
     name: Report failures
     runs-on: ubuntu-latest
     needs:
-      - update-schemas
+      - commit
       - sync-tags
-    if: failure() && github.event_name != 'workflow_dispatch'
+    if: failure() && github.repository == 'CustomResourceDefinition/catalog' && github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch'
     permissions:
       contents: read
       issues: write
@@ -96,7 +166,8 @@ jobs:
   sync-tags:
     name: Synchronize tags with kubernetes
     runs-on: ubuntu-latest
-    needs: update-schemas
+    needs: commit
+    if: github.repository == 'CustomResourceDefinition/catalog' && github.ref == 'refs/heads/main'
     permissions:
       contents: write
     steps:

.github/workflows/tests.yaml

@@ -13,7 +13,3 @@ jobs:
     steps:
       - uses: actions/checkout@v6
       - run: make test
-        env:
-          GITHUB_ACTOR: ${{ github.actor }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_REF: ${{ github.ref }}

make.d/config-chunks.mk

@@ -0,0 +1,9 @@
+CHUNK_SIZE ?= 100
+
+config-chunks:
+	cd build && \
+		yq -o=json '.' ../configuration.yaml \
+		| jq -c ". as \$$arr \
+		| [range(0; \$$arr | length; $(CHUNK_SIZE)) | \$$arr[. : .+$(CHUNK_SIZE)]]" \
+		| yq -P \
+		| yq '.[]' -s '"configuration_" + $$index'

make.d/help.txt

@@ -8,8 +8,9 @@ Available targets:
   check         Runs update using the `build/configuration.yaml` file and outputting to `build/ephemeral/schema` - this is meant as a tool to test a new configuration item locally before adding it
   clean         Removes temporarily generated files and builds and removes local container images
   comparison    Updates COMPARISON.md based on the /schema contents
+  config-chunks Generates configuration chunk files
   help          Displays this help
-  schema-check Verifies all schema files against jsonschema schema file - this takes a very long time
+  schema-check  Verifies all schema files against jsonschema schema file - this takes a very long time
   tags          Synchronize semantic version tags from kubernetes into this repository
   test          Runs the entire test suite
   update        Updates the /schema contents

make.d/update.mk

@@ -1,3 +1,5 @@
+CONFIGURATION ?= configuration.yaml
+
 update: build/bin/catalog
 	df -h
-	build/bin/catalog update --configuration configuration.yaml --output schema --definitions definitions
+	build/bin/catalog update --configuration $(CONFIGURATION) --output schema --definitions definitions