Jeff/ci integration p2

.dockerignore

@@ -16,4 +16,11 @@ vcpkg_installed
 build_scripts
 venv
 releases
-shared-lib
+shared-lib
+.agent-system
+.gemini
+.work
+.artifacts
+.docs
+.github
+.githooks

.github/workflows/Dockerfile.build

@@ -1,5 +1,5 @@
 # The Dockerfile for CI build & tests (for GitHub Actions)
-ARG BASE_IMAGE=381492230138.dkr.ecr.us-east-1.amazonaws.com/devsupport:invalid
+ARG BASE_IMAGE=ghcr.io/springtail-inc/devsupport:invalid
 FROM ${BASE_IMAGE}
 
 ARG CI_BUILD_TYPE
@@ -26,9 +26,9 @@ RUN ln -s /home/dev/debug debug && \
 
 RUN if [ "$SPRINGTAIL_INCLUDE_TIME_TRACES" = "false" ]; then unset SPRINGTAIL_INCLUDE_TIME_TRACES ; fi && \
     if [ "$CI_BUILD_TYPE" = "debug" ]; then \
-        cmake -B "debug" -S . -D'CMAKE_BUILD_TYPE=Debug' -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build";  \
+    cmake -B "debug" -S . -D'CMAKE_BUILD_TYPE=Debug' -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build";  \
     else \
-        cmake -B "release" -S . -D'CMAKE_BUILD_TYPE=Release' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DNDEBUG=1 -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build";  \
+    cmake -B "release" -S . -D'CMAKE_BUILD_TYPE=Release' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DNDEBUG=1 -DVCPKG_INSTALL_OPTIONS="--allow-unsupported;--clean-after-build";  \
     fi && \
     rm -rf  /home/dev/external/vcpkg/buildtrees /home/dev/external/vcpkg/packages /home/dev/external/vcpkg/downloads
 

.github/workflows/build-coordinator.yml

@@ -15,37 +15,21 @@ permissions:
   id-token: write
   contents: read
   checks: write
+  packages: write
 
 jobs:
   build:
-    runs-on: warp-ubuntu-latest-arm64-4x-spot
+    runs-on: ubuntu-22.04-arm
     timeout-minutes: 35
     container:
-      image: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.build_type == 'release'  && vars.RELEASE_BUILDER_IMAGE_VERSION  || vars.BUILDER_IMAGE_VERSION }}
+      image: ghcr.io/springtail-inc/${{ inputs.build_type == 'release'  && vars.RELEASE_BUILDER_IMAGE_VERSION  || vars.BUILDER_IMAGE_VERSION }}
       credentials:
-        username: AWS
-        password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"
-    env:
-      PACKAGE_BUCKET_NAME: ${{ inputs.build_type == 'release' && 'prod-data-share.springtail.internal' || 'data-share.springtail.internal' }}
-      COORDINATOR_S3_PREFIX: stc
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: install-aws-cli-action
-        uses: unfor19/install-aws-cli-action@v1
-        with:
-          version: 2                         # default
-          verbose: false                     # default
-          arch: arm64
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ secrets.DEV_AWS_ROLE_ARN }}
-          role-session-name: tempSession
-          aws-region: us-east-1
-
       - name: Build
         run: |
           git config --global --add safe.directory /__w/springtail/springtail
@@ -66,14 +50,8 @@ jobs:
           echo "log_rotation_count: 10" >> stc/coordinator/config.yaml
           tar -czf stc.tar.gz stc
 
-      - name: Upload Coordinator
-        run: |
-          # generate a name with timestamp  
-          TIMESTAMP=$(date +%s)
-          name="stc-${TIMESTAMP}.tar.gz"
-          # Add this name to an current file
-          echo $name > current
-          aws s3 cp stc.tar.gz s3://${PACKAGE_BUCKET_NAME}/${COORDINATOR_S3_PREFIX}/${name}
-          aws s3 cp current s3://${PACKAGE_BUCKET_NAME}/${COORDINATOR_S3_PREFIX}/current
-          rm stc.tar.gz
-          rm current
+      - name: Upload Coordinator Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: 'Coordinator Package'
+          path: stc.tar.gz

.github/workflows/build-packages.yml

@@ -19,22 +19,25 @@ permissions:
   id-token: write
   contents: read
   checks: write
+  packages: write
 
 jobs:
   system-settings-check:
     uses: ./.github/workflows/system-settings-check.yml
+    secrets: inherit
     with:
       canonical_copy: 'system.json.settings'
       target_files: 'prod.system.settings.json'
   release-build:
     uses: ./.github/workflows/release-build.yml
     needs: [ system-settings-check ]
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write
     with:
       num_procs: ${{ inputs.num_procs || 4 }}
       package_bucket_name: ${{ inputs.build_type == 'release' && 'prod-data-share.springtail.internal' || 'data-share.springtail.internal' }}
-      image_version_tag: ${{ vars.DEV_SUPPORT_ECR_REPO_URI }}:${{ inputs.build_type == 'release'  && vars.RELEASE_BUILDER_IMAGE_VERSION  || vars.BUILDER_IMAGE_VERSION }}
+      image_version_tag: ghcr.io/springtail-inc/${{ inputs.build_type == 'release'  && vars.RELEASE_BUILDER_IMAGE_VERSION  || vars.BUILDER_IMAGE_VERSION }}
       slack_webhook_url: "${{ vars.GENERIC_SLACK_WEBHOOK_URL }}"
       build_type: ${{ inputs.build_type || 'release' }}
-    secrets:
-      aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}"
-      aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"

.github/workflows/nightly.yml

@@ -6,6 +6,7 @@ permissions:
   id-token: write
   contents: read
   checks: write
+  packages: write
 
 jobs:
   prepare:
@@ -28,68 +29,75 @@ jobs:
 
   debug-build:
     uses: ./.github/workflows/rw-build-container.yml
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write
     with:
       base_image_tag: ${{ vars.IMAGE_TAG_DEBUG }}
       build_type: "debug"
-    secrets:
-      aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}"
-      aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"
 
   release-build-with-time-traces:
     uses: ./.github/workflows/rw-build-container.yml
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write
     with:
       base_image_tag: ${{ vars.IMAGE_TAG_RELEASE }}
       build_type: "release"
       springtail_include_time_traces: true
-    secrets:
-      aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}"
-      aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"
 
   unit-test:
     uses: ./.github/workflows/rw-run-unit-test.yml
     needs: [ debug-build, prepare ]
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write
+      checks: write
     with:
       image_tag: ${{ needs.debug-build.outputs.output_image_tag }}
-      s3_log_prefix: "nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}"
       build_type: "debug"
       reuse_code: true
-    secrets:
-      aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}"
-      aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"
 
   multi-rounds-integration-test:
     uses: ./.github/workflows/rw-run-integration-test.yml
     needs: [ debug-build, prepare ]
+    secrets: inherit
+    permissions:
+      contents: read
+      packages: write
+      checks: write
     with:
       image_tag: ${{ needs.debug-build.outputs.output_image_tag }}
-      s3_log_prefix: "nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}"
       config_to_run: "nightly"
       system_settings_override: '.github/workflows/nightly.overrides.json'
       iterations: "1"
       build_type: "debug"
       reuse_code: true
       timeout_minutes: 120
-    secrets:
-      aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}"
-      aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"
 
   performance-test:
     uses: ./.github/workflows/rw-run-performance-test.yml
     needs: [ release-build-with-time-traces, prepare ]
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+      packages: write
+      checks: write
     with:
       image_tag: ${{ needs.release-build-with-time-traces.outputs.output_image_tag }}
-      s3_log_prefix: "nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}"
       system_settings_override: '.github/workflows/nightly.overrides.json'
       build_type: "release"
       reuse_code: true
-    secrets:
-      aws_role_arn: "${{ secrets.DEV_AWS_ROLE_ARN }}"
-      aws_ecr_docker_password: "${{ secrets.DEV_SUPPORT_ECR_PW }}"
 
   slack-notification:
     uses: ./.github/workflows/rw-notify-slack.yml
     needs: [ prepare, unit-test, multi-rounds-integration-test, performance-test ]
     if: needs.unit-test.result != 'skipped' && needs.multi-rounds-integration-test.result != 'skipped' && needs.performance-test.result != 'skipped'
+    secrets: inherit
     with:
       slack_webhook_url: ${{ vars.GENERIC_SLACK_WEBHOOK_URL }}
       # Need to create a new channel for PR notifications.
@@ -98,32 +106,27 @@ jobs:
         Nightly builds has completed for ${{ needs.prepare.outputs.date_prefix }} - ${{ needs.prepare.outputs.commit_sha }}.
       attachment: |
         {
-          color": "${{ needs.unit-test.result == 'success' && needs.multi-rounds-integration-test.result == 'success' && needs.performance-test.result == 'success' && 'good' || 'danger' }}",
+          "color": "${{ needs.unit-test.result == 'success' && needs.multi-rounds-integration-test.result == 'success' && needs.performance-test.result == 'success' && 'good' || 'danger' }}",
           "fields": [
             {
               "title": "Unit Test",
-              "value": "${{ needs.unit-test.result }}; logs: <${{ needs.unit-test.outputs.test_log_s3_uri }}|here>",
+              "value": "${{ needs.unit-test.result }}; <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>",
               "short": false
             },
             {
               "title": "Multi-rounds Integration Test",
-              "value": "${{ needs.multi-rounds-integration-test.result }}; last logs: <${{ needs.multi-rounds-integration-test.outputs.test_log_s3_uri }}|here>; last springtail logs: <${{ needs.multi-rounds-integration-test.outputs.springtail_log_s3_uri }}|here>",
+              "value": "${{ needs.multi-rounds-integration-test.result }}; <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>",
               "short": false
             },
             {
               "title": "Performance Test",
-              "value": "${{ needs.performance-test.result }}; logs: <${{ needs.performance-test.outputs.test_log_s3_uri }}|here>",
+              "value": "${{ needs.performance-test.result }}; <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Artifacts>",
               "short": false
             },
             {
                 "title": "Commit SHA",
                 "value": "${{ needs.prepare.outputs.commit_sha }}",
                 "short": false
-            },
-            {
-                "title": "All Logs",
-                "value": "s3://devsupport.springtail.internal/nightly-testlogs/${{ needs.prepare.outputs.date_prefix }}/${{ needs.prepare.outputs.commit_sha }}",
-                "short": false
             }
           ]
         }