deploy.yml

name: Deploy 🚀

permissions:
  id-token: write
  contents: read
  issues: write


on:
  workflow_call:
    inputs:
      environment:
        type: string
        required: true
        description: Environment to deploy to
      DEPLOY_BACKEND:
        type: string
        required: true
        default: true
        description: DEPLOY_BACKEND
      DEPLOY_FEATURES_API:
        type: string
        required: true
        default: false
        description: DEPLOY_FEATURES_API
      DEPLOY_ROUTES:
        type: string
        required: true
        default: false
        description: DEPLOY_ROUTES
      DEPLOY_SM2A:
        type: string
        required: true
        default: true
        description: DEPLOY_SM2A
      DEPLOY_MONITORING:
        type: string
        required: true
        default: false
        description: DEPLOY_MONITORING
      DEPLOY_TITILER_MULTIDIM:
        type: string
        required: true
        default: false
        description: DEPLOY_TITILER_MULTIDIM
      DEPLOY_S3_DISASTER_RECOVERY:
        type: string
        required: true
        default: false
        description: DEPLOY_S3_DISASTER_RECOVERY
      DEPLOY_TITILER_CMR:
        type: string
        required: true
        default: false
        description: DEPLOY_TITILER_CMR

concurrency:
  group: ${{ inputs.environment }}
  cancel-in-progress: true

run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }}

jobs:

  deploy-veda-backend:
    name: Deploy VEDA backend ⚙️
    runs-on: ubuntu-latest
    env:
      DIRECTORY: veda-backend
      ENVIRONMENT: ${{ inputs.environment }}
    if: ${{ inputs.DEPLOY_BACKEND == 'true' }}
    environment: ${{ inputs.environment }}

    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          lfs: "true"
          submodules: "false"

      - name: Checkout veda-backend submodule
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "NASA-IMPACT/${{ env.DIRECTORY }}"
          path: ${{ env.DIRECTORY }}
          ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment"
          aws-region: "us-west-2"

      - name: Run deployment
        uses: "./veda-backend/.github/actions/cdk-deploy"
        id: deploy_backend_stack
        with:
          env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }}
          dir: "${{ env.DIRECTORY }}"
          script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

      - name: Get Backend Output Values
        id: get_backend_stack
        shell: bash
        run: |
          stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json)
          echo "backend_stackname=$stack" >> $GITHUB_OUTPUT

          raster_api_url=$(jq '.[keys_unsorted[0]].rasterapiurl' ${HOME}/cdk-outputs.json)
          echo "raster_api_url=$raster_api_url" >> $GITHUB_OUTPUT

          ingest_api_url=$(jq '.[keys_unsorted[0]].ingestapiurl' ${HOME}/cdk-outputs.json)
          echo "ingest_api_url=$ingest_api_url" >> $GITHUB_OUTPUT

          stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json)
          echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT

          stac_browser_bucket=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json)
          echo "stac_browser_bucket=$stac_browser_bucket" >> $GITHUB_OUTPUT

    outputs:
      backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }}
      raster_api_url: ${{ steps.get_backend_stack.outputs.raster_api_url }}
      ingest_api_url: ${{ steps.get_backend_stack.outputs.ingest_api_url }}
      stac_api_url: ${{ steps.get_backend_stack.outputs.stac_api_url }}
      stac_browser_bucket: ${{ steps.get_backend_stack.outputs.stac_browser_bucket }}

  deploy-veda-data-airflow-sm2a:
    name: deploy VEDA data airflow SM2A 🛸
    runs-on: ubuntu-latest
    env:
      DIRECTORY: veda-data-airflow
      AWS_REGION: "us-west-2"
      ENVIRONMENT: ${{ inputs.environment }}
    if: ${{ inputs.DEPLOY_SM2A == 'true' }}
    environment: ${{ inputs.environment }}

    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          lfs: "true"
          submodules: "false"

      - name: Checkout veda-data-airflow submodule
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "NASA-IMPACT/${{ env.DIRECTORY }}"
          path: ${{ env.DIRECTORY }}
          ref: ${{ vars.VEDA_SM2A_DATA_AIRFLOW_GIT_REF || 'main'}}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-sm2a-deployment"
          aws-region: "${{ env.AWS_REGION }}"

      - name: Run SM2A deployment
        id: deploy_sm2a
        uses: "./veda-data-airflow/.github/actions/terraform-deploy-sm2a"
        with:
          env-file: ".env"
          env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }}
          dir: "${{ env.DIRECTORY }}"
          script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

      - name: Get SM2A Workflows API Endpoint
        id: get_sm2a_workflows_api_endpoint
        shell: bash
        run: |
          workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json 2>/dev/null | tr -d '"' | tr -d '\n' )
          echo "workflows_sm2a_api_value=${workflows_sm2a_api_value:-NA}" >> $GITHUB_OUTPUT

    outputs:
      workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }}

  deploy-veda-features-api:
    name: Deploy VEDA features-api 🗺️
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    if: ${{ inputs.DEPLOY_FEATURES_API == 'true' }}
    env:
      DIRECTORY: veda-features-api-cdk
      ENVIRONMENT: ${{ inputs.environment }}

    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          lfs: "true"
          submodules: "false"

      - name: Checkout veda-features-api submodule
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "NASA-IMPACT/${{ env.DIRECTORY }}"
          path: ${{ env.DIRECTORY }}
          ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment"
          aws-region: "us-west-2"

      - name: Run deployment
        uses: "./veda-features-api-cdk/.github/actions/cdk-deploy"
        id: deploy_features_stack
        with:
          env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }}
          dir: "${{ env.DIRECTORY }}"
          script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

      - name: Get Features API Stack Name
        id: get_features_api_stack
        shell: bash
        run: |
          stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json)
          echo "features_api_stackname=$stack" >> $GITHUB_OUTPUT

    outputs:
      features_api_stack_name: ${{ steps.get_features_api_stack.outputs.features_api_stackname }}

  deploy-veda-monitoring:
    name: deploy VEDA monitoring 👁️
    runs-on: ubuntu-latest
    env:
      DIRECTORY: veda-monitoring
      AWS_REGION: "us-west-2"
      ENVIRONMENT: ${{ inputs.environment }}
      GH_PAT_CHECK: ${{ secrets.GH_PAT }}
    if: ${{ inputs.DEPLOY_MONITORING == 'true' }}
    environment: ${{ inputs.environment }}

    steps:
      - name: Checkout
        if: ${{ env.GH_PAT_CHECK != '' }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          lfs: "true"
          submodules: "false"

      - name: Checkout veda-monitoring "submodule"
        if: ${{ env.GH_PAT_CHECK != '' }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "NASA-IMPACT/${{ env.DIRECTORY }}"
          path: ${{ env.DIRECTORY }}
          ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }}
          token: ${{ secrets.GH_PAT }}

      - name: Configure AWS Credentials
        if: ${{ env.GH_PAT_CHECK != '' }}
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment"
          aws-region: "${{ env.AWS_REGION }}"

      - name: Run deployment
        if: ${{ env.GH_PAT_CHECK != '' }}
        uses: "./veda-monitoring/.github/actions/cdk-deploy"
        with:
          env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }}
          dir: "${{ env.DIRECTORY }}"
          script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

  deploy-titiler-multidim:
      name: Deploy titiler-multidim 🌎
      runs-on: ubuntu-latest
      env:
        DIRECTORY: titiler-multidim
        ENVIRONMENT: ${{ inputs.environment }}
      environment: ${{ inputs.environment }}
      if: ${{ inputs.DEPLOY_TITILER_MULTIDIM == 'true' }}

      steps:
        - name: Checkout
          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
          with:
            lfs: "true"
            submodules: "false"

        - name: Checkout titiler-multidim submodule
          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
          with:
            repository: "developmentseed/${{ env.DIRECTORY }}"
            path: ${{ env.DIRECTORY }}
            ref: ${{ vars.TITILER_MULTIDIM_GIT_REF || 'main'}}

        - name: Configure AWS Credentials
          uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
          with:
            role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
            role-session-name: "gh-${{ env.ENVIRONMENT }}-titiler-multidim-deployment"
            aws-region: "us-west-2"

        - name: Run titiler-multidim deployment
          id: deploy_titiler_multidim
          uses: "./titiler-multidim/.github/actions/cdk-deploy"
          with:
            dir: "${{ env.DIRECTORY }}/infrastructure/aws"
            env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }}
            script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

  deploy-s3-disaster-recovery:
      name: Deploy s3-disaster-recovery
      runs-on: ubuntu-latest
      env:
        DIRECTORY: s3-disaster-recovery
        ENVIRONMENT: ${{ inputs.environment }}
      environment: ${{ inputs.environment }}
      if: ${{ inputs.DEPLOY_S3_DISASTER_RECOVERY == 'true' }}

      steps:
        - name: Checkout
          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
          with:
            lfs: "true"
            submodules: "false"

        - name: Checkout s3-disaster-recovery submodule
          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
          with:
            repository: "NASA-IMPACT/${{ env.DIRECTORY }}"
            path: ${{ env.DIRECTORY }}
            ref: ${{ vars.S3_DISASTER_RECOVERY_GIT_REF || 'main'}}

        - name: Configure AWS Credentials
          uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
          with:
            role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
            role-session-name: "gh-${{ env.ENVIRONMENT }}-s3-disaster-recovery-deployment"
            aws-region: "us-west-2"

        - name: Run s3-disaster-recovery deployment
          id: deploy_s3-disaster-recovery
          uses: "./s3-disaster-recovery/.github/workflows"
          with:
            env_file: ".env"
            aws_region: ${{ vars.AWS_REGION }}
            allow_batch_replication: ${{ vars.VEDA_S3_DISASTER_RECOV_ALLOW_BATCH_REPLICATION }}
            source_bucket: ${{ vars.VEDA_S3_DISASTER_RECOV_SOURCE_BUCKET_NAME }}
            destination_bucket: ${{ vars.VEDA_S3_DISASTER_RECOV_DESTINATION_BUCKET_NAME }}
            env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }}
            dir: "${{ env.DIRECTORY }}"
            script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

  deploy-titiler-cmr:
      name: Deploy titiler-cmr 📚
      runs-on: ubuntu-latest
      env:
        DIRECTORY: titiler-cmr
        ENVIRONMENT: ${{ inputs.environment }}
      environment: ${{ inputs.environment }}
      if: ${{ inputs.DEPLOY_TITILER_CMR == 'true' }}

      steps:
        - name: Checkout
          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
          with:
            # curious why this is needed
            lfs: "true"
            submodules: "false"

        - name: Checkout titiler-cmr submodule
          uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
          with:
            repository: "developmentseed/${{ env.DIRECTORY }}"
            path: ${{ env.DIRECTORY }}
            ref: ${{ vars.TITILER_CMR_GIT_REF || 'develop'}}

        - name: Configure AWS Credentials
          uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
          with:
            role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
            role-session-name: "gh-${{ env.ENVIRONMENT }}-titiler-cmr-deployment"
            aws-region: "us-west-2"

        - name: Run titiler-cmr deployment
          id: deploy_titiler_cmr
          uses: "./titiler-cmr/.github/actions/cdk-deploy"
          with:
            dir: "${{ env.DIRECTORY }}/infrastructure/aws"
            env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }}
            script_path: "${{ github.workspace }}/scripts/generate_env_file.py"

  deploy-veda-routes:
    name: deploy VEDA routes 🔀
    runs-on: ubuntu-latest
    env:
      DIRECTORY: veda-routes
      AWS_REGION: "us-west-2"
      ENVIRONMENT: ${{ github.event.inputs.environment }}
    needs: [ deploy-veda-backend ]
    if: ${{ github.event.inputs.DEPLOY_ROUTES == 'true' }}
    environment: ${{ github.event.inputs.environment }}

    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          lfs: "true"
          submodules: "false"

      - name: Checkout veda-routes "submodule"
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: "NASA-IMPACT/${{ env.DIRECTORY }}"
          path: ${{ env.DIRECTORY }}
          ref: ${{ vars.VEDA_ROUTES_GIT_REF || 'dev' }}
          token: ${{ secrets.GH_PAT }}

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "gh-${{ env.ENVIRONMENT }}-routes-deployment"
          aws-region: "${{ env.AWS_REGION }}"

      - name: Set up Python
        uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa #v4.8.0
        with:
          python-version: "3.10"

      - name: Setup python cache
        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf #v4.2.2
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt') }}

      - name: Install python dependencies
        working-directory: ${{ env.DIRECTORY }}
        shell: bash
        run: |
          pip install -r ../requirements.txt
          pip install -r requirements.txt

      - name: Update Veda Routes secrets with dependency outputs
        shell: bash
        run: |
          python3 "${{ github.workspace }}/scripts/update_secret_with_inputs.py" \
            --secret-id ${{ vars.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }} \
            --raster_api_url=${{ needs.deploy-veda-backend.outputs.raster_api_url }} \
            --ingest_api_url=${{ needs.deploy-veda-backend.outputs.ingest_api_url }} \
            --stac_api_url=${{ needs.deploy-veda-backend.outputs.stac_api_url }} \
            --stac_browser_bucket=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket }}

      - name: Run deployment
        uses: "./veda-routes/.github/actions/cdk-deploy"
        with:
          env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }}
          dir: "${{ env.DIRECTORY }}"


  test-deployment:
    name: Test Deployment 👨🏻‍🔬
    runs-on: ubuntu-latest
    timeout-minutes: 15 # Normally takes ~30s
    needs: [ deploy-veda-backend ]
    env:
      DIRECTORY: integration_test
      ENVIRONMENT: ${{ inputs.environment }}
      AWS_DEFAULT_REGION: us-west-2
    environment: ${{ inputs.environment }}
    steps:
      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 #v2.7.0

      - name: Set up Python
        uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa #v4.8.0
        with:
          python-version: "3.10"
          cache: "pip"
          cache-dependency-path: |
            ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0
        with:
          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
          role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test"
          aws-region: "${{ env.AWS_DEFAULT_REGION }}"

      - name: Install python dependencies
        shell: bash
        working-directory: ${{ env.DIRECTORY }}
        run: |
          pip install -r requirements.txt
          python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}"

      - name: Integration test
        shell: bash
        working-directory: ${{ env.DIRECTORY }}
        run: |
          pytest .

  run-playwright-checks:
    name: playwright end to end tests
    if: ${{ needs.define-environment.outputs.env_name }}
    runs-on: ubuntu-latest
    needs: [ deploy-veda-backend ]
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: NASA-IMPACT/veda-config
          ref: add-playwright

      - name: Use Node.js 22
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
        with:
          node-version: 22

      - name: Run veda setup
        run: ./.veda/setup

      - name: Checkout generate_env script
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          sparse-checkout: |
            /scripts/generate_env_file.py
          sparse-checkout-cone-mode: false

      - name: Generate .env file from cdk.out
        shell: bash
        working-directory: ${{ env.DIRECTORY }}
        run: |
          pip install -r requirements.txt
          python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}"

      - name: Load .env file
        id: dotenv
        uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 #v1.1.4

      - name: Create env.local file
        env:
          MAPBOX_TOKEN: ${{secrets.MAPBOX_TOKEN}}
        run: |
          touch .env.local
          echo "API_RASTER_ENDPOINT=${{steps.dotenv.outputs.VEDA_RASTER_URL}}" >> .env.local
          echo "API_STAC_ENDPOINT=${{steps.dotenv.outputs.VEDA_STAC_URL}}" >> .env.local

          echo "Testing deployment URLs"
          cat .env.local

          echo "MAPBOX_TOKEN="$MAPBOX_TOKEN" >> .env.local

      - name: Install Playwright Browsers
        run: yarn playwright install --with-deps

      - name: Playwright tests
        run: yarn test:e2e
      - uses: actions/upload-artifact@c24449f33cd45d4826c6702db7e49f7cdb9b551d #v3.2.1-node20
        if: always()
        with:
          name: playwright-report
          path: playwright-report/
          retention-days: 30