Merge pull request #53 from ONS-Innovation/concourse

Concourse pipeline

Author: delterr

concourse/ci.yml

@@ -0,0 +1,80 @@
+resources:
+- name: resource-repo
+  type: git
+  icon: github
+  source:
+    uri: https://github.com/ONS-Innovation/github-copilot-usage-dashboard
+    username: ((github_access_token))
+    password: x-oauth-basic
+    branch: ((branch))
+
+jobs:
+- name:  build-and-push
+  public: true
+  plan:
+  - get: resource-repo
+    timeout: 5m
+    trigger: true
+    timeout: 5m
+  - task: build-image
+    privileged: true
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source:
+          repository: hashicorp/terraform
+      inputs:
+      - name: resource-repo
+      params:
+        aws_account_id: ((aws_account_sdp_((env))))
+        aws_role_arn: arn:aws:iam::((aws_account_sdp_((env)))):role/sdp-concourse-((env))
+        secrets: ((sdp_((env))_github_copilot_secrets))
+      run: # binary used to build the image
+        path: sh
+        args:
+        - -cx
+        - |
+          apk add --no-cache aws-cli podman jq iptables curl
+
+          if [[ "((env))" == "prod" ]]; then
+            tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-copilot-usage-dashboard/releases" | jq -r '.[0].tag_name')
+            export tag
+          else
+            export tag=((tag))
+          fi
+          git rev-parse --abbrev-ref HEAD
+          chmod u+x ./resource-repo/concourse/scripts/assume_role.sh
+          chmod u+x ./resource-repo/concourse/scripts/build_image.sh
+          source ./resource-repo/concourse/scripts/assume_role.sh
+          ./resource-repo/concourse/scripts/build_image.sh
+    timeout: 10m
+  - task: terraform
+    privileged: true
+    config:
+      platform: linux
+      image_resource:
+        type: docker-image
+        source: {repository: hashicorp/terraform}
+      inputs:
+      - name: resource-repo
+      params:
+        secrets: ((sdp_((env))_github_copilot_secrets))
+        github_access_token: ((github_access_token))
+      run:
+        path: sh
+        args:
+        - -cx
+        - |
+          apk add --no-cache jq curl
+
+          if [[ "((env))" == "prod" ]]; then
+            tag=$(curl "https://api.github.com/repos/ONS-Innovation/github-copilot-usage-dashboard/releases" | jq -r '.[0].tag_name')
+            export tag
+          else 
+            export tag=((tag))
+          fi
+          chmod u+x ./resource-repo/concourse/scripts/terraform_infra.sh
+          export env=((env))
+          ./resource-repo/concourse/scripts/terraform_infra.sh
+    timeout: 30m

concourse/scripts/assume_role.sh

@@ -0,0 +1,12 @@
+set -euo pipefail
+
+aws sts assume-role --output text \
+    --role-arn "${aws_role_arn}" \
+    --role-session-name concourse-pipeline-run \
+    --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
+    | awk -F '\t' '{print $1 > ("AccessKeyId")}{print $2 > ("SecretAccessKey")}{print $3 > ("SessionToken")}'
+
+
+export AWS_ACCESS_KEY_ID="$(cat AccessKeyId)"
+export AWS_SECRET_ACCESS_KEY="$(cat SecretAccessKey)"
+export AWS_SESSION_TOKEN="$(cat SessionToken)"

concourse/scripts/build_image.sh

@@ -0,0 +1,14 @@
+set -euo pipefail
+
+export STORAGE_DRIVER=vfs
+export PODMAN_SYSTEMD_UNIT=concourse-task
+
+container_image=$(echo "$secrets" | jq -r .container_image)
+
+aws ecr get-login-password --region eu-west-2 | podman --storage-driver=vfs login --username AWS --password-stdin ${aws_account_id}.dkr.ecr.eu-west-2.amazonaws.com
+
+podman build -t ${container_image}:${tag} resource-repo/lambda_data_logger/
+
+podman tag ${container_image}:${tag} ${aws_account_id}.dkr.ecr.eu-west-2.amazonaws.com/${container_image}:${tag}
+
+podman push ${aws_account_id}.dkr.ecr.eu-west-2.amazonaws.com/${container_image}:${tag}

concourse/scripts/set_pipeline.sh

@@ -0,0 +1,27 @@
+repo_name=${1}
+
+if [[ $# -gt 1 ]]; then
+    branch=${2}
+    git rev-parse --verify ${branch}
+    if [[ $? -ne 0 ]]; then
+        echo "Branch \"${branch}\" does not exist"
+        exit 1
+    fi
+else
+    branch=$(git rev-parse --abbrev-ref HEAD)
+fi
+
+if [[ ${branch} == "main" || ${branch} == "master" ]]; then
+    env="prod"
+else
+    env="dev"
+fi
+
+if [[ ${env} == "dev" ]]; then
+    tag=$(git rev-parse HEAD)
+else
+    tag=$(git tag | tail -n 1)
+fi
+
+fly -t aws-sdp set-pipeline -c concourse/ci.yml -p ${repo_name}-${branch} -v branch=${branch} -v tag=${tag} -v env=${env}
+

concourse/scripts/terraform_infra.sh

@@ -0,0 +1,47 @@
+set -euo pipefail
+
+aws_account_id=$(echo "$secrets" | jq -r .aws_account_id)
+aws_access_key_id=$(echo "$secrets" | jq -r .aws_access_key_id)
+
+aws_secret_access_key=$(echo "$secrets" | jq -r .aws_secret_access_key)
+aws_secret_name=$(echo "$secrets" | jq -r .aws_secret_name)
+
+env_name=$(echo "$secrets" | jq -r .env_name)
+lambda_name=$(echo "$secrets" | jq -r .lambda_name)
+
+github_app_client_id=$(echo "$secrets" | jq -r .github_app_client_id)
+lambda_arch=$(echo "$secrets" | jq -r .lambda_arch)
+
+github_org=$(echo "$secrets" | jq -r .github_org)
+container_image=$(echo "$secrets" | jq -r .container_image)
+
+schedule=$(echo "$secrets" | jq -r .schedule)
+lambda_timeout=$(echo "$secrets" | jq -r .lambda_timeout)
+
+export AWS_ACCESS_KEY_ID=$aws_access_key_id
+export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key
+
+git config --global url."https://x-access-token:$github_access_token@github.com/".insteadOf "https://github.com/"
+
+if [[ ${env} != "prod" ]]; then
+    env="dev"
+fi
+
+cd resource-repo/terraform/data_logger/
+
+terraform init -backend-config=env/${env}/backend-${env}.tfbackend -reconfigure
+
+terraform apply \
+-var "aws_account_id=$aws_account_id" \
+-var "aws_access_key_id=$aws_access_key_id" \
+-var "aws_secret_access_key=$aws_secret_access_key" \
+-var "aws_secret_name=$aws_secret_name" \
+-var "env_name=$env_name" \
+-var "lambda_version=${tag}" \
+-var "lambda_name=$lambda_name" \
+-var "lambda_arch=$lambda_arch" \
+-var "lambda_timeout=$lambda_timeout" \
+-var "github_app_client_id=$github_app_client_id" \
+-var "github_org=$github_org" \
+-var "schedule=$schedule" \
+-auto-approve

lambda_data_logger/README.md

@@ -243,4 +243,39 @@ Delete the service resources by running the following ensuring your reference th
   terraform refresh -var-file=env/dev/dev.tfvars
 
   terraform destroy -var-file=env/dev/dev.tfvars
-  ```
+  ```
+
+### Deployments with Concourse
+
+#### Allowlisting your IP
+To setup the deployment pipeline with concourse, you must first allowlist your IP address on the Concourse
+server. IP addresses are flushed everyday at 00:00 so this must be done at the beginning of every working day
+whenever the deployment pipeline needs to be used. Follow the instructions on the Confluence page (SDP Homepage > SDP Concourse > Concourse Login) to
+login. All our pipelines run on sdp-pipeline-prod, whereas sdp-pipeline-dev is the account used for
+changes to Concourse instance itself. Make sure to export all necessary environment variables from sdp-pipeline-prod (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN).
+
+#### Setting up a pipeline
+When setting up our pipelines, we use ecs-infra-user on sdp-dev to be able to interact with our infrastructure on AWS. The credentials for this are stored on
+AWS Secrets Manager so you do not need to set up anything yourself.
+
+To set the pipeline, run the following script:
+```bash
+chmod u+x ./concourse/scripts/set_pipeline.sh
+./concourse/scripts/set_pipeline.sh github-copilot-usage-lambda
+```
+Note that you only have to run chmod the first time running the script in order to give permissions.
+This script will set the branch and pipeline name to whatever branch you are currently on. It will also set the image tag on ECR to the current commit hash at the time of setting the pipeline.
+
+The pipeline name itself will usually follow a pattern as follows: `<repo-name>-<branch-name>`
+If you wish to set a pipeline for another branch without checking out, you can run the following:
+```bash
+./concourse/scripts/set_pipeline.sh github-copilot-usage-lambda <branch_name>
+```
+
+If the branch you are deploying is "main" or "master", it will trigger a deployment to the sdp-prod environment. To set the ECR image tag, you must draft a Github release pointing to the latest release of the main/master branch that has a tag in the form of vX.Y.Z. Drafting up a release will automatically deploy the latest version of the main/master branch with the associated release tag, but you can also manually trigger a build through the Concourse UI or the terminal prompt.
+
+#### Triggering a pipeline
+Once the pipeline has been set, you can manually trigger a build on the Concourse UI, or run the following command:
+```bash
+fly -t aws-sdp trigger-job -j github-copilot-usage-lambda-<branch-name>/build-and-push
+```

terraform/data_logger/providers.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 5.0"
+      version = "~> 6.2.0"
     }
   }
 }