Ready: WPB-24090 Refactored terraform logic for CD purposes

.gitignore

@@ -30,6 +30,7 @@ secrets_cache/
 
 terraform.tfstate
 terraform.tfstate.backup
+*.auto.tfvars.json
 kubeconfig.new
 .vscode/*
 

ansible/inventory/demo/host.yml

@@ -39,6 +39,9 @@ wiab:
     pod_network_cidr: "10.233.0.0/16"
     minikube_node_subnet: "192.168.99.0/24"
 
+    # will dump logs on failure when deploying helm charts
+    dump_logs_on_failure: true
+    # will use certmanager for certs
     use_cert_manager: true
     # networking iptables dnat rules
     http_dnat_rules:

ansible/wiab-demo/helm_install.yml

@@ -117,6 +117,13 @@
 
   - name: Deploy core Wire service Helm charts
     block:
+    - name: Reset core Helm deployment status
+      set_fact:
+        helm_deploy_failed: false
+        helm_deploy_failure_task: ''
+        helm_deploy_failure_message: ''
+        deployment_messages: []
+
     - name: Display charts that will be deployed
       debug:
         msg: "Following charts will be deployed: {{ charts_to_deploy | join(', ') }}"
@@ -165,40 +172,54 @@
       loop: "{{ charts_to_deploy }}"
       register: helm_deploy_result
 
-    - name: Report deployment status for all core charts
-      block:
+    rescue:
+      - name: Store core Helm deployment failure details
+        set_fact:
+          helm_deploy_failed: true
+          helm_deploy_failure_task: "{{ ansible_failed_task.name | default('Deploy core Wire charts using their available configuration files') }}"
+          helm_deploy_failure_message: "{{ ansible_failed_result.msg | default(ansible_failed_result.stderr | default('Unknown error during Helm chart deployment')) }}"
+
+    always:
       - name: Build deployment status list
         set_fact:
-          deployment_messages: "{{ deployment_messages | default([]) + [item.item + ': ' + ('Deployed' if item.changed else 'Already up-to-date')] }}"
-        loop: "{{ helm_deploy_result.results }}"
-        when: helm_deploy_result.results is defined
+          deployment_messages: "{{ deployment_messages + [item.item + ': ' + ('Failed' if item.failed | default(false) else ('Deployed' if item.changed | default(false) else 'Already up-to-date'))] }}"
+        loop: "{{ helm_deploy_result.results | default([]) }}"
         no_log: true
 
       - name: Display chart deployment status
         debug:
           msg: "{{ ['Chart deployment status:'] + (deployment_messages | map('regex_replace', '^', '- ') | list) }}"
-        when: helm_deploy_result.results is defined
+        when: deployment_messages | length > 0
 
-    - name: Retrieve running pods from default namespace
-      kubernetes.core.k8s_info:
-        kind: Pod
-        namespace: default
-        kubeconfig: "{{ kube_config }}"
-      register: pods_info
+      - name: Display core Helm deployment failure details
+        debug:
+          msg:
+            - "Core Wire chart deployment failed."
+            - "Failed task: {{ helm_deploy_failure_task }}"
+            - "Error: {{ helm_deploy_failure_message }}"
+        when: helm_deploy_failed | default(false)
+
+      - name: Retrieve running pods from default namespace
+        kubernetes.core.k8s_info:
+          kind: Pod
+          namespace: default
+          kubeconfig: "{{ kube_config }}"
+        register: pods_info
 
-    - name: Display running pods sorted by creation time
-      block:
       - name: Count running pods
         set_fact:
-          running_pods_count: "{{ pods_info.resources | length }}"
+          total_pods_count: "{{ pods_info.resources | length }}"
+          all_pods: "{{ pods_info.resources | map(attribute='metadata.name') | list }}"
+          failing_pod_resources: "{{ pods_info.resources | rejectattr('status.phase', 'equalto', 'Running') | rejectattr('status.phase', 'equalto', 'Succeeded') | list }}"
+          failing_pods: "{{ pods_info.resources | rejectattr('status.phase', 'equalto', 'Running') | rejectattr('status.phase', 'equalto', 'Succeeded') | map(attribute='metadata.name') | list }}"
           running_pods: "{{ pods_info.resources | selectattr('status.phase', 'equalto', 'Running') | map(attribute='metadata.name') | list }}"
           succeeded_pods: "{{ pods_info.resources | selectattr('status.phase', 'equalto', 'Succeeded') | map(attribute='metadata.name') | list }}"
           pending_pods: "{{ pods_info.resources | selectattr('status.phase', 'equalto', 'Pending') | map(attribute='metadata.name') | list }}"
 
       - name: Display pods summary
         debug:
           msg:
-            - "Total running pods: {{ running_pods_count }}"
+            - "Total pods: {{ total_pods_count }}"
             - ""
             - "Running ({{ running_pods | length }}):"
             - "{{ running_pods | map('regex_replace', '^', '  - ') | list }}"
@@ -209,6 +230,59 @@
             - "Pending ({{ pending_pods | length }}):"
             - "{{ pending_pods | map('regex_replace', '^', '  - ') | list }}"
 
+      - name: Display failing pod details
+        debug:
+          msg: |
+            Failing pod details:
+            {{ failing_pod_resources | to_nice_yaml }}
+        when:
+          - helm_deploy_failed | default(false)
+          - dump_logs_on_failure | default(false)
+          - failing_pod_resources | length > 0
+
+      - name: Note when no failing pods were found
+        debug:
+          msg: "No failing pods found in pod inventory; skipping pod detail dump."
+        when:
+          - helm_deploy_failed | default(false)
+          - dump_logs_on_failure | default(false)
+          - failing_pod_resources | length == 0
+
+      - name: Collect logs from all pods
+        kubernetes.core.k8s_log:
+          name: "{{ item }}"
+          namespace: default
+          kubeconfig: "{{ kube_config }}"
+          tail_lines: 30
+          all_containers: true
+        failed_when: false
+        register: pod_logs
+        loop: "{{ all_pods }}"
+        when:
+          - helm_deploy_failed | default(false)
+          - dump_logs_on_failure | default(false)
+
+      - name: Display pod logs
+        debug:
+          msg: |
+            Pod logs for {{ item.item }}:
+            {% if item.failed | default(false) %}
+            Failed to collect logs: {{ item.msg | default('Unknown error') }}
+            {% else %}
+            {{ item.log | default(item.content | default('No logs returned')) }}
+            {% endif %}
+        loop: "{{ pod_logs.results | default([]) }}"
+        loop_control:
+          label: "{{ item.item }}"
+        when:
+          - helm_deploy_failed | default(false)
+          - dump_logs_on_failure | default(false)
+
+      - name: Stop play after core Helm deployment failure
+        fail:
+          msg: "Core Wire chart deployment failed in task '{{ helm_deploy_failure_task }}': {{ helm_deploy_failure_message }}"
+        when: helm_deploy_failed | default(false)
+
   - name: Deploy nginx-ingress-services with TLS configuration
     block:
 

ansible/wiab-demo/wire_secrets.yml

@@ -369,6 +369,120 @@
 
       when: "'postgresql' in charts_to_deploy"
 
+    - name: Manage MLS private keys for galley
+      block:
+        - name: Create temporary directory for MLS key files
+          tempfile:
+            state: directory
+            suffix: _mls_keys
+          register: mls_temp_dir
+          changed_when: false
+
+        - name: Generate MLS private keys using openssl
+          shell: >-
+            openssl genpkey {{ item.command }} -out '{{ mls_temp_dir.path }}/{{ item.filename }}' 2>/dev/null
+          args:
+            executable: /bin/bash
+          changed_when: false
+          no_log: true
+          loop:
+            - name: mls_ed25519_key
+              filename: ed25519.pem
+              command: "-algorithm ed25519"
+              pem_bytes: 119
+              der_bytes: 48
+            - name: mls_ecdsa_p256_key
+              filename: ecdsa_p256.pem
+              command: "-algorithm ec -pkeyopt ec_paramgen_curve:P-256"
+              pem_bytes: 241
+              der_bytes: 121
+            - name: mls_ecdsa_p384_key
+              filename: ecdsa_p384.pem
+              command: "-algorithm ec -pkeyopt ec_paramgen_curve:P-384"
+              pem_bytes: 306
+              der_bytes: 167
+            - name: mls_ecdsa_p521_key
+              filename: ecdsa_p521.pem
+              command: "-algorithm ec -pkeyopt ec_paramgen_curve:P-521"
+              pem_bytes: 384
+              der_bytes: 223
+
+        - name: Read generated MLS private key files
+          slurp:
+            src: "{{ mls_temp_dir.path }}/{{ item.filename }}"
+          register: mls_key_files
+          no_log: true
+          loop:
+            - name: mls_ed25519_key
+              filename: ed25519.pem
+              pem_bytes: 119
+              der_bytes: 48
+            - name: mls_ecdsa_p256_key
+              filename: ecdsa_p256.pem
+              pem_bytes: 241
+              der_bytes: 121
+            - name: mls_ecdsa_p384_key
+              filename: ecdsa_p384.pem
+              pem_bytes: 306
+              der_bytes: 167
+            - name: mls_ecdsa_p521_key
+              filename: ecdsa_p521.pem
+              pem_bytes: 384
+              der_bytes: 223
+
+        - name: Set MLS private keys as facts
+          set_fact:
+            "{{ item.item.name }}": "{{ item.content | b64decode }}"
+          loop: "{{ mls_key_files.results }}"
+          no_log: true
+
+        - name: Validate exact MLS private key PEM sizes
+          assert:
+            that:
+              - item.content | b64decode | length == item.item.pem_bytes
+            fail_msg: "MLS private key PEM size mismatch for {{ item.item.name }}"
+            quiet: yes
+          loop: "{{ mls_key_files.results }}"
+          no_log: true
+
+        - name: Validate exact MLS private key DER sizes
+          shell: >-
+            openssl pkey -in '{{ mls_temp_dir.path }}/{{ item.filename }}' -outform DER 2>/dev/null | wc -c
+          args:
+            executable: /bin/bash
+          register: mls_key_der_sizes
+          changed_when: false
+          no_log: true
+          loop:
+            - name: mls_ed25519_key
+              filename: ed25519.pem
+              der_bytes: 48
+            - name: mls_ecdsa_p256_key
+              filename: ecdsa_p256.pem
+              der_bytes: 121
+            - name: mls_ecdsa_p384_key
+              filename: ecdsa_p384.pem
+              der_bytes: 167
+            - name: mls_ecdsa_p521_key
+              filename: ecdsa_p521.pem
+              der_bytes: 223
+
+        - name: Assert exact MLS private key DER sizes
+          assert:
+            that:
+              - item.stdout | int == item.item.der_bytes
+            fail_msg: "MLS private key DER size mismatch for {{ item.item.name }}"
+            quiet: yes
+          loop: "{{ mls_key_der_sizes.results }}"
+
+      always:
+        - name: Cleanup MLS temporary directory
+          file:
+            path: "{{ mls_temp_dir.path }}"
+            state: absent
+          changed_when: false
+          when: mls_temp_dir.path is defined
+
     - name: Configure wire-server service secrets (brig, nginz, cargohold, galley)
       block:
         - name: Check if wire-server secrets file exists
@@ -405,6 +519,14 @@
                 secrets:
                   awsKeyId: "{{ minio_access_key }}"
                   awsSecretKey: "{{ minio_secret_key }}"
+              galley:
+                secrets:
+                  mlsPrivateKeys:
+                    removal:
+                      ed25519: "{{ mls_ed25519_key }}"
+                      ecdsa_secp256r1_sha256: "{{ mls_ecdsa_p256_key }}"
+                      ecdsa_secp384r1_sha384: "{{ mls_ecdsa_p384_key }}"
+                      ecdsa_secp521r1_sha512: "{{ mls_ecdsa_p521_key }}"
             no_log: true
 
         - name: Add pgPassword to update dictionary

bin/helm-operations.sh

@@ -78,7 +78,7 @@ process_values() {
 
   ENV=$1
   TYPE=$2
-  charts=(fake-aws demo-smtp rabbitmq databases-ephemeral reaper wire-server webapp account-pages team-settings ingress-nginx-controller)
+  charts=(fake-aws smtp rabbitmq databases-ephemeral reaper wire-server webapp account-pages team-settings ingress-nginx-controller)
 
   if [[ "$DEPLOY_CERT_MANAGER" == "TRUE" ]]; then
     charts+=(nginx-ingress-services cert-manager)

changelog.d/2-wire-builds/release-5.25

@@ -0,0 +1,2 @@
+Fixed: update reference for 5.25 to 5.25.21 without any pinned component
+Added: logging in case of helm chart failure

changelog.d/2-wire-builds/smpt-values-fix

@@ -0,0 +1 @@
+Fixed: values for smtp helm chart

changelog.d/3-deploy-builds/heztner-deployment

@@ -0,0 +1 @@
+Fixed: Refactored terraform logic for CD purposes for all solutions wiab-dev(demo), wiab-staging and default (equivalent). All logic to pick up the region and server type remains in the respective scripts, there will be an iteration over regions first, terraform would just validate the regions and server types 

changelog.d/3-deploy-builds/wiab-dev-mls-kys

@@ -0,0 +1,2 @@
+Fixed: smtp helm chart values 
+Fixed: issue due to requirement of mls keys for webapp for wiab-dev when MLS is not required