add pod clusterpolicy for mpijob pods

.github/workflows/_kyverno_policy.yaml

@@ -49,6 +49,10 @@ jobs:
           kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/kueue/v0.15.0/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
           kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/kueue/v0.15.0/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml
 
+      - name: Install mpi-operator CRDs
+        run: |
+          kubectl apply --server-side -f https://raw.githubusercontent.com/kubeflow/mpi-operator/v0.7.0/deploy/v2beta1/mpi-operator.yaml
+
       - name: Deploy and wait for Policies, ClusterRoles, and ClusterRoleBindings
         run: |
           helm template sessionspaces charts/sessionspaces | yq e '. | select(.kind == "Policy" or .kind == "ClusterPolicy" or .kind == "GeneratingPolicy" or .kind == "ClusterRole" or .kind == "ClusterRoleBinding")' | tee -a /tmp/policies.yaml | kubectl apply -f -

charts/sessionspaces/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: sessionspaces
 description: Namespace controller for creating session namespaces
 type: application
-version: 0.3.17
+version: 0.3.18
 appVersion: 0.1.3
 dependencies:
   - name: common

charts/sessionspaces/templates/pod-clusterpolicy.yaml

@@ -3,6 +3,8 @@ apiVersion: kyverno.io/v1
 kind: ClusterPolicy
 metadata:
   name: {{ include "common.names.fullname" $ }}-pod
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
 spec:
   background: false
   validationFailureAction: Enforce
@@ -16,6 +18,11 @@ spec:
           namespaceSelector:
             matchLabels:
               app.kubernetes.io/managed-by: sessionspaces
+      preconditions:
+        all:
+          - key: "{{ `{{request.object.metadata.labels.\"training.kubeflow.org/job-name\" || '' }}` }}"
+            operator: Equals
+            value: ""
       context:
         - name: namespace
           variable:
@@ -26,7 +33,70 @@ spec:
         - name: uid
           apiCall:
             urlPath: /apis/argoproj.io/v1alpha1/namespaces/{{ `{{ namespace }}` }}/workflows/{{ `{{ workflow }}` }}
-            jmesPath: 'metadata.labels | "workflows.diamond.ac.uk/creator-posix-uid"'
+            jmesPath: 'metadata.labels."workflows.diamond.ac.uk/creator-posix-uid"'
+        - name: values
+          configMap:
+            name: sessionspaces
+            namespace: "{{ `{{ namespace }}` }}"
+      mutate:
+        patchStrategicMerge:
+          spec:
+            securityContext:
+              runAsGroup: "{{ `{{values.data.gid | parse_json(@).to_number(@)}}` }}"
+              runAsUser: "{{ `{{ uid | parse_json(@).to_number(@) }}` }}"
+            containers:
+              - (name): "*"
+                securityContext:
+                  runAsGroup: "{{ `{{values.data.gid | parse_json(@).to_number(@)}}` }}"
+                  runAsUser: "{{ `{{ uid | parse_json(@).to_number(@) }}` }}"
+                  allowPrivilegeEscalation: false
+                  readOnlyRootFilesystem: true
+            initContainers:
+              - (name): "*"
+                securityContext:
+                  runAsGroup: "{{ `{{values.data.gid | parse_json(@).to_number(@)}}` }}"
+                  runAsUser: "{{ `{{ uid | parse_json(@).to_number(@) }}` }}"
+                  allowPrivilegeEscalation: false
+                  readOnlyRootFilesystem: true
+            ephemeralContainers:
+              - (name): "*"
+                securityContext:
+                  runAsGroup: "{{ `{{values.data.gid | parse_json(@).to_number(@)}}` }}"
+                  runAsUser: "{{ `{{ uid | parse_json(@).to_number(@) }}` }}"
+                  allowPrivilegeEscalation: false
+                  readOnlyRootFilesystem: true
+
+    - name: mpijob-pod-securitycontext
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaceSelector:
+            matchLabels:
+              app.kubernetes.io/managed-by: sessionspaces
+      preconditions:
+        all:
+          - key: "{{ `{{request.object.metadata.labels.\"training.kubeflow.org/job-name\" || '' }}` }}"
+            operator: NotEquals
+            value: ""
+      context:
+        - name: namespace
+          variable:
+            value: "{{ `{{request.object.metadata.namespace}}` }}"
+        - name: mpijobName
+          variable:
+            jmesPath: request.object.metadata.labels."training.kubeflow.org/job-name"
+        - name: mpijob
+          apiCall:
+            urlPath: /apis/kubeflow.org/v2beta1/namespaces/{{ `{{ namespace }}` }}/mpijobs/{{ `{{ mpijobName }}` }}
+            jmesPath: '@'
+        - name: workflowName
+          variable:
+            jmesPath: "mpijob.metadata.ownerReferences[?kind=='Workflow'] | [0].name"
+        - name: uid
+          apiCall:
+            urlPath: /apis/argoproj.io/v1alpha1/namespaces/{{ `{{ namespace }}` }}/workflows/{{ `{{ workflowName }}` }}
+            jmesPath: 'metadata.labels."workflows.diamond.ac.uk/creator-posix-uid"'
         - name: values
           configMap:
             name: sessionspaces

charts/sessionspaces/test-policy/pod-securitycontext/chainsaw-test.yaml

@@ -87,3 +87,128 @@ spec:
                       runAsUser: 4321
                       allowPrivilegeEscalation: false
                       readOnlyRootFilesystem: true
+        - create:
+            resource:
+              apiVersion: kubeflow.org/v2beta1
+              kind: MPIJob
+              metadata:
+                name: test-mpijob
+                ownerReferences:
+                  - name: test-workflow
+                    kind: Workflow
+                    apiVersion: argoproj.io/v1alpha1
+                    uid: ($workflow_uid)
+              spec:
+                mpiReplicaSpecs:
+                  Launcher:
+                    replicas: 1
+                    template:
+                      spec:
+                        containers:
+                          - name: launcher
+                            image: docker.io/library/busybox:latest
+                  Worker:
+                    replicas: 1
+                    template:
+                      spec:
+                        containers:
+                          - name: worker
+                            image: docker.io/library/busybox:latest
+        - command:
+            env:
+              - name: namespace
+                value: ($namespace)
+            entrypoint: kubectl
+            args:
+              - get
+              - mpijob
+              - test-mpijob
+              - --namespace=$namespace
+              - --output=jsonpath={.metadata.uid}
+            outputs:
+              - name: mpijob_uid
+                value: ($stdout)
+        - create:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: test-mpijob-launcher
+                labels:
+                  training.kubeflow.org/job-name: test-mpijob
+                  training.kubeflow.org/replica-type: launcher
+                ownerReferences:
+                  - name: test-mpijob
+                    kind: MPIJob
+                    apiVersion: kubeflow.org/v2beta1
+                    uid: ($mpijob_uid)
+              spec:
+                containers:
+                  - name: launcher
+                    image: docker.io/library/busybox:latest
+                initContainers:
+                  - name: init-container
+                    image: docker.io/library/busybox:latest
+        - assert:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: test-mpijob-launcher
+                labels:
+                  training.kubeflow.org/job-name: test-mpijob
+              spec:
+                securityContext:
+                  runAsGroup: 1234
+                  runAsUser: 4321
+                containers:
+                  - name: launcher
+                    securityContext:
+                      runAsGroup: 1234
+                      runAsUser: 4321
+                      allowPrivilegeEscalation: false
+                      readOnlyRootFilesystem: true
+                initContainers:
+                  - name: init-container
+                    securityContext:
+                      runAsGroup: 1234
+                      runAsUser: 4321
+                      allowPrivilegeEscalation: false
+                      readOnlyRootFilesystem: true
+        - create:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: test-mpijob-worker
+                labels:
+                  training.kubeflow.org/job-name: test-mpijob
+                  training.kubeflow.org/replica-type: worker
+                ownerReferences:
+                  - name: test-mpijob
+                    kind: MPIJob
+                    apiVersion: kubeflow.org/v2beta1
+                    uid: ($mpijob_uid)
+              spec:
+                containers:
+                  - name: worker
+                    image: docker.io/library/busybox:latest
+        - assert:
+            resource:
+              apiVersion: v1
+              kind: Pod
+              metadata:
+                name: test-mpijob-worker
+                labels:
+                  training.kubeflow.org/job-name: test-mpijob
+              spec:
+                securityContext:
+                  runAsGroup: 1234
+                  runAsUser: 4321
+                containers:
+                  - name: worker
+                    securityContext:
+                      runAsGroup: 1234
+                      runAsUser: 4321
+                      allowPrivilegeEscalation: false
+                      readOnlyRootFilesystem: true

charts/workflows/Chart.yaml

@@ -3,7 +3,7 @@ name: workflows
 description: Data Analysis workflow orchestration
 type: application
 
-version: 0.13.32
+version: 0.13.33
 
 dependencies:
   - name: argo-workflows

charts/workflows/templates/kyverno-clusterrole.yaml

@@ -67,3 +67,11 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - "kubeflow.org"
+  resources:
+  - mpijobs
+  verbs:
+  - get
+  - list
+  - watch