add e2e tests to all nodes clusters

.pipelines/azure_pipeline_mergedbranches.yaml

@@ -1050,3 +1050,19 @@ extends:
           imageRepository: $(ImageRepositoryOverride)
           environment: 'CI-Agent-Dev-Aks-All-Nodes'
           azureSubscription: 'ContainerInsights_Build_Subscription_CI'
+
+      # Cluster: ci-logs-dev-aks-all-nodes — Run E2E Tests
+      # PerNodeLogCoverage=true enables the per-node ContainerLogV2 ingestion
+      # check; this cluster has a log generator deployed on every node so
+      # every node is expected to ship logs.
+      - template: /.pipelines/e2e-test-templates/test-ci-image-in-aks-cluster.yml@self
+        parameters:
+          clusterName: 'ci-logs-dev-aks-all-nodes'
+          resourceGroup: 'ci-logs-dev-aks-eastus'
+          azureSubscription: 'ContainerInsights_Build_Subscription_CI'
+          environmentName: 'CI-Agent-Dev-Aks-All-Nodes'
+          dependsOnDeployJob: 'Deploy_AmaLogs_ci_logs_dev_aks_all_nodes'
+          azureClientId: 'AllNodesClientId'
+          azureTenantId: $(CI_BUILD_AZURE_TENANT_ID)
+          teamsWebhookUri: $(TeamsWebhookUri)
+          additionalTestParams: 'PerNodeLogCoverage=true'

test/scenario/log-gen-all-nodes-daemonset.yaml

@@ -0,0 +1,90 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: log-gen-all-nodes
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-gen-linux
+  namespace: log-gen-all-nodes
+  labels:
+    app: log-gen
+    os: linux
+spec:
+  selector:
+    matchLabels:
+      app: log-gen
+      os: linux
+  template:
+    metadata:
+      labels:
+        app: log-gen
+        os: linux
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: log-gen
+        image: mcr.microsoft.com/cbl-mariner/busybox:2.0
+        command: ["sh", "-c"]
+        args:
+        - 'i=0; while true; do i=$((i+1)); echo "log-gen-linux line $i node=$NODE_NAME ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)"; sleep 1; done'
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        resources:
+          requests:
+            cpu: 5m
+            memory: 16Mi
+          limits:
+            cpu: 50m
+            memory: 64Mi
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-gen-windows
+  namespace: log-gen-all-nodes
+  labels:
+    app: log-gen
+    os: windows
+spec:
+  selector:
+    matchLabels:
+      app: log-gen
+      os: windows
+  template:
+    metadata:
+      labels:
+        app: log-gen
+        os: windows
+    spec:
+      nodeSelector:
+        kubernetes.io/os: windows
+        kubernetes.azure.com/os-sku: Windows2022
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: log-gen
+        image: mcr.microsoft.com/windows/servercore:ltsc2022
+        command: ["powershell", "-c"]
+        args:
+        - '$i = 0; while ($true) { $i++; Write-Output "log-gen-windows line $i node=$env:NODE_NAME ts=$(Get-Date -Format o)"; Start-Sleep -Seconds 1 }'
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        resources:
+          requests:
+            cpu: 50m
+            memory: 256Mi
+          limits:
+            cpu: 200m
+            memory: 512Mi

test/testkube/install-and-execute-testkube-tests.sh

@@ -11,6 +11,7 @@ do
            TeamsWebhookUri) TeamsWebhookUri=$VALUE ;;
            LinuxTestsOnly) LinuxTestsOnly=$VALUE ;;
            GenevaIntegration) GenevaIntegration=$VALUE ;;
+           PerNodeLogCoverage) PerNodeLogCoverage=$VALUE ;;
            *)
     esac
 done
@@ -66,6 +67,7 @@ export AZURE_CLIENT_ID=$AzureClientId
 export AZURE_TENANT_ID=$AzureTenantId
 export WEBHOOK_URI=$TeamsWebhookUri
 export GENEVA_INTEGRATION=$GenevaIntegration
+export PER_NODE_LOG_COVERAGE=$PerNodeLogCoverage
 kubectl apply -f ./api-server-permissions.yaml
 kubectl apply -f ./testkube-test-crs.yaml
 
@@ -88,6 +90,7 @@ for wf in "${workflows[@]}"; do
     echo "Running workflow: $wf"
     kubectl testkube run testworkflow "$wf" \
         --config GENEVA_INTEGRATION="$GENEVA_INTEGRATION" \
+        --config PER_NODE_LOG_COVERAGE="$PER_NODE_LOG_COVERAGE" \
         --config AZURE_TENANT_ID="$AZURE_TENANT_ID" \
         --config AZURE_CLIENT_ID="$AZURE_CLIENT_ID" \
         --config GOTOOLCHAIN="auto" \

test/testkube/testkube-test-crs.yaml

@@ -11,6 +11,9 @@ spec:
     GENEVA_INTEGRATION:
       type: string
       default: "false"
+    PER_NODE_LOG_COVERAGE:
+      type: string
+      default: "false"
     GOTOOLCHAIN:
       type: string
       default: ""
@@ -49,6 +52,9 @@ spec:
     GENEVA_INTEGRATION:
       type: string
       default: "false"
+    PER_NODE_LOG_COVERAGE:
+      type: string
+      default: "false"
     GOTOOLCHAIN:
       type: string
       default: ""
@@ -124,6 +130,9 @@ spec:
     GENEVA_INTEGRATION:
       type: string
       default: "false"
+    PER_NODE_LOG_COVERAGE:
+      type: string
+      default: "false"
     GOTOOLCHAIN:
       type: string
       default: ""
@@ -145,6 +154,8 @@ spec:
         value: "{{config.AZURE_CLIENT_ID}}"
       - name: GENEVA_INTEGRATION
         value: "{{config.GENEVA_INTEGRATION}}"
+      - name: PER_NODE_LOG_COVERAGE
+        value: "{{config.PER_NODE_LOG_COVERAGE}}"
       - name: GOTOOLCHAIN
         value: "{{config.GOTOOLCHAIN}}"
     shell: ginkgo ./querylogs