CLI live test framework + initial API group testing

.gitignore

@@ -3,6 +3,7 @@
 /legal/
 /prebuilt/
 /test/bin/
+/test/live/bin/
 /vendor/
 
 # RPM/DEB

.semaphore/live-tests.yml

@@ -0,0 +1,49 @@
+version: v1.0
+name: Live Integration Tests for Confluent CLI
+agent:
+  machine:
+    type: s1-prod-ubuntu24-04-amd64-2
+
+auto_cancel:
+  running:
+    when: "false"
+
+global_job_config:
+  prologue:
+    commands:
+      - checkout
+      - sem-version go $(cat .go-version)
+      - export PATH=$(go env GOPATH)/bin:$PATH
+
+execution_time_limit:
+  hours: 24
+
+blocks:
+  - name: "Live Integration Tests"
+    task:
+      jobs:
+        - name: "Run Live Tests"
+          commands:
+            - . vault-sem-get-secret v1/ci/kv/apif/cli/live-testing-data
+            - . vault-sem-get-secret v1/ci/kv/apif/cli/slack-notifications-live-testing
+
+            - |
+              set -e
+
+              trap '
+                RC=$?
+                if [ $RC -ne 0 ]; then
+                  echo "Live tests failed, sending Slack notification..."
+                  curl -X POST -H "Content-type: application/json" --data "{}" "$SLACK_WEBHOOK_URL"
+                fi
+                exit $RC
+              ' EXIT
+
+              case "$CLI_LIVE_TEST_GROUPS" in
+                "essential") CLI_LIVE_TEST_GROUPS="core,kafka" ;;
+                "all"|"") CLI_LIVE_TEST_GROUPS="" ;;
+                *) CLI_LIVE_TEST_GROUPS="$CLI_LIVE_TEST_GROUPS" ;;
+              esac
+
+              echo "Running live tests for: ${CLI_LIVE_TEST_GROUPS:-all groups}"
+              make live-test ${CLI_LIVE_TEST_GROUPS:+CLI_LIVE_TEST_GROUPS="$CLI_LIVE_TEST_GROUPS"}

.semaphore/semaphore.yml

@@ -103,3 +103,24 @@ after_pipeline:
 promotions:
   - name: "Run macOS builds (manual)"
     pipeline_file: ".semaphore/macos.yml"
+  - name: "Run live integration tests"
+    pipeline_file: ".semaphore/live-tests.yml"
+    parameters:
+      env_vars:
+        - required: false
+          options:
+            - essential
+            - core
+            - kafka
+            - all
+          default_value: essential
+          description: "Test group to run (essential=core+kafka, all=everything)"
+          name: CLI_LIVE_TEST_GROUPS
+        - required: false
+          default_value: aws
+          description: "Cloud provider (aws, gcp, azure)"
+          name: CLI_LIVE_TEST_CLOUD
+        - required: false
+          default_value: us-east-1
+          description: "Cloud region for cluster creation"
+          name: CLI_LIVE_TEST_REGION

Makefile

@@ -129,6 +129,33 @@ endif
 .PHONY: test
 test: unit-test integration-test
 
+.PHONY: build-for-live-test
+build-for-live-test:
+	go build -ldflags="-s -w -X main.commit=00000000 -X main.date=1970-01-01T00:00:00Z -X main.disableUpdates=true" \
+		-o test/live/bin/confluent ./cmd/confluent
+
+.PHONY: live-test
+live-test: build-for-live-test
+	@if [ -z "$(CLI_LIVE_TEST_GROUPS)" ]; then \
+		CLI_LIVE_TEST=1 go test ./test/live/ -v -run=".*Live$$" \
+			-tags="live_test,all" -timeout 1440m -parallel 10; \
+	else \
+		TAGS="live_test"; \
+		for group in $$(echo "$(CLI_LIVE_TEST_GROUPS)" | tr ',' ' '); do \
+			TAGS="$$TAGS,$$group"; \
+		done; \
+		CLI_LIVE_TEST=1 go test ./test/live/ -v -run=".*Live$$" \
+			-tags="$$TAGS" -timeout 1440m -parallel 10; \
+	fi
+
+.PHONY: live-test-core
+live-test-core:
+	@$(MAKE) live-test CLI_LIVE_TEST_GROUPS="core"
+
+.PHONY: live-test-essential
+live-test-essential:
+	@$(MAKE) live-test CLI_LIVE_TEST_GROUPS="core,kafka"
+
 .PHONY: generate-packaging-patch
 generate-packaging-patch:
 	diff -u Makefile debian/Makefile | sed "1 s_Makefile_cli/Makefile_" > debian/patches/standard_build_layout.patch

test/live/README.md

@@ -0,0 +1,174 @@
+# CLI Live Integration Tests
+
+Live integration tests run the real CLI binary against Confluent Cloud. They create, read, update, and delete real resources to verify end-to-end CLI behavior.
+
+## Prerequisites
+
+1. **CLI binary** — Build with `make build-for-live-test`
+2. **Confluent Cloud credentials** — Set the following environment variables:
+
+| Variable | Required | Description |
+|---|---|---|
+| `CONFLUENT_CLOUD_EMAIL` | Yes | Confluent Cloud login email |
+| `CONFLUENT_CLOUD_PASSWORD` | Yes | Confluent Cloud login password |
+| `CLI_LIVE_TEST_CLOUD` | No | Cloud provider: `aws` (default), `gcp`, `azure` |
+| `CLI_LIVE_TEST_REGION` | No | Cloud region (default: `us-east-1`) |
+| `LIVE_TEST_ENVIRONMENT_ID` | Kafka topics only | Pre-existing environment ID for topic tests |
+| `KAFKA_STANDARD_AWS_CLUSTER_ID` | Kafka topics only | Pre-existing cluster ID for topic tests |
+
+## Running Tests
+
+### All tests
+```bash
+make live-test
+```
+
+### By group
+```bash
+make live-test-core        # environment, service account, API key
+make live-test-essential   # core + kafka
+make live-test CLI_LIVE_TEST_GROUPS="kafka"  # kafka only
+```
+
+### Multi-cloud
+```bash
+CLI_LIVE_TEST_CLOUD=gcp CLI_LIVE_TEST_REGION=us-east1 make live-test-essential
+```
+
+### Single test
+```bash
+CLI_LIVE_TEST=1 go test ./test/live/ -v -run TestLive/TestKafkaClusterCRUDLive \
+    -tags="live_test,kafka" -timeout 30m
+```
+
+## Test Groups
+
+Tests are organized into groups via Go build tags:
+
+| Group | Tag | Tests |
+|---|---|---|
+| Core | `core` | Environment, Service Account, API Key CRUD |
+| Kafka | `kafka` | Kafka Cluster CRUD, Kafka Topic CRUD |
+| All | `all` | Everything |
+
+The `essential` group in Semaphore/Makefile maps to `core,kafka`.
+
+## Concurrency Model
+
+- Each test method calls `s.setupTestContext(t)` which creates an **isolated HOME directory** and logs in. This means each test has its own CLI config — no shared state.
+- Tests opt in to concurrency by calling `t.Parallel()` at the start. All current tests do this.
+- The `-parallel 10` flag in the Makefile controls max concurrent tests.
+- Tests that need sequential execution (e.g., tests modifying shared external state) should simply omit the `t.Parallel()` call.
+
+## Writing a New Test
+
+### 1. Create a test file
+
+```go
+//go:build live_test && (all || mygroup)
+
+package live
+
+func (s *CLILiveTestSuite) TestMyResourceCRUDLive() {
+    t := s.T()
+    t.Parallel()
+    state := s.setupTestContext(t)
+
+    // ... test body ...
+}
+```
+
+The test method name **must** end with `Live` to match the `-run=".*Live$"` filter.
+
+### 2. Define test steps
+
+Use `CLILiveTest` structs for each CLI command:
+
+```go
+steps := []CLILiveTest{
+    {
+        Name:     "Create resource",
+        Args:     "resource create my-name -o json",
+        JSONFieldsExist: []string{"id"},
+        CaptureID: "resource_id",  // captures JSON "id" field into state
+    },
+    {
+        Name:         "Describe resource",
+        Args:         "resource describe {{.resource_id}} -o json",
+        UseStateVars: true,  // enables {{.key}} template substitution
+        JSONFields:   map[string]string{"name": "my-name"},
+    },
+}
+```
+
+### 3. Register cleanup
+
+Always register cleanup **before** creating resources (LIFO execution order):
+
+```go
+s.registerCleanup(t, "resource delete {{.resource_id}} --force", state)
+```
+
+### 4. Run steps
+
+```go
+for _, step := range steps {
+    t.Run(step.Name, func(t *testing.T) {
+        s.runLiveCommand(t, step, state)
+    })
+}
+```
+
+### CLILiveTest Field Reference
+
+| Field | Type | Description |
+|---|---|---|
+| `Name` | `string` | Step name shown in output |
+| `Args` | `string` | CLI arguments (supports `{{.key}}` when `UseStateVars` is true) |
+| `ExitCode` | `int` | Expected exit code (default 0) |
+| `Input` | `string` | Stdin content |
+| `Contains` | `[]string` | Strings that must appear in output |
+| `NotContains` | `[]string` | Strings that must NOT appear in output |
+| `Regex` | `[]string` | Regex patterns output must match |
+| `JSONFields` | `map[string]string` | JSON fields to check (empty value = any non-empty value) |
+| `JSONFieldsExist` | `[]string` | JSON fields that must exist (any value) |
+| `WantFunc` | `func(t, output, state)` | Custom assertion function |
+| `CaptureID` | `string` | State key to store extracted JSON "id" field |
+| `UseStateVars` | `bool` | Enable `{{.key}}` template substitution in Args |
+
+### Async Operations
+
+For operations that take time (e.g., cluster provisioning), use `waitForCondition`:
+
+```go
+s.waitForCondition(t,
+    "kafka cluster describe {{.cluster_id}} -o json",
+    state,
+    func(output string) bool {
+        return strings.EqualFold(extractJSONField(t, output, "status"), "UP")
+    },
+    30*time.Second,   // poll interval
+    10*time.Minute,   // timeout
+)
+```
+
+## Adding a New Test Group
+
+1. Create test file(s) with build tag: `//go:build live_test && (all || mygroup)`
+2. Add a Makefile target:
+   ```makefile
+   .PHONY: live-test-mygroup
+   live-test-mygroup:
+       @$(MAKE) live-test CLI_LIVE_TEST_GROUPS="mygroup"
+   ```
+3. Update the Semaphore promotion parameters if the group should be selectable in CI.
+
+## CI (Semaphore)
+
+Live tests are triggered via the "Run live integration tests" promotion in `.semaphore/semaphore.yml`. Parameters:
+
+- **CLI_LIVE_TEST_GROUPS** — Test group to run (default: `essential`)
+- **CLI_LIVE_TEST_CLOUD** — Cloud provider (default: `aws`)
+- **CLI_LIVE_TEST_REGION** — Cloud region (default: `us-east-1`)
+
+Credentials are loaded from Vault secrets in the Semaphore pipeline.

test/live/api_key_live_test.go

@@ -0,0 +1,87 @@
+//go:build live_test && (all || core)
+
+package live
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func (s *CLILiveTestSuite) TestApiKeyCRUDLive() {
+	t := s.T()
+	t.Parallel()
+	state := s.setupTestContext(t)
+
+	saName := uniqueName("apikey-sa")
+	apiKeyDescription := "Live test API key"
+	updatedDescription := "Updated live test API key"
+
+	// Cleanup in LIFO order: delete API key first, then service account
+	s.registerCleanup(t, "iam service-account delete {{.sa_id}} --force", state)
+	s.registerCleanup(t, "api-key delete {{.api_key_id}} --force", state)
+
+	steps := []CLILiveTest{
+		{
+			Name:     "Create service account for API key",
+			Args:     `iam service-account create ` + saName + ` --description "SA for API key live test" -o json`,
+			ExitCode: 0,
+			JSONFieldsExist: []string{"id"},
+			CaptureID:       "sa_id",
+		},
+		{
+			Name:            "Create API key",
+			Args:            `api-key create --resource cloud --service-account {{.sa_id}} --description "` + apiKeyDescription + `" -o json`,
+			UseStateVars:    true,
+			ExitCode:        0,
+			JSONFieldsExist: []string{"api_key", "api_secret"},
+			WantFunc: func(t *testing.T, output string, state *LiveTestState) {
+				t.Helper()
+				id := extractJSONField(t, output, "api_key")
+				require.NotEmpty(t, id, "failed to extract api_key from output:\n%s", output)
+				state.Set("api_key_id", id)
+				t.Logf("Captured api_key_id = %s", id)
+			},
+		},
+		{
+			Name:         "List API keys",
+			Args:         "api-key list --service-account {{.sa_id}}",
+			UseStateVars: true,
+			ExitCode:     0,
+			WantFunc: func(t *testing.T, output string, state *LiveTestState) {
+				t.Helper()
+				apiKeyID := state.Get("api_key_id")
+				if apiKeyID != "" {
+					require.Contains(t, output, apiKeyID, "API key list should contain the created key")
+				}
+			},
+		},
+		{
+			Name:         "Update API key description",
+			Args:         `api-key update {{.api_key_id}} --description "` + updatedDescription + `"`,
+			UseStateVars: true,
+			ExitCode:     0,
+		},
+		{
+			Name:         "Describe updated API key",
+			Args:         "api-key describe {{.api_key_id}} -o json",
+			UseStateVars: true,
+			ExitCode:     0,
+			JSONFields: map[string]string{
+				"description": updatedDescription,
+			},
+		},
+		{
+			Name:         "Delete API key",
+			Args:         "api-key delete {{.api_key_id}} --force",
+			UseStateVars: true,
+			ExitCode:     0,
+		},
+	}
+
+	for _, step := range steps {
+		t.Run(step.Name, func(t *testing.T) {
+			s.runLiveCommand(t, step, state)
+		})
+	}
+}