Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,14 @@
{{- range ($cfg.providers | default (list)) }}
{{- $cmd := (.execConfig | default dict).command | default "" }}
{{- if $cmd }}
{{- $cmdDir := dir $cmd }}
{{- if not (has $cmdDir $mountPaths) }}
{{- fail (printf "provider %q: command dir %q does not match any plugins[].mountPath %v" .name $cmdDir $mountPaths) }}
{{- $matched := false }}
{{- range $mountPath := $mountPaths }}
{{- if or (eq $cmd $mountPath) (hasPrefix (printf "%s/" $mountPath) $cmd) }}
{{- $matched = true }}
{{- end }}
{{- end }}
{{- if not $matched }}
{{- fail (printf "provider %q: command %q is not under any plugins[].mountPath %v" .name $cmd $mountPaths) }}
{{- end }}
{{- end }}
{{- end }}
Expand Down
10 changes: 6 additions & 4 deletions config/charts/knative-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,23 @@ knative_operator:
# - name: secretreader
# execConfig:
# apiVersion: client.authentication.k8s.io/v1
# command: /access-plugins/secretreader/secretreader-plugin
# command: /access-plugins/secretreader/bin/secretreader-plugin
# interactiveMode: Never
# provideClusterInfo: true
# - name: kubeconfig-secretreader
# execConfig:
# apiVersion: client.authentication.k8s.io/v1
# command: /access-plugins/secretreader/kubeconfig-secretreader-plugin
# command: /access-plugins/kubeconfig-secretreader/bin/kubeconfig-secretreader-plugin
# interactiveMode: Never
# provideClusterInfo: true
# plugins[] uses the Kubernetes "image" volume type
plugins: []
# plugins:
# - name: secretreader
# image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.0
# image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.2
# mountPath: /access-plugins/secretreader
# - name: kubeconfig-secretreader
# image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.0
# image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.2
# mountPath: /access-plugins/kubeconfig-secretreader
# Polling cadence for spoke deployment readiness.
remoteDeploymentsPollInterval: 10s
Expand Down
17 changes: 9 additions & 8 deletions docs/development/e2e-multicluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,13 @@ Apply the operator from source:
ko apply -Rf config/
```

Generate a spoke bootstrap token and mount the access provider plumbing. The
helper script does this end to end:
Generate a spoke bootstrap token, store it as `knative-operator/${SPOKE_CLUSTER_NAME}`,
and mount the official `secretreader` image as the access provider. The helper
script does this end to end:

```bash
source test/e2e-common.sh
install_access_provider_config # builds and installs the token-exec-plugin
install_access_provider_config # installs the secretreader access provider
apply_cluster_profile default # creates the ClusterProfile on the hub
```

Expand All @@ -70,11 +71,11 @@ A minimal provider config (written by the helper to
{
"providers": [
{
"name": "e2e-static-token",
"name": "secretreader",
"execConfig": {
"apiVersion": "client.authentication.k8s.io/v1",
"command": "/etc/cluster-inventory/plugin/ko-app/token-exec-plugin",
"args": ["/etc/cluster-inventory/access/token"],
"command": "/etc/cluster-inventory/plugin/bin/secretreader-plugin",
"provideClusterInfo": true,
"interactiveMode": "Never"
}
}
Expand Down Expand Up @@ -205,8 +206,8 @@ which reuses the same helpers this guide calls manually:
`AccessProviderFailed`, exec into the operator and run the plugin manually:
```bash
kubectl -n knative-operator exec deploy/knative-operator -- \
/etc/cluster-inventory/plugin/ko-app/token-exec-plugin \
/etc/cluster-inventory/access/token
env KUBERNETES_EXEC_INFO="{\"apiVersion\":\"client.authentication.k8s.io/v1\",\"kind\":\"ExecCredential\",\"spec\":{\"cluster\":{\"server\":\"https://debug.invalid\",\"config\":{\"clusterName\":\"${SPOKE_CLUSTER_NAME}\"}}}}" \
/etc/cluster-inventory/plugin/bin/secretreader-plugin
```

## 8. Cleanup
Expand Down
20 changes: 15 additions & 5 deletions docs/multicluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,25 @@ knative_operator:
enabled: true
accessProvidersConfig:
providers:
- name: token-secretreader
- name: secretreader
execConfig:
apiVersion: client.authentication.k8s.io/v1
command: /access-plugins/token-secretreader/kubeconfig-secretreader-plugin
command: /access-plugins/secretreader/bin/secretreader-plugin
interactiveMode: Never
provideClusterInfo: true
- name: kubeconfig-secretreader
execConfig:
apiVersion: client.authentication.k8s.io/v1
command: /access-plugins/kubeconfig-secretreader/bin/kubeconfig-secretreader-plugin
interactiveMode: Never
provideClusterInfo: true
plugins:
- name: token-secretreader
image: ghcr.io/example/plugin:v1.0.0
mountPath: /access-plugins/token-secretreader
- name: secretreader
image: registry.k8s.io/cluster-inventory-api/secretreader:v0.1.2
mountPath: /access-plugins/secretreader
- name: kubeconfig-secretreader
image: registry.k8s.io/cluster-inventory-api/kubeconfig-secretreader:v0.1.2
mountPath: /access-plugins/kubeconfig-secretreader
```

The chart creates a `ConfigMap` with the provider config and mounts each
Expand Down
49 changes: 0 additions & 49 deletions test/cmd/token-exec-plugin/main.go

This file was deleted.

4 changes: 4 additions & 0 deletions test/config/multicluster/clusterprofile-status.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ status:
cluster:
server: "${SPOKE_INTERNAL_ENDPOINT}"
certificate-authority-data: "${SPOKE_CA_DATA_B64}"
extensions:
- name: client.authentication.k8s.io/exec
extension:
clusterName: "${SPOKE_CLUSTER_NAME}"
conditions:
- type: ControlPlaneHealthy
status: "True"
Expand Down
29 changes: 12 additions & 17 deletions test/e2e-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ export KO_FLAGS="${KO_FLAGS:-}"
export INGRESS_CLASS=${INGRESS_CLASS:-istio.ingress.networking.knative.dev}
export TIMEOUT_CI=30m

: "${MC_PROVIDER_CONFIGMAP:=clusterprofile-provider-file}"
: "${MC_PROVIDER_MOUNT_PATH:=/etc/cluster-inventory}"
: "${MC_PROVIDER_PLUGIN_MOUNT_PATH:=/etc/cluster-inventory/plugin}"
: "${MC_PROVIDER_PLUGIN_IMAGE:=registry.k8s.io/cluster-inventory-api/secretreader:v0.1.2}"
: "${MC_PROVIDER_NAME:=secretreader}"

# Boolean used to indicate whether to generate serving YAML based on the latest code in the branch KNATIVE_SERVING_REPO_BRANCH.
GENERATE_SERVING_YAML=0

Expand Down Expand Up @@ -464,8 +470,9 @@ function apply_cluster_profile() {
SPOKE_INTERNAL_ENDPOINT="${spoke_endpoint}" \
SPOKE_CA_DATA_B64="${spoke_ca_b64}" \
MC_PROVIDER_NAME="${MC_PROVIDER_NAME}" \
SPOKE_CLUSTER_NAME="${SPOKE_CLUSTER_NAME}" \
TRANSITION="$(date -u +%FT%TZ)" \
envsubst '${SPOKE_INTERNAL_ENDPOINT} ${SPOKE_CA_DATA_B64} ${MC_PROVIDER_NAME} ${TRANSITION}' \
envsubst '${SPOKE_INTERNAL_ENDPOINT} ${SPOKE_CA_DATA_B64} ${MC_PROVIDER_NAME} ${SPOKE_CLUSTER_NAME} ${TRANSITION}' \
< test/config/multicluster/clusterprofile-status.yaml.tmpl \
> "${status_file}" || return 1

Expand All @@ -477,23 +484,14 @@ function apply_cluster_profile() {
}

function install_access_provider_config() {
echo ">> Building token-exec-plugin image via ko"
local plugin_image
plugin_image="$(ko build ./test/cmd/token-exec-plugin)" || return 1
if [[ -z "${plugin_image}" ]]; then
echo "ERROR: ko build did not emit an image reference for token-exec-plugin" >&2
return 1
fi
echo ">> token-exec-plugin image: ${plugin_image}"

echo ">> Installing access provider ConfigMap/Secret and patching operator deployment"
local tmpdir
tmpdir="$(mktemp -d)" || return 1
add_trap "rm -rf ${tmpdir}" EXIT
local token_file="${tmpdir}/token"
_spoke_bootstrap_token "${token_file}" || return 1

local plugin_command="${MC_PROVIDER_PLUGIN_MOUNT_PATH}/ko-app/token-exec-plugin"
local plugin_command="${MC_PROVIDER_PLUGIN_MOUNT_PATH}/bin/secretreader-plugin"
cat > "${tmpdir}/provider-config.json" <<EOF
{
"providers": [
Expand All @@ -502,7 +500,7 @@ function install_access_provider_config() {
"execConfig": {
"apiVersion": "client.authentication.k8s.io/v1",
"command": "${plugin_command}",
"args": ["${MC_PROVIDER_TOKEN_MOUNT_PATH}/token"],
"provideClusterInfo": true,
"interactiveMode": "Never"
}
}
Expand All @@ -514,24 +512,21 @@ EOF
--from-file=config.json="${tmpdir}/provider-config.json" \
--dry-run=client -o yaml | kubectl apply -f - || return 1

kubectl -n "${TEST_OPERATOR_NAMESPACE}" create secret generic "${MC_PROVIDER_TOKEN_SECRET}" \
kubectl -n "${TEST_OPERATOR_NAMESPACE}" create secret generic "${SPOKE_CLUSTER_NAME}" \
--from-file=token="${token_file}" \
--dry-run=client -o yaml | kubectl apply -f - || return 1

# 0444
kubectl -n "${TEST_OPERATOR_NAMESPACE}" patch deployment knative-operator \
--type=json \
-p "$(cat <<EOF
[
{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": ["--clusterprofile-provider-file=${MC_PROVIDER_MOUNT_PATH}/config.json"]},
{"op": "add", "path": "/spec/template/spec/volumes", "value": [
{"name": "access-config", "configMap": {"name": "${MC_PROVIDER_CONFIGMAP}"}},
{"name": "provider-token", "secret": {"secretName": "${MC_PROVIDER_TOKEN_SECRET}", "defaultMode": 292}},
{"name": "access-plugin", "image": {"reference": "${plugin_image}", "pullPolicy": "IfNotPresent"}}
{"name": "access-plugin", "image": {"reference": "${MC_PROVIDER_PLUGIN_IMAGE}", "pullPolicy": "IfNotPresent"}}
]},
{"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts", "value": [
{"name": "access-config", "mountPath": "${MC_PROVIDER_MOUNT_PATH}", "readOnly": true},
{"name": "provider-token", "mountPath": "${MC_PROVIDER_TOKEN_MOUNT_PATH}", "readOnly": true},
{"name": "access-plugin", "mountPath": "${MC_PROVIDER_PLUGIN_MOUNT_PATH}", "readOnly": true}
]}
]
Expand Down
12 changes: 3 additions & 9 deletions test/e2e-tests-multicluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,11 @@ source "$(dirname "$0")/e2e-common.sh"
: "${SPOKE_KUBECONFIG:=/tmp/spoke.kubeconfig}"
: "${SPOKE_HOST_KUBECONFIG:=/tmp/spoke-host.kubeconfig}"
: "${CLUSTER_INVENTORY_CRD_URL:=https://raw.githubusercontent.com/kubernetes-sigs/cluster-inventory-api/v0.1.0/config/crd/bases/multicluster.x-k8s.io_clusterprofiles.yaml}"
: "${MC_PROVIDER_CONFIGMAP:=clusterprofile-provider-file}"
: "${MC_PROVIDER_TOKEN_SECRET:=clusterprofile-provider-token}"
: "${MC_PROVIDER_MOUNT_PATH:=/etc/cluster-inventory}"
: "${MC_PROVIDER_TOKEN_MOUNT_PATH:=/etc/cluster-inventory/access}"
: "${MC_PROVIDER_PLUGIN_MOUNT_PATH:=/etc/cluster-inventory/plugin}"
: "${MC_PROVIDER_NAME:=e2e-static-token}"
: "${SPOKE_GATEWAY_API_CRD_URL:=https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.1/experimental-install.yaml}"
export SPOKE_CLUSTER_NAME SPOKE_KUBECONFIG SPOKE_HOST_KUBECONFIG
export CLUSTER_INVENTORY_CRD_URL MC_PROVIDER_CONFIGMAP MC_PROVIDER_TOKEN_SECRET
export MC_PROVIDER_MOUNT_PATH MC_PROVIDER_TOKEN_MOUNT_PATH
export MC_PROVIDER_PLUGIN_MOUNT_PATH MC_PROVIDER_NAME
export CLUSTER_INVENTORY_CRD_URL MC_PROVIDER_CONFIGMAP
export MC_PROVIDER_MOUNT_PATH MC_PROVIDER_PLUGIN_MOUNT_PATH
export MC_PROVIDER_PLUGIN_IMAGE MC_PROVIDER_NAME
export SPOKE_GATEWAY_API_CRD_URL

function knative_setup() {
Expand Down
Loading