feat(rcv1p): unify cert bootstrap flow and add Windows CA refresh task

.pipelines/e2e-rcv1p.yaml

@@ -0,0 +1,19 @@
+name: $(Date:yyyyMMdd)$(Rev:.r)
+variables:
+  TAGS_TO_RUN: "rcv1pcertmode=true"
+  SKIP_E2E_TESTS: false
+  E2E_GO_TEST_TIMEOUT: "75m"
+schedules:
+  - cron: "0 11 * * *"
+    displayName: Daily 3am PST
+    branches:
+      include:
+        - main
+    always: true
+trigger: none
+pr: none
+jobs:
+  - template: ./templates/e2e-template.yaml
+    parameters:
+      name: RCV1P Cert Mode Tests
+      IgnoreScenariosWithMissingVhd: false

.pipelines/scripts/e2e_run.sh

@@ -35,6 +35,7 @@ VHD_BUILD_ID="${VHD_BUILD_ID:-}"
 IGNORE_SCENARIOS_WITH_MISSING_VHD="${IGNORE_SCENARIOS_WITH_MISSING_VHD:-}"
 LOGGING_DIR="${LOGGING_DIR:-}"
 E2E_SUBSCRIPTION_ID="${E2E_SUBSCRIPTION_ID:-}"
+RCV1P_SUBSCRIPTION_ID="${RCV1P_SUBSCRIPTION_ID:-}"
 ENABLE_SECURE_TLS_BOOTSTRAPPING="${ENABLE_SECURE_TLS_BOOTSTRAPPING:-true}"
 TAGS_TO_SKIP="${TAGS_TO_SKIP:-}"
 TAGS_TO_RUN="${TAGS_TO_RUN:-}"
@@ -47,6 +48,7 @@ echo "VHD_BUILD_ID: ${VHD_BUILD_ID}"
 echo "IGNORE_SCENARIOS_WITH_MISSING_VHD: ${IGNORE_SCENARIOS_WITH_MISSING_VHD}"
 echo "LOGGING_DIR: ${LOGGING_DIR}"
 echo "E2E_SUBSCRIPTION_ID: ${E2E_SUBSCRIPTION_ID}"
+echo "RCV1P_SUBSCRIPTION_ID: ${RCV1P_SUBSCRIPTION_ID}"
 echo "ENABLE_SECURE_TLS_BOOTSTRAPPING: ${ENABLE_SECURE_TLS_BOOTSTRAPPING}"
 echo "TAGS_TO_SKIP: ${TAGS_TO_SKIP}"
 echo "TAGS_TO_RUN: ${TAGS_TO_RUN}"
@@ -95,10 +97,11 @@ tar -xzf "$temp_file" -C bin
 chmod +x bin/gotestsum
 rm -f "$temp_file"
 
+# REVERT ME: added -v to see t.Logf output from passing tests (azcopy/wireserver diagnostics)
 # gotestsum configure to only show logs for failed tests, json file for detailed logs
 # Run the tests! Yey!
 test_exit_code=0
-./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 60 -timeout "${E2E_GO_TEST_TIMEOUT}" || test_exit_code=$?
+./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -v -parallel 60 -timeout "${E2E_GO_TEST_TIMEOUT}" || test_exit_code=$?
 
 # Upload test results as Azure DevOps artifacts
 echo "##vso[artifact.upload containerfolder=test-results;artifactname=e2e-test-log]${BUILD_SRC_DIR}/e2e/test-log.json"

.pipelines/templates/e2e-template.yaml

@@ -38,6 +38,7 @@ jobs:
         displayName: Run AgentBaker E2E
         env:
           E2E_SUBSCRIPTION_ID: $(E2E_SUBSCRIPTION_ID)
+          RCV1P_SUBSCRIPTION_ID: $(RCV1P_SUBSCRIPTION_ID)
           SYS_SSH_PUBLIC_KEY: $(SYS_SSH_PUBLIC_KEY)
           SYS_SSH_PRIVATE_KEY_B64: $(SYS_SSH_PRIVATE_KEY_B64)
           BUILD_SRC_DIR: $(System.DefaultWorkingDirectory)

aks-node-controller/parser/helper.go

@@ -64,6 +64,7 @@ func getFuncMap() template.FuncMap {
 	return template.FuncMap{
 		"getInitAKSCustomCloudFilepath": getInitAKSCustomCloudFilepath,
 		"getIsAksCustomCloud":           getIsAksCustomCloud,
+		"getCloudLocation":              getCloudLocation,
 	}
 }
 
@@ -538,11 +539,15 @@ func getIsAksCustomCloud(customCloudConfig *aksnodeconfigv1.CustomCloudConfig) b
 	return strings.EqualFold(customCloudConfig.GetCustomCloudEnvName(), helpers.AksCustomCloudName)
 }
 
+func getCloudLocation(v *aksnodeconfigv1.Configuration) string {
+	return strings.ToLower(strings.Join(strings.Fields(v.GetClusterConfig().GetLocation()), ""))
+}
+
 /* GetCloudTargetEnv determines and returns whether the region is a sovereign cloud which
 have their own data compliance regulations (China/Germany/USGov) or standard.  */
 // Azure public cloud.
 func getCloudTargetEnv(v *aksnodeconfigv1.Configuration) string {
-	loc := strings.ToLower(strings.Join(strings.Fields(v.GetClusterConfig().GetLocation()), ""))
+	loc := getCloudLocation(v)
 	switch {
 	case strings.HasPrefix(loc, "china"):
 		return "AzureChinaCloud"

aks-node-controller/parser/templates/cse_cmd.sh.gtpl

@@ -1,6 +1,7 @@
 echo $(date),$(hostname) > ${PROVISION_OUTPUT};
 {{if getIsAksCustomCloud .CustomCloudConfig}}
 REPO_DEPOT_ENDPOINT="{{.CustomCloudConfig.RepoDepotEndpoint}}"
-{{getInitAKSCustomCloudFilepath}} >> /var/log/azure/cluster-provision.log 2>&1;
 {{end}}
+LOCATION="{{getCloudLocation .}}"
+{{getInitAKSCustomCloudFilepath}} >> /var/log/azure/cluster-provision.log 2>&1;
 /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh"

e2e/aks_model.go

@@ -299,22 +299,23 @@ func getFirewall(ctx context.Context, location, firewallSubnetID, publicIPID str
 }
 
 func addFirewallRules(
-	ctx context.Context, clusterModel *armcontainerservice.ManagedCluster,
+	ctx context.Context, infra *ClusterInfra, clusterModel *armcontainerservice.ManagedCluster,
 ) error {
 	location := *clusterModel.Location
 	defer toolkit.LogStepCtx(ctx, "adding firewall rules")()
 
 	rg := *clusterModel.Properties.NodeResourceGroup
-	vnet, err := getClusterVNet(ctx, rg)
+	vnet, err := getClusterVNet(ctx, infra, rg)
 	if err != nil {
 		return err
 	}
 
-	// For kubenet, the AKS-managed route table must stay attached so that pod
-	// routes (managed by cloud-provider-azure) and firewall routes coexist.
-	// For Azure CNI variants, the subnet may not have any route table, so we
-	// create and associate a dedicated one before adding the firewall routes.
-	aksSubnetResp, err := config.Azure.Subnet.Get(ctx, rg, vnet.name, "aks-subnet", nil)
+	// Find the AKS-managed route table currently associated with the subnet.
+	// We add firewall routes directly to this table so that both pod routes
+	// (managed by cloud-provider-azure) and firewall routes coexist. Creating
+	// a separate route table and swapping the subnet association disconnects
+	// the pod routes and breaks kubenet networking.
+	aksSubnetResp, err := infra.Azure.Subnet.Get(ctx, rg, vnet.name, "aks-subnet", nil)
 	if err != nil {
 		return fmt.Errorf("failed to get AKS subnet: %w", err)
 	}
@@ -332,7 +333,7 @@ func addFirewallRules(
 	}
 
 	toolkit.Logf(ctx, "Creating subnet %s in VNet %s", firewallSubnetName, vnet.name)
-	subnetPoller, err := config.Azure.Subnet.BeginCreateOrUpdate(
+	subnetPoller, err := infra.Azure.Subnet.BeginCreateOrUpdate(
 		ctx,
 		rg,
 		vnet.name,
@@ -365,7 +366,7 @@ func addFirewallRules(
 	}
 
 	toolkit.Logf(ctx, "Creating public IP %s", publicIPName)
-	pipPoller, err := config.Azure.PublicIPAddresses.BeginCreateOrUpdate(
+	pipPoller, err := infra.Azure.PublicIPAddresses.BeginCreateOrUpdate(
 		ctx,
 		rg,
 		publicIPName,
@@ -386,7 +387,7 @@ func addFirewallRules(
 
 	firewallName := "abe2e-fw"
 	firewall := getFirewall(ctx, location, firewallSubnetID, publicIPID)
-	fwPoller, err := config.Azure.AzureFirewall.BeginCreateOrUpdate(ctx, rg, firewallName, *firewall, nil)
+	fwPoller, err := infra.Azure.AzureFirewall.BeginCreateOrUpdate(ctx, rg, firewallName, *firewall, nil)
 	if err != nil {
 		return fmt.Errorf("failed to start Firewall creation: %w", err)
 	}
@@ -432,7 +433,7 @@ func addFirewallRules(
 
 	for _, route := range firewallRoutes {
 		toolkit.Logf(ctx, "Adding route %q to AKS route table %q", *route.Name, aksRTName)
-		poller, err := config.Azure.Routes.BeginCreateOrUpdate(ctx, rg, aksRTName, *route.Name, route, nil)
+		poller, err := infra.Azure.Routes.BeginCreateOrUpdate(ctx, rg, aksRTName, *route.Name, route, nil)
 		if err != nil {
 			return fmt.Errorf("failed to start adding route %q: %w", *route.Name, err)
 		}
@@ -510,7 +511,7 @@ func addPrivateAzureContainerRegistry(ctx context.Context, cluster *armcontainer
 	if err := createPrivateAzureContainerRegistryPullSecret(ctx, cluster, kube, resourceGroupName, isNonAnonymousPull); err != nil {
 		return fmt.Errorf("create private acr pull secret: %w", err)
 	}
-	vnet, err := getClusterVNet(ctx, *cluster.Properties.NodeResourceGroup)
+	vnet, err := getClusterVNet(ctx, DefaultClusterInfra, *cluster.Properties.NodeResourceGroup)
 	if err != nil {
 		return err
 	}
@@ -531,7 +532,7 @@ func addNetworkIsolatedSettings(ctx context.Context, clusterModel *armcontainers
 	location := *clusterModel.Location
 	defer toolkit.LogStepCtx(ctx, fmt.Sprintf("Adding network settings for network isolated cluster %s in rg %s", *clusterModel.Name, *clusterModel.Properties.NodeResourceGroup))
 
-	vnet, err := getClusterVNet(ctx, *clusterModel.Properties.NodeResourceGroup)
+	vnet, err := getClusterVNet(ctx, DefaultClusterInfra, *clusterModel.Properties.NodeResourceGroup)
 	if err != nil {
 		return err
 	}
@@ -678,7 +679,7 @@ func createPrivateAzureContainerRegistry(ctx context.Context, cluster *armcontai
 		}
 		// if ACR gets recreated so should the cluster
 		toolkit.Logf(ctx, "Private ACR deleted, deleting cluster %s", *cluster.Name)
-		if err := deleteCluster(ctx, *cluster.Name, resourceGroup); err != nil {
+		if err := deleteCluster(ctx, DefaultClusterInfra, *cluster.Name, resourceGroup); err != nil {
 			return fmt.Errorf("failed to delete cluster: %w", err)
 		}
 	} else {

e2e/cache.go

@@ -10,6 +10,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3"
 )
 
 // cachedFunc creates a thread-safe memoized version of a function.
@@ -150,56 +151,67 @@ func clusterLatestKubernetesVersion(ctx context.Context, request ClusterRequest)
 	if err != nil {
 		return nil, fmt.Errorf("getting latest kubernetes version cluster model: %w", err)
 	}
-	return prepareCluster(ctx, model, false, false)
+	return prepareCluster(ctx, DefaultClusterInfra, model, false, false)
 }
 
 var ClusterKubenet = cachedFunc(clusterKubenet)
 
 // clusterKubenet creates a basic cluster using kubenet networking
 func clusterKubenet(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getKubenetClusterModel("abe2e-kubenet-v4", request.Location, request.K8sSystemPoolSKU), false, false)
+	return prepareCluster(ctx, DefaultClusterInfra, getKubenetClusterModel("abe2e-kubenet-v4", request.Location, request.K8sSystemPoolSKU), false, false)
 }
 
 var ClusterAzureNetwork = cachedFunc(clusterAzureNetwork)
 
 // clusterAzureNetwork creates a cluster with Azure CNI networking
 func clusterAzureNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
+	return prepareCluster(ctx, DefaultClusterInfra, getAzureNetworkClusterModel("abe2e-azure-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
 }
 
 var ClusterAzureBootstrapProfileCache = cachedFunc(clusterAzureBootstrapProfileCache)
 
 // clusterAzureBootstrapProfileCache creates a cluster with bootstrap profile cache but without network isolation
 func clusterAzureBootstrapProfileCache(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-bootstrapprofile-cache-v1", request.Location, request.K8sSystemPoolSKU), false, true)
+	return prepareCluster(ctx, DefaultClusterInfra, getAzureNetworkClusterModel("abe2e-azure-bootstrapprofile-cache-v1", request.Location, request.K8sSystemPoolSKU), false, true)
 }
 
 var ClusterAzureNetworkIsolated = cachedFunc(clusterAzureNetworkIsolated)
 
 // clusterAzureNetworkIsolated creates a networkisolated Azure network cluster (no internet access)
 func clusterAzureNetworkIsolated(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getAzureNetworkClusterModel("abe2e-azure-networkisolated-v1", request.Location, request.K8sSystemPoolSKU), true, false)
+	return prepareCluster(ctx, DefaultClusterInfra, getAzureNetworkClusterModel("abe2e-azure-networkisolated-v1", request.Location, request.K8sSystemPoolSKU), true, false)
 }
 
 var ClusterAzureOverlayNetwork = cachedFunc(clusterAzureOverlayNetwork)
 
 // clusterAzureOverlayNetwork creates a cluster with Azure CNI Overlay networking
 func clusterAzureOverlayNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getAzureOverlayNetworkClusterModel("abe2e-azure-overlay-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
+	return prepareCluster(ctx, DefaultClusterInfra, getAzureOverlayNetworkClusterModel("abe2e-azure-overlay-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
 }
 
 var ClusterAzureOverlayNetworkDualStack = cachedFunc(clusterAzureOverlayNetworkDualStack)
 
 // clusterAzureOverlayNetworkDualStack creates a dual-stack (IPv4+IPv6) Azure CNI Overlay cluster
 func clusterAzureOverlayNetworkDualStack(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v3", request.Location, request.K8sSystemPoolSKU), false, false)
+	return prepareCluster(ctx, DefaultClusterInfra, getAzureOverlayNetworkDualStackClusterModel("abe2e-azure-overlay-dualstack-v3", request.Location, request.K8sSystemPoolSKU), false, false)
 }
 
 var ClusterCiliumNetwork = cachedFunc(clusterCiliumNetwork)
 
 // clusterCiliumNetwork creates a cluster with Cilium CNI networking
 func clusterCiliumNetwork(ctx context.Context, request ClusterRequest) (*Cluster, error) {
-	return prepareCluster(ctx, getCiliumNetworkClusterModel("abe2e-cilium-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
+	return prepareCluster(ctx, DefaultClusterInfra, getCiliumNetworkClusterModel("abe2e-cilium-network-v3", request.Location, request.K8sSystemPoolSKU), false, false)
+}
+
+var ClusterRCV1PKubenet = cachedFunc(clusterRCV1PKubenet)
+
+// clusterRCV1PKubenet creates a kubenet cluster in the RCV1P subscription for cert mode testing.
+func clusterRCV1PKubenet(ctx context.Context, request ClusterRequest) (*Cluster, error) {
+	infra := RCV1PClusterInfra()
+	if infra == nil {
+		return nil, fmt.Errorf("RCV1P_SUBSCRIPTION_ID not set, cannot create RCV1P cluster")
+	}
+	return prepareCluster(ctx, infra, getKubenetClusterModel("abe2e-rcv1p-kubenet-v1", request.Location, request.K8sSystemPoolSKU), false, false)
 }
 
 // isNotFoundErr checks if an error represents a "not found" response from Azure API
@@ -228,6 +240,25 @@ var CachedEnsureResourceGroup = cachedFunc(ensureResourceGroup)
 var CachedCreateVMManagedIdentity = cachedFunc(config.Azure.CreateVMManagedIdentity)
 var CachedCompileAndUploadAKSNodeController = cachedFunc(compileAndUploadAKSNodeController)
 
+// CachedRCV1PEnsureResourceGroup creates the resource group in the RCV1P subscription.
+var CachedRCV1PEnsureResourceGroup = cachedFunc(ensureRCV1PResourceGroup)
+
+// CachedRCV1PCreateVMManagedIdentity creates a VM managed identity in the RCV1P subscription.
+var CachedRCV1PCreateVMManagedIdentity = cachedFunc(func(ctx context.Context, location string) (string, error) {
+	if config.RCV1PAzure == nil {
+		return "", fmt.Errorf("RCV1P_SUBSCRIPTION_ID not set")
+	}
+	return config.RCV1PAzure.CreateVMManagedIdentityInRG(ctx, config.RCV1PResourceGroupName(location), location)
+})
+
+func ensureRCV1PResourceGroup(ctx context.Context, location string) (armresources.ResourceGroup, error) {
+	infra := RCV1PClusterInfra()
+	if infra == nil {
+		return armresources.ResourceGroup{}, fmt.Errorf("RCV1P_SUBSCRIPTION_ID not set")
+	}
+	return ensureResourceGroupWithInfra(ctx, infra, location)
+}
+
 // VMSizeSKURequest is the cache key for Resource SKU lookups by VM size and location.
 type VMSizeSKURequest struct {
 	Location string