MCO-2208: MCO-2125: Migrate mco registry units

test/extended-priv/const.go

@@ -134,6 +134,13 @@ const (
 	NodeDisruptionPolicyActionRestart      = "Restart"
 	NodeDisruptionPolicyActionDrain        = "Drain"
 	NodeDisruptionPolicyActionDaemonReload = "DaemonReload"
+	NodeDisruptionPolicyFiles              = "files"
+	NodeDisruptionPolicyUnits              = "units"
+	NodeDisruptionPolicySshkey             = "sshkey"
+
+	// Regexp used to know if MCD logs is reporting that crio was reloaded
+	MCDCrioReloadedRegexp = "crio.* reloaded successfully"
+
 	// BootImageSkewEnforcement feature constants
 
 	// RHCOSVersionMode is used to signify that the cluster boot image is described via RHCOS version

test/extended-priv/machineconfig.go

@@ -138,6 +138,11 @@ func (mc *MachineConfig) GetAuthorizedKeysByUserAsList(user string) ([]string, e
 	return listKeys, err
 }
 
+// GetExtensions returns all the extensions configured in this MC
+func (mc *MachineConfig) GetExtensions() (string, error) {
+	return mc.Get(`{.spec.extensions}`)
+}
+
 // GetIgnitionVersion returns the ignition version used in the MC
 func (mc *MachineConfig) GetIgnitionVersion() (string, error) {
 	return mc.Get(`{.spec.config.ignition.version}`)

test/extended-priv/mco.go

@@ -83,6 +83,17 @@ func skipTestIfRHELVersion(node *Node, operator, constraintVersion string) {
 	}
 }
 
+// skipTestIfClusterVersion skips the test case if the provided version matches the constraints.
+func skipTestIfClusterVersion(oc *exutil.CLI, operator, constraintVersion string) {
+	clusterVersion, _, err := exutil.GetClusterVersion(oc)
+	o.ExpectWithOffset(1, err).NotTo(o.HaveOccurred())
+
+	if CompareVersions(clusterVersion, operator, constraintVersion) {
+		g.Skip(fmt.Sprintf("Test case skipped because current cluster version %s %s %s",
+			clusterVersion, operator, constraintVersion))
+	}
+}
+
 func verifyRenderedMcs(oc *exutil.CLI, renderSuffix string, allRes []ResourceInterface) []*Resource {
 	// TODO: Use MachineConfigList when MC code is refactored
 	allMcs, err := NewResourceList(oc.AsAdmin(), "mc").GetAll()

test/extended-priv/mco_units.go

@@ -8,6 +8,7 @@ import (
 	g "github.com/onsi/ginkgo/v2"
 	o "github.com/onsi/gomega"
 	exutil "github.com/openshift/machine-config-operator/test/extended-priv/util"
+	"github.com/openshift/machine-config-operator/test/extended-priv/util/bootstrap"
 	logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
 )
 
@@ -149,6 +150,34 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati
 		o.Expect(svcUnMaskedOuput).ShouldNot(o.ContainSubstring(inactiveString))
 	})
 
+	g.It("[PolarionID:53960][OTP] No failed units in the bootstrap machine", g.Label("Platform:aws", "Platform:azure"), func() {
+		failedUnitsCommand := "sudo systemctl list-units --failed --all"
+
+		// If no bootstrap is found, we skip the case.
+		// The test can only be executed in deployments that didn't remove the bootstrap machine
+		bs, err := bootstrap.GetBootstrap(oc)
+		if err != nil {
+			if _, notFound := err.(*bootstrap.InstanceNotFound); notFound {
+				g.Skip("skip test because bootstrap machine does not exist in the current cluster")
+			}
+		}
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		exutil.By("Verify that there is no failed units in the bootstrap machine")
+		// ssh client is a bit unstable, and it can return an empty string for no apparent reason every now and then.
+		// Hence we use 'Eventually' to verify the command to make the test robust.
+		o.Eventually(func() string {
+			logger.Infof("Executing command in bootstrap: %s", failedUnitsCommand)
+			failedUnits, err := bs.SSH.RunOutput(failedUnitsCommand)
+			logger.Infof("Command output:\n%s", failedUnits)
+			if err != nil {
+				logger.Errorf("Command Error:\n%s", err)
+			}
+			return failedUnits
+		}, "3m", "15s").Should(o.ContainSubstring("0 loaded units listed"),
+			"There are failed units in the bootstrap machine")
+	})
+
 	g.It("[PolarionID:56614][OTP] Create unit with content and mask=true[Disruptive]", func() {
 		var (
 			workerNode     = NewNodeList(oc.AsAdmin()).GetAllLinuxWorkerNodesOrFail()[0]

test/extended-priv/testdata/files/add-image-tag-mirror-set.yaml

@@ -0,0 +1,17 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: image-tag-mirror-set
+objects:
+  - kind: ImageTagMirrorSet
+    apiVersion: config.openshift.io/v1
+    metadata:
+      name: "${NAME}"
+    spec:
+      imageTagMirrors:
+      - mirrors:
+        - mirror.example.com/redhat
+        source: registry.redhat.io/openshift4
+        mirrorSourcePolicy: AllowContactingSource
+parameters:
+  - name: NAME

test/extended-priv/testdata/files/repository-mirror.yaml

@@ -0,0 +1,15 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: repository-mirror
+objects:
+  - kind: ImageContentSourcePolicy
+    apiVersion: operator.openshift.io/v1alpha1
+    metadata:
+      name: "${NAME}"
+    spec:
+      repositoryDigestMirrors: ${{REPOSITORYDIGESTMIRRORS}}
+parameters:
+  - name: NAME
+  - name: REPOSITORYDIGESTMIRRORS
+    value: '[{"mirrors":["example.io/example/ubi-minimal", "example.com/example/ubi-minimal"], "source": "registry.access.redhat.com/ubi8/ubi-minimal"}]'

test/extended-priv/util.go

@@ -70,6 +70,11 @@ type ImageTagMirrorSet struct {
 	Template
 }
 
+// NewImageTagMirrorSet create a new ImageTagMirrorSet struct
+func NewImageTagMirrorSet(oc *exutil.CLI, name string, t Template) *ImageTagMirrorSet {
+	return &ImageTagMirrorSet{Resource: *NewResource(oc, "ImageTagMirrorSet", name), Template: t}
+}
+
 // TextToVerify is a helper struct to verify configurations using the `createMcAndVerifyMCValue` function
 type TextToVerify struct {
 	textToVerifyForMC   string
@@ -1388,3 +1393,56 @@ func ToInterfaceSlice(r gjson.Result) []interface{} {
 	}
 	return result
 }
+
+// skipTestIfExtensionsAreUsed skips the current test if any extension has been deployed in the nodes
+func skipTestIfExtensionsAreUsed(oc *exutil.CLI) {
+	wMcp := NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolWorker)
+	mMcp := NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolMaster)
+
+	wCurrentMC, err := wMcp.GetConfiguredMachineConfig()
+	o.ExpectWithOffset(1, err).NotTo(o.HaveOccurred(), "Error trying to get the current MC configured in worker pool")
+
+	mCurrentMC, err := mMcp.GetConfiguredMachineConfig()
+	o.ExpectWithOffset(1, err).NotTo(o.HaveOccurred(), "Error trying to get the current MC configured in master pool")
+
+	wExtensions, err := wCurrentMC.GetExtensions()
+	o.ExpectWithOffset(1, err).NotTo(o.HaveOccurred(), "Error trying to get the extensions configured in MC: %s", wCurrentMC.GetName())
+
+	mExtensions, err := mCurrentMC.GetExtensions()
+	o.ExpectWithOffset(1, err).NotTo(o.HaveOccurred(), "Error trying to get the extensions configured in MC: %s", mCurrentMC.GetName())
+
+	if wExtensions != "[]" || mExtensions != "[]" {
+		g.Skip("Current cluster is using extensions. This test cannot be execute in a cluster using extensions")
+	}
+
+}
+
+// GetImageRegistryCertificates returns a map with the image registry certificates content. Key=certificate file name, Value=certificate content
+func GetImageRegistryCertificates(oc *exutil.CLI) (map[string]string, error) {
+	return GetDataFromConfigMap(oc.AsAdmin(), "openshift-config-managed", "image-registry-ca")
+}
+
+// GetManagedMergedTrustedImageRegistryCertificates returns a map with the merged trusted image registry certificates content. Key=certificate file name, Value=certificate content
+func GetManagedMergedTrustedImageRegistryCertificates(oc *exutil.CLI) (map[string]string, error) {
+	return GetDataFromConfigMap(oc.AsAdmin(), "openshift-config-managed", "merged-trusted-image-registry-ca")
+}
+
+// GetDataFromConfigMap returns a map[string]string with the information of the ".data" section of a configmap
+func GetDataFromConfigMap(oc *exutil.CLI, namespace, name string) (map[string]string, error) {
+	data := map[string]string{}
+	cm := NewNamespacedResource(oc.AsAdmin(), "ConfigMap", namespace, name)
+	dataJSON, err := cm.Get(`{.data}`)
+	if err != nil {
+		return nil, err
+	}
+
+	if dataJSON == "" {
+		return data, nil
+	}
+
+	if err := json.Unmarshal([]byte(dataJSON), &data); err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}

test/extended-priv/util/bootstrap/aws.go

@@ -0,0 +1,85 @@
+package bootstrap
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	util "github.com/openshift/machine-config-operator/test/extended-priv/util"
+	logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
+)
+
+const (
+	// EnvVarSSHCloudPrivAWSUser stores the environment variable for the AWS ssh user
+	EnvVarSSHCloudPrivAWSUser = "SSH_CLOUD_PRIV_AWS_USER"
+)
+
+// AWSBSInfoProvider implements interface BSInfoProvider for AWS
+type AWSBSInfoProvider struct{}
+
+// GetIPs returns the IPs of the bootstrap machine if this machine exists in AWS
+func (a AWSBSInfoProvider) GetIPs(oc *util.CLI) (*Ips, error) {
+	infraName, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.infrastructureName}").Output()
+	if err != nil {
+		logger.Errorf("Could not get bootstrap's IP in AWS. Unable to get infrastructure's name. Error: %s", err)
+		return nil, err
+	}
+
+	bootstrapName := infraName + "-bootstrap"
+
+	publicIP, err := awsCLIGetInstancePublicIP(bootstrapName)
+	if err != nil {
+		return nil, &InstanceNotFound{bootstrapName}
+	}
+	if publicIP == "" || publicIP == "None" {
+		logger.Infof("Bootstrap instance '%s' has no public IP or is terminated", bootstrapName)
+		return nil, &InstanceNotFound{bootstrapName}
+	}
+
+	privateIP, _ := awsCLIGetInstancePrivateIP(bootstrapName)
+	logger.Infof("Bootstrap instance '%s': publicIP=%s, privateIP=%s", bootstrapName, publicIP, privateIP)
+	return &Ips{PublicIPAddress: publicIP, PrivateIPAddress: privateIP}, nil
+}
+
+// GetSSHUser returns the user needed to connect to the bootstrap machine via ssh
+func (a AWSBSInfoProvider) GetSSHUser() string {
+	if user, exists := os.LookupEnv(EnvVarSSHCloudPrivAWSUser); exists {
+		return user
+	}
+	return DefaultSSHUser
+}
+
+func awsCLIGetInstancePublicIP(instanceName string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, "aws", "ec2", "describe-instances",
+		"--filters", fmt.Sprintf("Name=tag:Name,Values=%s", instanceName),
+		"--query", "Reservations[0].Instances[0].PublicIpAddress",
+		"--output", "text",
+	).Output()
+	if err != nil {
+		return "", err
+	}
+	result := strings.TrimSpace(string(out))
+	if result == "null" {
+		return "", fmt.Errorf("no instance found with name %s", instanceName)
+	}
+	return result, nil
+}
+
+func awsCLIGetInstancePrivateIP(instanceName string) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, "aws", "ec2", "describe-instances",
+		"--filters", fmt.Sprintf("Name=tag:Name,Values=%s", instanceName),
+		"--query", "Reservations[0].Instances[0].PrivateIpAddress",
+		"--output", "text",
+	).Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}

test/extended-priv/util/bootstrap/azure.go

@@ -0,0 +1,31 @@
+package bootstrap
+
+import (
+	util "github.com/openshift/machine-config-operator/test/extended-priv/util"
+	logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
+)
+
+const (
+	// EnvVarSSHCloudPrivAzureUser stores the environment variable for the Azure ssh user
+	EnvVarSSHCloudPrivAzureUser = "SSH_CLOUD_PRIV_AZURE_USER"
+)
+
+// AzureBSInfoProvider implements interface BSInfoProvider for Azure
+type AzureBSInfoProvider struct{}
+
+// GetIPs returns the IPs of the bootstrap machine if this machine exists in Azure.
+// Azure bootstrap discovery is not yet implemented; returns InstanceNotFound so the test skips.
+func (a AzureBSInfoProvider) GetIPs(oc *util.CLI) (*Ips, error) {
+	infraName, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.infrastructureName}").Output()
+	if err != nil {
+		return nil, err
+	}
+	bootstrapName := infraName + "-bootstrap"
+	logger.Infof("Azure bootstrap discovery is not yet implemented, skipping bootstrap machine: %s", bootstrapName)
+	return nil, &InstanceNotFound{bootstrapName}
+}
+
+// GetSSHUser returns the user needed to connect to the bootstrap machine via ssh
+func (a AzureBSInfoProvider) GetSSHUser() string {
+	return DefaultSSHUser
+}

test/extended-priv/util/bootstrap/bootstrap.go

@@ -0,0 +1,81 @@
+package bootstrap
+
+import (
+	"fmt"
+
+	util "github.com/openshift/machine-config-operator/test/extended-priv/util"
+	logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
+)
+
+const (
+	// EnvVarSSHCloudPrivKey stores the environment variable for the ssh private key path
+	EnvVarSSHCloudPrivKey = "SSH_CLOUD_PRIV_KEY"
+	// DefaultSSHUser is the default ssh user for bootstrap machines
+	DefaultSSHUser = "core"
+)
+
+// InstanceNotFound reports an error because the bootstrap instance is not found. It can be used to skip the test case.
+type InstanceNotFound struct{ InstanceName string }
+
+// Error implements the error interface
+func (inferr *InstanceNotFound) Error() string {
+	return fmt.Sprintf("Instance %s has 'terminated' status", inferr.InstanceName)
+}
+
+// BSInfoProvider any struct implementing this interface can be used to create a Bootstrap object.
+type BSInfoProvider interface {
+	GetIPs(*util.CLI) (*Ips, error)
+	GetSSHUser() string
+}
+
+// Bootstrap contains the functionality regarding the bootstrap machine
+type Bootstrap struct {
+	SSH util.SshClient
+	IPs Ips
+}
+
+// Ips struct to store the public and the private IPs of the bootstrap machine
+type Ips struct {
+	PrivateIPAddress string
+	PublicIPAddress  string
+}
+
+// GetBootstrap returns a bootstrap struct pointing to the bootstrap machine if it exists
+func GetBootstrap(oc *util.CLI) (*Bootstrap, error) {
+	bsInfoProvider, err := GetBSInfoProvider(oc)
+	if err != nil {
+		return nil, err
+	}
+
+	bootstrapIPs, err := bsInfoProvider.GetIPs(oc.AsAdmin())
+	if err != nil {
+		return nil, err
+	}
+
+	user := bsInfoProvider.GetSSHUser()
+	return buildBootstrap(user, *bootstrapIPs, 22), nil
+}
+
+// GetBSInfoProvider returns a struct implementing BSInfoProvider for the current platform
+func GetBSInfoProvider(oc *util.CLI) (BSInfoProvider, error) {
+	platform := util.CheckPlatform(oc)
+	switch platform {
+	case "aws":
+		return AWSBSInfoProvider{}, nil
+	case "azure":
+		return AzureBSInfoProvider{}, nil
+	default:
+		return nil, fmt.Errorf("platform not supported. Cannot get bootstrap information for platform: %s", platform)
+	}
+}
+
+func buildBootstrap(user string, bootstrapIPs Ips, port int) *Bootstrap {
+	privateKey := util.GetSSHPrivateKey()
+	publicIP := bootstrapIPs.PublicIPAddress
+	logger.Infof("Creating bootstrap with ip '%s', user: '%s', private key: '%s', port '%d'",
+		publicIP, user, privateKey, port)
+	return &Bootstrap{
+		SSH: util.SshClient{User: user, Host: publicIP, Port: port, PrivateKey: privateKey},
+		IPs: bootstrapIPs,
+	}
+}