WIP: run macos functional tests in prow

hack/prow/boskos-cfg-macos.json

@@ -0,0 +1,5 @@
+{
+    "BoskosAcquireTimeoutSeconds": 180,
+    "BoskosHeartbeatIntervalSeconds": 10,
+    "BoskosLocation": "http://boskos.test-pods.svc.cluster.local"
+}

hack/prow/common.sh

@@ -65,52 +65,76 @@ function install_dependencies() {
 	if [ "$(uname)" != "Darwin" ]; then
 		sudo apt-get -y install lsof psmisc dnsutils
 	else
-		brew install pstree coreutils pidof
+		# install brew if not present
+		if ! command -v brew >/dev/null 2>&1; then
+			/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+			echo >> /Users/ec2-user/.zprofile
+			echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/ec2-user/.zprofile
+			eval "$(/opt/homebrew/bin/brew shellenv)"
+		fi
+		# install docker cli
+		brew install docker
+
+
+		# install vfkit
+		brew update
+		brew install vfkit pstree coreutils pidof
 		ln -s /usr/local/bin/gtimeout /usr/local/bin/timeout || true
+		# install vmnet shared on macos in non-interactive mode
+		curl -fsSL https://github.com/minikube-machine/vmnet-helper/releases/latest/download/install.sh | sudo VMNET_INTERACTIVE=0 bash
+
+		# ensure go dirs are owned by current user so that go install can write
+		sudo chown -R $(whoami) $HOME/go
+		sudo chown -R $(whoami) $HOME/Library/Caches/go-build
 	fi
 	# do NOT change manually - only using make update-golang-version
 	GOLANG_VERSION_TO_INSTALL=1.25.5
 	# install golang if not present
-	sudo -E hack/prow/installer/check_install_golang.sh /usr/local $GOLANG_VERSION_TO_INSTALL || true
+	hack/prow/installer/check_install_golang.sh /usr/local $GOLANG_VERSION_TO_INSTALL || true
 	# install gotestsum if not present
-	GOROOT="/usr/local/go" hack/prow/installer/check_install_gotestsum.sh || true
+	hack/prow/installer/check_install_gotestsum.sh || true
 	# install gopogh
 	hack/prow/installer/check_install_gopogh.sh || true
 
 	# install jq
 	if ! type "jq" >/dev/null; then
 		echo ">> Installing jq"
 		if [ "${ARCH}" == "arm64" && "${OS}" == "linux" ]; then
+		# linux arm64
 			sudo apt-get install jq -y
-		elif [ "${ARCH}" == "arm64" ]; then
-			echo "Unable to install 'jq' automatically for arm64 on Darwin, please install 'jq' manually."
-			exit 5
-		elif [ "${OS}" != "darwin" ]; then
+		elif [ "${ARCH}" == "arm64" && "${OS}" == "darwin"]; then
+		# macos arm64
+			brew install jq
+		elif [ "${OS}" == "linux" ]; then
+		# linux x86
 			curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && sudo install jq-linux64 /usr/local/bin/jq
 		else
+		# macos ax86
 			curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 && sudo install jq-osx-amd64 /usr/local/bin/jq
 		fi
 	fi
 
 }
 
 function docker_setup() {
-
-	# clean all docker artifacts up
-	docker system prune -a --volumes -f || true
-	docker system df || true
-	docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
-
-	# read only token, never expires
-	#todo: do we need this token
-	# docker login -u minikubebot -p "$DOCKERHUB_READONLY_TOKEN"
+	if [ "$(uname)" != "Darwin" ]; then
+		# clean all docker artifacts up
+		docker system prune -a --volumes -f || true
+		docker system df || true
+		docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
+
+		# read only token, never expires
+		#todo: do we need this token
+		# docker login -u minikubebot -p "$DOCKERHUB_READONLY_TOKEN"
+	fi
 }
 
 function gvisor_image_build() {
-	# Build the gvisor image so that we can integration test changes to pkg/gvisor
-	chmod +x testdata/gvisor-addon
+
 	# skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137
 	if [ "$(uname)" != "Darwin" ]; then
+		# Build the gvisor image so that we can integration test changes to pkg/gvisor
+		chmod +x testdata/gvisor-addon
 		# Should match GVISOR_IMAGE_VERSION in Makefile
 		docker build -t gcr.io/k8s-minikube/gvisor-addon:2 -f testdata/gvisor-addon-Dockerfile ./testdata
 	fi
@@ -125,8 +149,8 @@ function run_gopogh() {
 
 # this is where the script starts
 readonly OS_ARCH="${OS}-${ARCH}"
-readonly TEST_ROOT="${HOME}/minikube-integration"
-readonly TEST_HOME="${TEST_ROOT}/${MINIKUBE_LOCATION}-$$"
+readonly TEST_ROOT="${PWD}/minikube-integration"
+readonly TEST_HOME="$(pwd)/${MINIKUBE_LOCATION}-$$"
 
 export GOPATH="$HOME/go"
 export KUBECONFIG="${TEST_HOME}/kubeconfig"

hack/prow/installer/check_install_containerd.sh

@@ -16,7 +16,8 @@
 
 set -eux -o pipefail
 
-ARCH=${ARCH:=amd64}
+OS=$1
+ARCH=$2
 
 echo "Installing containerd"
 # Install docker's internal containerd if we can, or just install containerd from apt if available, 
@@ -48,6 +49,4 @@ sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/conf
 
 sudo systemctl restart containerd
 
-echo "Installing latest kubectl"
-curl -LO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
-sudo install ./kubectl /usr/local/bin/kubectl
+./hack/prow/installer/check_install_kubectl.sh ${OS} ${ARCH}

hack/prow/installer/check_install_docker.sh

@@ -16,7 +16,8 @@
 
 set -eux -o pipefail
 
-ARCH=${ARCH:=amd64}
+OS=$1
+ARCH=$2
 
 
 echo "Installing latest docker"
@@ -26,6 +27,4 @@ rm get-docker.sh
 
 sudo usermod -aG docker minitest || true
 
-echo "Installing latest kubectl"
-curl -LO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
-sudo install ./kubectl /usr/local/bin/kubectl
+./hack/prow/installer/check_install_kubectl.sh ${OS} ${ARCH}

hack/prow/installer/check_install_golang.sh

@@ -50,27 +50,9 @@ ARCH=${ARCH:=$(current_arch)}
 
 # installs or updates golang if right version doesn't exists
 function check_and_install_golang() {
-  if ! go version &>/dev/null; then
-    echo "WARNING: No golang installation found in your environment."
+	# always reinstall the requested version to avoid permission issues
     install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
-    return
-  fi
 
-  sudo chown -R $USER:$USER "$INSTALL_PATH"/go
-
-  # golang has been installed and check its version
-  if [[ $(go version | cut -d' ' -f 3) =~ go(([0-9]+)\.([0-9]+).([0-9]+)*) ]]; then
-    HOST_VERSION=${BASH_REMATCH[1]}
-    if [ $HOST_VERSION == $VERSION_TO_INSTALL ]; then
-      echo "go version on the host looks good : $HOST_VERSION"
-    else
-      echo "WARNING: expected go version to be $VERSION_TO_INSTALL but got $HOST_VERSION"
-      install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
-    fi
-  else
-    echo "ERROR: Failed to parse golang version: $(go version)"
-    return
-  fi
 }
 
 # install_golang takes two parameters version and path to install.

hack/prow/installer/check_install_gotestsum.sh

@@ -18,7 +18,7 @@ set -eux -o pipefail
 
 function install_gotestsum() {
   rm -f $(which gotestsum)
-   sudo env "GOBIN=$GOROOT/bin" "PATH=$PATH" go install gotest.tools/gotestsum@v1.13.0
+  go install gotest.tools/gotestsum@v1.13.0
 }
 echo "Installing gotestsum"
 which gotestsum || install_gotestsum

hack/prow/installer/check_install_kubectl.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux -o pipefail
+
+OS=$1
+ARCH=$2
+
+
+echo "Installing latest kubectl for ${ARCH}"
+curl -LO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/${OS}/${ARCH}/kubectl"
+sudo install ./kubectl /usr/local/bin/kubectl

hack/prow/integration_pre_install.sh

@@ -28,11 +28,11 @@ EXTRA_TEST_ARGS=""
 
 # install runtime if not present
 if [ "${CONTAINER_RUNTIME}" == "containerd" ]; then
-    ARCH="$ARCH" hack/prow/installer/check_install_containerd.sh || true
+    ARCH="$ARCH" hack/prow/installer/check_install_containerd.sh ${OS} ${ARCH} || true
 fi
 
 # instsll docker for all of them
-ARCH="$ARCH" hack/prow/installer/check_install_docker.sh || true
+ARCH="$ARCH" hack/prow/installer/check_install_docker.sh ${OS} ${ARCH} || true
 sudo adduser $(whoami) docker || true
 
 

hack/prow/integration_vfkit_docker_macos_arm64.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+OS="darwin"
+ARCH="arm64"
+DRIVER="vfkit"
+CONTAINER_RUNTIME="docker"
+# in prow, if you want libvirtd to be run, you have to start a privileged container as root
+EXTRA_START_ARGS="--network=vmnet-shared" 
+EXTRA_TEST_ARGS="-test.run TestFunctional -binary=out/minikube"
+JOB_NAME="KVM_Containerd_Linux_x86"
+#  marking all directories ('*') as trusted, since .git belongs to root, not minikube user
+git config --global --add safe.directory '*'
+
+# aws macos instance doesn't have kubectl pre-installed
+./hack/prow/installer/check_install_kubectl.sh ${OS} ${ARCH}
+
+# vmnet-helper breaks when the logfile name is too long(>125 chars), so we use short commit hash here
+COMMIT=$(git rev-parse HEAD | cut -c1-8)
+MINIKUBE_LOCATION=$COMMIT
+echo "running test in $(pwd)"
+
+set -e
+source ./hack/prow/common.sh 

hack/prow/minikube_cross_build.sh

@@ -40,15 +40,17 @@ export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
 export E2E_BIN="out/e2e-${OS_ARCH}"
 chmod +x "${MINIKUBE_BIN}" "${E2E_BIN}"
 
-BUILT_VERSION=$("out/minikube-$(go env GOOS)-$(go env GOARCH)" version)
-echo ${BUILT_VERSION}
+if [ "${OS}" != "darwin" ]; then
 
-COMMIT=$(echo ${BUILT_VERSION} | grep 'commit:' | awk '{print $2}')
-if (echo ${COMMIT} | grep -q dirty); then
-  echo "'minikube version' reports dirty commit: ${COMMIT}"
-  exit 1
-fi
+	BUILT_VERSION=$("out/minikube-${OS_ARCH}" version)
+	echo ${BUILT_VERSION}
 
+	COMMIT=$(echo ${BUILT_VERSION} | grep 'commit:' | awk '{print $2}')
+	if (echo ${COMMIT} | grep -q dirty); then
+		echo "'minikube version' reports dirty commit: ${COMMIT}"
+		exit 1
+	fi
+fi
 if [[ "${failed}" -ne 0 ]]; then
   echo "build failed"
   exit "${failed}"

hack/prow/minitest/deployer/boskos_config.go

@@ -17,10 +17,10 @@ limitations under the License.
 package deployer
 
 type MiniTestBoskosConfig struct {
-	GCPZone       string
-	InstanceImage string
-	InstanceType  string
-	DiskGiB       int
+	GCPZone       string `json:"GCPZone,omitempty"`
+	InstanceImage string `json:"InstanceImage,omitempty"`
+	InstanceType  string `json:"InstanceType,omitempty"`
+	DiskGiB       int    `json:"DiskGiB,omitempty"`
 	// Boskos flags correspond to https://github.com/kubernetes-sigs/kubetest2/blob/71238a9645df6fbd7eaac9a36f635c22f1566168/kubetest2-gce/deployer/deployer.go
 	BoskosAcquireTimeoutSeconds    int
 	BoskosHeartbeatIntervalSeconds int