Makefile

.DEFAULT_GOAL := up
.EXPORT_ALL_VARIABLES:

-include .env

# Commands
YQ=docker run --rm -i -v $(shell pwd):/workdir mikefarah/yq:4

KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml)
KUBECONFIG := $(shell pwd)/.kubeconfig

METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin)
METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal)

MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf
# Default values
CONTAINERLAB=$(shell which containerlab)

# extra vars can be used by projects that built on the mini-lab, which want to override default configuration
ANSIBLE_EXTRA_VARS_FILE := $(or $(ANSIBLE_EXTRA_VARS_FILE),)
# do not show skipped ansible tasks
ANSIBLE_DISPLAY_SKIPPED_HOSTS=false

MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),sonic)
MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)

MACHINE_OS=debian-12.0
MAX_RETRIES := 30

# Machine flavors
ifeq ($(MINI_LAB_FLAVOR),sonic)
LAB_TOPOLOGY=mini-lab.sonic.yaml
MONITORING_ENABLED := $(or $(MONITORING_ENABLED),true)
else ifeq ($(MINI_LAB_FLAVOR),dell_sonic)
LAB_TOPOLOGY=mini-lab.dell_sonic.yaml
MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
else ifeq ($(MINI_LAB_FLAVOR),capms)
LAB_TOPOLOGY=mini-lab.capms.yaml
MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
else ifeq ($(MINI_LAB_FLAVOR),kamaji)
LAB_TOPOLOGY=mini-lab.kamaji.yaml
KAMAJI_ENABLED=true
else ifeq ($(MINI_LAB_FLAVOR),gardener)
GARDENER_ENABLED=true
# usually gardener restricts the maximum version for k8s:
K8S_VERSION=1.32.5
LAB_TOPOLOGY=mini-lab.sonic.yaml
else
$(error Unknown flavor $(MINI_LAB_FLAVOR))
endif

KIND_ARGS=
ifneq ($(K8S_VERSION),)
KIND_ARGS=--image kindest/node:v$(K8S_VERSION)
endif

ifeq ($(CI),true)
  DOCKER_COMPOSE_RUN_ARG=--no-TTY --rm
else
  DOCKER_COMPOSE_RUN_ARG=--rm
endif

.PHONY: up
up: env gen-certs control-plane-bake partition-bake
	@chmod 600 files/ssh/id_ed25519
	docker compose up --pull=always --abort-on-container-failure --remove-orphans --force-recreate control-plane partition
	@$(MAKE)	--no-print-directory	start-machines
# for some reason an allocated machine will not be able to phone home
# without restarting the metal-core
# TODO: should be investigated and fixed if possible
# check that underlay gets working
	sleep 10
	ssh -F files/ssh/config leaf01 'systemctl restart metal-core'
	ssh -F files/ssh/config leaf02 'systemctl restart metal-core'

.PHONY: restart
restart: down up

.PHONY: down
down: cleanup

.PHONY: gen-certs
gen-certs:
	@if ! [ -f "files/certs/ca.pem" ]; then \
		echo "certificate generation required, running cfssl container"; \
		docker run --rm \
			--user $$(id -u):$$(id -g) \
			--entrypoint bash \
			-v ${PWD}:/work \
			cfssl/cfssl /work/scripts/roll_certs.sh; fi

.PHONY: roll-certs
roll-certs:
	rm files/certs/ca.pem
	$(MAKE) gen-certs

.PHONY: control-plane
control-plane: control-plane-bake env
	docker compose up --remove-orphans --force-recreate control-plane

.PHONY: create-proxy-registries
create-proxy-registries:
	docker compose up -d --force-recreate proxy-docker proxy-ghcr proxy-gcr proxy-k8s proxy-quay

.PHONY: control-plane-bake
control-plane-bake:
	@if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi
	@if ! kind get clusters | grep metal-control-plane > /dev/null; then \
		kind create cluster $(KIND_ARGS) \
			--name metal-control-plane \
			--config $(KINDCONFIG) \
			--kubeconfig $(KUBECONFIG); fi
	$(MAKE) create-proxy-registries

.PHONY: partition
partition: partition-bake
	docker compose up --remove-orphans --force-recreate partition

.PHONY: partition-bake
partition-bake: external_network
	docker pull $(MINI_LAB_VM_IMAGE)
ifeq ($(CI),true)
	docker pull $(MINI_LAB_SONIC_IMAGE)
endif
ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms),$(MINI_LAB_FLAVOR))
	docker pull $(MINI_LAB_SONIC_IMAGE)
endif
	@if ! sudo $(CONTAINERLAB) --topo $(LAB_TOPOLOGY) inspect | grep -i leaf01 > /dev/null; then \
		sudo --preserve-env=MINI_LAB_SONIC_IMAGE --preserve-env=MINI_LAB_DELL_SONIC_VERSION --preserve-env=MINI_LAB_VM_IMAGE $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
		./scripts/deactivate_offloading.sh; fi

.PHONY: external_network
external_network:
	@if ! docker network ls | grep -q mini_lab_ext; then \
  		docker network create mini_lab_ext \
			--driver=bridge \
			--gateway=203.0.113.1 \
			--subnet=203.0.113.0/24 \
			--ip-range=203.0.113.0/26 \
			--ipv6 \
			--gateway=2001:db8::1 \
			--subnet=2001:db8::/48 \
			--opt "com.docker.network.driver.mtu=9000" \
			--opt "com.docker.network.bridge.name=mini_lab_ext" \
			--opt "com.docker.network.bridge.enable_ip_masquerade=true" && \
		sudo ip route add 203.0.113.128/25 via 203.0.113.128 dev mini_lab_ext && \
		sudo ip -6 route add 2001:db8:0:113::/64 via 2001:db8:0:1::1 dev mini_lab_ext; \
	fi

.PHONY: env
env:
	@./env.sh

.PHONY: cleanup
cleanup: cleanup-control-plane cleanup-partition

.PHONY: cleanup-control-plane
cleanup-control-plane:
	kind delete cluster --name metal-control-plane
	docker compose down
	rm -f $(KUBECONFIG)

.PHONY: cleanup-partition
cleanup-partition:
	mkdir -p clab-mini-lab
	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.dell_sonic.yaml
	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.capms.yaml
	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.kamaji.yaml
	docker network rm --force mini_lab_ext

.PHONY: _privatenet
_privatenet: env
	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network

.PHONY: update-userdata
update-userdata:
	cat files/ignition.yaml | docker run --rm -i ghcr.io/metal-stack/metal-deployment-base:$$DEPLOYMENT_BASE_IMAGE_TAG ct | jq > files/ignition.json

.PHONY: machine
machine: _privatenet update-userdata
	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine create \
		--description test \
		--name test \
		--hostname test \
		--project 00000000-0000-0000-0000-000000000001 \
		--partition mini-lab \
		--image $(MACHINE_OS) \
		--size v1-small-x86 \
		--userdata "@/tmp/ignition.json" \
		--networks $(shell docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')

.PHONY: firewall
firewall: _privatenet update-userdata
	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl firewall create \
		--description fw \
		--name fw \
		--hostname fw \
		--project 00000000-0000-0000-0000-000000000001 \
		--partition mini-lab \
		--image firewall-ubuntu-3.0 \
		--size v1-small-x86 \
		--userdata "@/tmp/ignition.json" \
		--firewall-rules-file=/tmp/rules.yaml \
		--networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')

.PHONY: public-ip
public-ip:
	@docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip create --name test --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --addressfamily IPv4 -o template --template "{{ .ipaddress }}"

.PHONY: public-ipv6
public-ipv6:
	@docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl network ip create --name test --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --addressfamily IPv6 -o template --template "{{ .ipaddress }}"

.PHONY: ls
ls: env
	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine ls

## SWITCH MANAGEMENT ##

.PHONY: ssh-leafconfig
ssh-leafconfig:
	@grep "Host leaf01" ~/.ssh/config || echo -e "Host leaf01\n    StrictHostKeyChecking no\n    IdentityFile $(shell pwd)/files/ssh/id_ed25519\n" >>~/.ssh/config
	@grep "Host leaf02" ~/.ssh/config || echo -e "Host leaf02\n    StrictHostKeyChecking no\n    IdentityFile $(shell pwd)/files/ssh/id_ed25519\n" >>~/.ssh/config

.PHONY: docker-leaf01
docker-leaf01:
	@echo "export DOCKER_HOST=ssh://root@leaf01/var/run/docker.sock"

.PHONY: docker-leaf02
docker-leaf02:
	@echo "export DOCKER_HOST=ssh://root@leaf02/var/run/docker.sock"

.PHONY: ssh-leaf01
ssh-leaf01:
	ssh -F files/ssh/config leaf01

.PHONY: ssh-leaf02
ssh-leaf02:
	ssh -F files/ssh/config leaf02

## MACHINE MANAGEMENT ##
.PHONY: _ipmi_power
_ipmi_power:
	docker exec $(VM) ipmitool -C 3 -I lanplus -U ADMIN -P ADMIN -H 127.0.0.1 chassis power $(COMMAND)

.PHONY: start-machines
start-machines:
	@for i in $$(docker container ps --filter label=clab-node-group=machines --quiet); do \
		$(MAKE) --no-print-directory _ipmi_power VM=$$i COMMAND='on'; \
	done

.PHONY: power-on-machine01
power-on-machine01:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine01 COMMAND=on

.PHONY: power-on-machine02
power-on-machine02:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine02 COMMAND=on

.PHONY: power-on-machine03
power-on-machine03:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine03 COMMAND=on

.PHONY: power-on-machine04
power-on-machine04:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine04 COMMAND=on

.PHONY: power-reset-machine01
power-reset-machine01:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine01 COMMAND=reset

.PHONY: power-reset-machine02
power-reset-machine02:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine02 COMMAND=reset

.PHONY: power-reset-machine03
power-reset-machine03:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine03 COMMAND=reset

.PHONY: power-reset-machine04
power-reset-machine04:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine04 COMMAND=reset

.PHONY: power-off-machine01
power-off-machine01:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine01 COMMAND=off

.PHONY: power-off-machine02
power-off-machine02:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine02 COMMAND=off

.PHONY: power-off-machine03
power-off-machine03:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine03 COMMAND=off

.PHONY: power-off-machine04
power-off-machine04:
	@$(MAKE) --no-print-directory _ipmi_power VM=machine04 COMMAND=off

.PHONY: _console
_console:
	docker exec --interactive --tty $(VM) ipmitool -C 3 -I lanplus -U ADMIN -P ADMIN -H 127.0.0.1 sol activate

.PHONY: console-machine01
console-machine01:
	@$(MAKE) --no-print-directory _console VM=machine01

.PHONY: console-machine02
console-machine02:
	@$(MAKE) --no-print-directory _console VM=machine02

.PHONY: console-machine03
console-machine03:
	@$(MAKE) --no-print-directory _console VM=machine03

.PHONY: console-machine04
console-machine04:
	@$(MAKE) --no-print-directory _console VM=machine04

.PHONY: _password
_password: env
	docker compose run $(DOCKER_COMPOSE_RUN_ARG) metalctl machine consolepassword $(MACHINE_UUID)

.PHONY: password-machine01
password-machine01:
	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine01 MACHINE_UUID=00000000-0000-0000-0000-000000000001

.PHONY: password-machine02
password-machine02:
	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine02 MACHINE_UUID=00000000-0000-0000-0000-000000000002

.PHONY: password-machine03
password-machine03:
	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine03 MACHINE_UUID=00000000-0000-0000-0000-000000000003

.PHONY: password-machine04
password-machine04:
	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine04 MACHINE_UUID=00000000-0000-0000-0000-000000000004

.PHONY: password-machine0%
password-machine0%:
	@$(MAKE) --no-print-directory _password	MACHINE_NAME=machine0$* MACHINE_UUID=00000000-0000-0000-0000-00000000000$*

## SSH TARGETS FOR MACHINES ##
# Python code could be replaced by jq, but it is not preinstalled on Cumulus
.PHONY: ssh-firewall
ssh-firewall:
	$(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \
		jq -r 'to_entries[0] | .value.bgpNeighborAddr + \"%\" + .key'" \
	))
	ssh -F files/ssh/config $(fw) $(COMMAND)

.PHONY: ssh-machine
ssh-machine:
	$(eval machine = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp vrf Vrf20 neighbors test json' | \
		jq -r 'to_entries[0] | .value.bgpNeighborAddr + \"%\" + .key'" \
	))
	ssh -F files/ssh/config $(machine) $(COMMAND)

.PHONY: test-connectivity-to-external-service
test-connectivity-to-external-service:
	@for i in $$(seq 1 $(MAX_RETRIES)); do \
		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent http://203.0.113.100" > /dev/null 2>&1; then \
			echo "Connected successfully"; \
			exit 0; \
		else \
			echo "Connection failed"; \
			if [ $$i -lt $(MAX_RETRIES) ]; then \
				echo "Retrying in 2 seconds..."; \
				sleep 2; \
			else \
				echo "Max retries reached"; \
				exit 1; \
			fi; \
		fi; \
	done

.PHONY: test-connectivity-to-external-service-via-ipv6
test-connectivity-to-external-service-via-ipv6:
	@for i in $$(seq 1 $(MAX_RETRIES)); do \
		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent http://[2001:db8::10]" > /dev/null 2>&1; then \
			echo "Connected successfully"; \
			exit 0; \
		else \
			echo "Connection failed"; \
			if [ $$i -lt $(MAX_RETRIES) ]; then \
				echo "Retrying in 2 seconds..."; \
				sleep 2; \
			else \
				echo "Max retries reached"; \
				exit 1; \
			fi; \
		fi; \
	done

.PHONY: build-sonic-base
build-sonic-base:
	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202311 images/sonic/base-202311
	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202411 images/sonic/base-202411
	docker build -t ghcr.io/metal-stack/mini-lab-sonic-base:202505 images/sonic/base-202505

## DEV TARGETS ##

.PHONY: dev-env
dev-env:
	@echo "export METALCTL_API_URL=${METALCTL_API_URL}"
	@echo "export METALCTL_HMAC=${METALCTL_HMAC}"
	@echo "export KUBECONFIG=$(KUBECONFIG)"

build-dell-sonic:
	if [ ! -f "sonic-vs.img" ]; then \
	    @echo "sonic-vs.img is expected in this directory"; exit; fi

	@git clone https://github.com/srl-labs/vrnetlab.git
	@cd vrnetlab && git checkout e41f48bc5cae777b56b71b67e3c5642fdbd8f315
	@cp ./sonic-vs.img vrnetlab/dell/dell_sonic/dell-sonic-$(MINI_LAB_DELL_SONIC_VERSION).qcow2
	@cd vrnetlab/dell/dell_sonic && make
	docker tag vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION) r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)
	@rm -rf ./vrnetlab

## Gardener integration

.PHONY: fetch-virtual-kubeconfig
fetch-virtual-kubeconfig:
	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443
	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden
	@kubectl --kubeconfig=$(KUBECONFIG) config unset clusters.garden
	@KUBECONFIG=$(KUBECONFIG):.virtual-kubeconfig kubectl config view --flatten > .merged-kubeconfig
	@rm .virtual-kubeconfig
	@mv .merged-kubeconfig $(KUBECONFIG)