calico: upstream pr preparation

Dockerfile

@@ -2,39 +2,60 @@ ARG BASE_IMAGE=docker.io/kindest/node:v1.33.0@sha256:91e9ed777db80279c22d1d1068c
 ARG CNI_PLUGINS_VERSION=v1.7.1
 ARG HELM_VERSION=v3.17.3
 ARG FLANNEL_VERSION=v0.26.7
+ARG CALICO_VERSION=v3.30.5
 FROM ${BASE_IMAGE}
 COPY Dockerfile.d/SHA256SUMS.d/ /tmp/SHA256SUMS.d
 ARG CNI_PLUGINS_VERSION
 ARG HELM_VERSION
 ARG FLANNEL_VERSION
-# This are private on our cluster and need to be copied to here
-COPY cspca.llnl.gov.cer.pem /usr/local/share/ca-certificates/
-COPY cspca.cer.pem /usr/local/share/ca-certificates/
+ARG CALICO_VERSION
 RUN update-ca-certificates
 RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
+  # CNI Plugins
   fname="cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz" && \
-  curl --insecure -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
+  curl -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
   grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
   mkdir -p /opt/cni/bin && \
   tar xzf "${fname}" -C /opt/cni/bin && \
   rm -f "${fname}" && \
+  # Helm
   fname="helm-${HELM_VERSION}-linux-${arch}.tar.gz" && \
-  curl --insecure -o "${fname}" -fSL "https://get.helm.sh/${fname}" && \
+  curl -o "${fname}" -fSL "https://get.helm.sh/${fname}" && \
   grep "${fname}" "/tmp/SHA256SUMS.d/helm-${HELM_VERSION}" | sha256sum -c && \
   tar xzf "${fname}" -C /usr/local/bin --strip-components=1 -- "linux-${arch}/helm" && \
   rm -f "${fname}" && \
+  # Flannel
   fname="flannel.tgz" && \
-  curl --insecure -o "${fname}" -fSL "https://github.com/flannel-io/flannel/releases/download/${FLANNEL_VERSION}/${fname}" && \
+  curl -o "${fname}" -fSL "https://github.com/flannel-io/flannel/releases/download/${FLANNEL_VERSION}/${fname}" && \
   grep "${fname}" "/tmp/SHA256SUMS.d/flannel-${FLANNEL_VERSION}" | sha256sum -c && \
   tar xzf "${fname}" -C / && \
-  rm -f "${fname}"
+  rm -f "${fname}" && \
+  # Calico (calicoctl)
+  fname="calicoctl-linux-${arch}" && \
+  curl -o "${fname}" -fSL "https://github.com/projectcalico/calico/releases/download/${CALICO_VERSION}/${fname}" && \
+  grep "${fname}" "/tmp/SHA256SUMS.d/calico-${CALICO_VERSION}" | sha256sum -c && \
+  chmod +x "${fname}" && \
+  mv "${fname}" /usr/local/bin/calicoctl && \
+  # Calico manifest - derive v3.31 from v3.31.5
+  calico_branch="$(echo "${CALICO_VERSION}" | cut -d. -f1,2)" && \
+  fname="calico.yaml" && \
+  curl -o "/${fname}" -fSL "https://raw.githubusercontent.com/projectcalico/calico/refs/heads/release-${calico_branch}/manifests/calico.yaml" && \
+  grep "${fname}" "/tmp/SHA256SUMS.d/calico-manifest-${CALICO_VERSION}" | sha256sum -c
+
 # gettext-base: for `envsubst`
 # moreutils: for `sponge`
 # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
+# ipset: for using calico and command line utils
+# yq: needed to install custom calico manifests
 RUN apt-get update && apt-get install -y --no-install-recommends \
   gettext-base \
   moreutils \
-  socat
-ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
+  socat \
+  ipset \
+  yq \
+  && rm -rf /var/lib/apt/lists/*
+# Calico
+ENV FELIX_IGNORELOOSERPF=true
+ADD Dockerfile.d/etc_udev_rules.d_90-flannel-calico.rules /etc/udev/rules.d/90-flannel-calico.rules
 ADD Dockerfile.d/u7s-entrypoint.sh /
-ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]
+ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Dockerfile.d/SHA256SUMS.d/calico-manifest-v3.30.5

@@ -0,0 +1 @@
+ed105288f27c7ba4cae143bcaa3f6425f02d84fa94a4786cc8e3d371412c1a36  calico.yaml

Dockerfile.d/SHA256SUMS.d/calico-v3.30.5

@@ -0,0 +1,2 @@
+6cdfb17b0276f648f4fdb051a5d75617a50b3c328d4cccfc40d087b96c361d80  calicoctl-linux-amd64
+7611343e7a56e770b95e2bb882dda787efbbd4331b1dd6316ff8ea189238dfaa  calicoctl-linux-arm64

Dockerfile.d/etc_udev_rules.d_90-flannel.rules → Dockerfile.d/etc_udev_rules.d_90-flannel-calico.rules

@@ -3,3 +3,4 @@
 # https://github.com/kubernetes/kops/pull/9074
 # https://github.com/karmab/kcli/commit/b1a8eff658d17cf4e28162f0fa2c8b2b10e5ad00
 SUBSYSTEM=="net", ACTION=="add|change|move", ENV{INTERFACE}=="flannel.1", RUN+="/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off"
+SUBSYSTEM=="net", ACTION=="add|change|move", ENV{INTERFACE}=="vxlan.calico", RUN+="/usr/sbin/ethtool -K vxlan.calico tx-checksum-ip-generic off"

Makefile

@@ -6,6 +6,7 @@ export PORT_ETCD ?= 2379
 export PORT_KUBELET ?= 10250
 export PORT_FLANNEL ?= 8472
 export PORT_KUBE_APISERVER ?= 6443
+export PORT_CALICO ?= 5473
 
 # HOSTNAME is the name of the physical host
 export HOSTNAME ?= $(shell hostname)
@@ -35,6 +36,7 @@ NODE_SHELL := $(COMPOSE) exec \
 	-e NODE_IP=$(NODE_IP) \
 	-e PORT_KUBE_APISERVER=$(PORT_KUBE_APISERVER) \
 	-e PORT_FLANNEL=$(PORT_FLANNEL) \
+	-e PORT_CALICO=$(PORT_CALICO) \
 	-e PORT_KUBELET=$(PORT_KUBELET) \
 	-e PORT_ETCD=$(PORT_ETCD) \
 	$(NODE_SERVICE_NAME)
@@ -160,3 +162,8 @@ install-flannel:
 	# We don't actually need it there, just on the physical node, so we use newer K8s and older flannel
 	$(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.25.1/kube-flannel.yml
 	#$(NODE_SHELL) /usernetes/Makefile.d/install-flannel.sh
+
+.PHONY: install-calico
+install-calico:
+	# Calico daemonset changes and node-level address changes
+	$(NODE_SHELL) /usernetes/Makefile.d/calico/install-calico.sh

Makefile.d/calico/calico-ethtool.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: calico-checksum-fix
+  namespace: kube-system
+  labels:
+    k8s-app: calico-checksum-fix
+spec:
+  selector:
+    matchLabels:
+      name: calico-checksum-fix
+  template:
+    metadata:
+      labels:
+        name: calico-checksum-fix
+    spec:
+      hostNetwork: true
+      hostPID: true
+      securityContext:
+        runAsUser: 0
+      initContainers:
+      - name: fix-checksum
+        image: ghcr.io/converged-computing/usernetes:alpine
+        # image: alpine:latest
+        command: ["/bin/sh", "-c"]
+        args:
+          - |
+            # nsenter -t 1 enters the init process's namespace (of the host)
+            # check if the interface exists before running ethtool
+            if [ -d /sys/class/net/vxlan.calico ]; then
+              echo "Applying ethtool fix to vxlan.calico..."
+              nsenter -t 1 -n -u -i -m -- ethtool -K vxlan.calico tx-checksum-ip-generic off
+            else
+              echo "vxlan.calico interface not found, skipping."
+            fi
+            iptables -I INPUT -p udp --dport 8472 -j ACCEPT
+            sysctl -w net.ipv4.conf.all.rp_filter=1
+            sysctl -w net.ipv4.conf.default.rp_filter=1
+            sysctl -w net.ipv4.conf.eth0.rp_filter=1
+            sysctl -w net.ipv4.conf.vxlan/calico.rp_filter=1
+        securityContext:
+          privileged: true
+      containers:
+      - name: pause
+        # image: registry.k8s.io/pause:3.9
+        image: ghcr.io/converged-computing/usernetes:pause
+      terminationGracePeriodSeconds: 0

Makefile.d/calico/install-calico.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Install standard Calico (downloaded in Dockerfile on build)
+CALICO_FILE="/calico.yaml"
+
+# backend to vxlan
+yq eval-all -i '(select(.kind == "ConfigMap" and .metadata.name == "calico-config").data.calico_backend) = "vxlan"' $CALICO_FILE
+
+# Disable IPIP and enable CrossSubnet VXLAN for IPv4/IPv6 in the Calico manifest
+yq eval-all -i '(select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].env[] | select(.name == "CALICO_IPV4POOL_IPIP").value) = "Never"' $CALICO_FILE
+yq eval-all -i '(select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].env[] | select(.name == "CALICO_IPV4POOL_VXLAN").value) = "CrossSubnet"' $CALICO_FILE
+yq eval-all -i '(select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].env[] | select(.name == "CALICO_IPV6POOL_VXLAN").value) = "CrossSubnet"' $CALICO_FILE
+
+# FELIX for rootless
+yq eval-all -i 'select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].env += {"name": "FELIX_IGNORELOOSERPF", "value": "true"}' $CALICO_FILE
+yq eval-all -i 'select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].env += {"name": "FELIX_VXLANPORT", "value": "8472"}' $CALICO_FILE
+yq eval-all -i 'select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].env += {"name": "FELIX_EXTERNALNODESCIDRLIST", "value": "10.100.0.0/16"}' $CALICO_FILE
+
+# health probes (Remove bird-ready and bird-live)
+yq eval-all -i '(select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].livenessProbe.exec.command) = ["/bin/calico-node", "-felix-live"]' $CALICO_FILE
+yq eval-all -i '(select(.kind == "DaemonSet" and .metadata.name == "calico-node").spec.template.spec.containers[0].readinessProbe.exec.command) = ["/bin/calico-node", "-felix-ready"]' $CALICO_FILE
+
+# install components with our rootless version
+kubectl apply -f ${CALICO_FILE}
+echo "Done. Final file is $CALICO_FILE"
+
+# Give a small break to settle - we need calico.vxlan to be created
+sleep 10
+
+# This must be removed or the address will be reset
+kubectl set env daemonset/calico-node IP- -n kube-system
+
+# Allow pods to recreate
+echo "Recreating calico pods..."
+sleep 10
+
+# https://youtu.be/noriIzBKYRk?si=mlOC27ntvSEDw_VM&t=299
+# These commands need to be done bringing up node
+# iptables -I INPUT -p udp --dport 8472 -j ACCEPT
+# sysctl -w net.ipv4.conf.all.rp_filter=2
+# sysctl -w net.ipv4.conf.default.rp_filter=2
+# sysctl -w net.ipv4.conf.eth0.rp_filter=2
+# sysctl -w "net.ipv4.conf.vxlan/calico.rp_filter=2"  
+
+# This needs to be done after daemonset is patched
+# Note that the calico-node has a warning after this, but it won't work if we don't do it
+for node in $(kubectl get nodes -o name); do
+    host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+    nodename=$(cut -d / -f 2 <<< $node)
+    calicoctl --allow-version-mismatch patch node ${nodename} --patch='{"spec": {"bgp":{"ipv4Address": "'"$host_ip"'"}}}'
+done
+
+# applies ethtool -K vxlan.calico tx-checksum-ip-generic off
+# check with: bridge fdb show dev vxlan.calico should have node address NOT 10.x address
+kubectl apply --server-side -f /usernetes/Makefile.d/calico/calico-ethtool.yaml
+
+# These should be run after calico installed
+# 1. make sync-external-ip and make install-calico
+# the second has a daemonset to apply these commands
+# ethtool -K vxlan.calico tx-checksum-ip-generic off
+
+for node in $(kubectl get nodes -o name); do
+    # Extracts the node name and tell calico host for its BGP peering
+	# BGP mesh will not connect without this correct address
+    nodename=$(cut -d / -f 2 <<< $node)
+    calicoctl --allow-version-mismatch patch node ${nodename} --patch='{"spec": {"bgp":{"ipv4Address": "'"$host_ip"'"}}}'
+done

Makefile.d/check-preflight.sh

@@ -16,6 +16,8 @@ script_dir="$(dirname "$0")"
 detect_engine="${script_dir}"/detect-container-engine.sh
 : "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}"
 : "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}"
+
+# 1 will deploy calico by default
 : "${QUICK:=0}"
 : "${BUSYBOX_IMAGE:=docker.io/library/busybox:latest}"
 

Makefile.d/sync-external-ip.sh

@@ -16,4 +16,4 @@ for node in $(kubectl get nodes -o name); do
 	if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
 		kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
 	fi
-done
+done

README.md

@@ -19,6 +19,7 @@ but Usernetes (Gen 2) supports creating a cluster with multiple hosts.
 - CRI: containerd
 - OCI: runc
 - CNI: Flannel
+- CNI: Calico
 
 ## Requirements
 
@@ -72,7 +73,8 @@ EOF
 sudo systemctl restart systemd-modules-load.service
 ```
 
-- sysctl:
+- sysctl
+
 ```
 sudo tee /etc/sysctl.d/99-usernetes.conf <<EOF >/dev/null
 net.ipv4.conf.default.rp_filter = 2
@@ -110,6 +112,8 @@ See `make help`.
 make up
 make kubeadm-init
 make install-flannel
+# or
+make install-calico
 
 # Enable kubectl
 make kubeconfig
@@ -187,6 +191,20 @@ make up
 
 ![docs/images/multi-tenancy.png](./docs/images/multi-tenancy.png)
 
+### Calico
+
+For debugging. In u7s this address should be same as host:
+
+```bash
+bridge fdb show dev vxlan.calico
+```
+```console
+# "this address"
+66:63:44:f3:b6:76 dst 192.168.128.222 self permanent
+```
+
+If you see the container interface (10.0.x) this is a bug. It could be that the calico-node daemonset still has the `IP` environment variable set to autodetect (which will clobber any changes you make) or you did not issue all the commands in the sync external ip script, or the daemonset to run ethtool.
+
 ### Rootful mode
 Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too.
 This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/).

docker-compose.yaml

@@ -12,6 +12,8 @@ services:
         ipv4_address: ${NODE_IP}
     ports:
       # <host>:<container>
+      # Calico
+      - ${PORT_CALICO}:${PORT_CALICO}
       # etcd (default: 2379)
       - ${PORT_ETCD}:${PORT_ETCD}
       # kube-apiserver (default: 6443)