Add support for Calico, use RBAC and secrets encryption.

.gitignore

@@ -4,3 +4,4 @@ _artifact/
 *.pem
 *.csr
 *.kubeconfig
+manifests/calico/tigera-operator.yaml

Dockerfile

@@ -188,7 +188,7 @@ COPY hack/etc_systemd_system_user@.service.d_delegate.conf /etc/systemd/system/u
 RUN chmod +x /docker-entrypoint.sh && \
 # As of Feb 2020, Fedora has wrong permission bits on newuidmap and newgidmap.
   chmod +s /usr/bin/newuidmap /usr/bin/newgidmap && \
-  dnf install -q -y conntrack findutils fuse3 git iproute iptables hostname procps-ng time which \
+  dnf install -q -y conntrack findutils fuse3 git iproute iptables hostname procps-ng time which gettext \
 # systemd-container: for machinectl
   systemd-container && \
   useradd --create-home --home-dir /home/user --uid 1000 -G systemd-journal user && \

Makefile

@@ -10,6 +10,7 @@ _binaries:
 	$(eval cid := $(shell docker create $(IMAGE)))
 	docker cp $(cid):/home/user/usernetes/bin ./bin
 	docker rm $(cid)
+	curl https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/tigera-operator.yaml --show-error --output manifests/calico/tigera-operator.yaml
 
 image:
 ifeq ($(DOCKER_BUILDKIT),1)

README.md

@@ -43,6 +43,7 @@ Usernetes aims to provide a reference distribution of Kubernetes that can be ins
   * crun
 * Multi-node CNI
   * Flannel (VXLAN)
+  * Calico (VXLAN)
 * CoreDNS
 
 Installer scripts are in POC status.
@@ -138,6 +139,7 @@ xt_multiport
 xt_nat
 xt_tcpudp
 ```
+On older kernels, `xt_MASQUERADE` should be `ipt_MASQUERADE` and `ip6t_MASQUERADE`.
 
 ### cgroup v2
 
@@ -366,7 +368,7 @@ $ sudo sh -c "echo 0   2147483647  > /proc/sys/net/ipv4/ping_group_range"
 
 * 10.5.0.0/16: The CIDR for Flannel
 
-* 10.88.0.0/16: The CIDR for single-node CNI
+* 10.88.0.0/16: The CIDR for single-node CNI and Calico
 
 ### Install Usernetes from source
 

boot/kube-apiserver.sh

@@ -19,6 +19,6 @@ exec $(dirname $0)/nsenter.sh kube-apiserver \
 	--service-account-signing-key-file=$XDG_CONFIG_HOME/usernetes/master/service-account-key.pem \
 	--advertise-address=$(cat $XDG_RUNTIME_DIR/usernetes/parent_ip) \
 	--allow-privileged \
+	--authorization-mode=Node,RBAC \
+	--encryption-provider-config=$XDG_CONFIG_HOME/usernetes/master/secrets-encryption.yml \
 	$@
-
-# TODO: enable --authorization-mode=Node,RBAC \

boot/rootlesskit.sh

@@ -12,6 +12,7 @@ rk_state_dir=$XDG_RUNTIME_DIR/usernetes/rootlesskit
 : ${U7S_ROOTLESSKIT_FLAGS=}
 : ${U7S_ROOTLESSKIT_PORTS=}
 : ${U7S_FLANNEL=}
+: ${U7S_CALICO=0}
 
 : ${_U7S_CHILD=0}
 if [[ $_U7S_CHILD == 0 ]]; then
@@ -64,7 +65,11 @@ else
 
 	# Copy CNI config to /etc/cni/net.d (Likely to be hardcoded in CNI installers)
 	mkdir -p /etc/cni/net.d
-	cp -f $U7S_BASE_DIR/config/cni_net.d/* /etc/cni/net.d
+
+	# Disable bridge cni when using calico
+	if [[ $U7S_CALICO == 0 ]]; then
+		cp -f $U7S_BASE_DIR/config/cni_net.d/* /etc/cni/net.d
+	fi
 	if [[ $U7S_FLANNEL == 1 ]]; then
 		cp -f $U7S_BASE_DIR/config/flannel/cni_net.d/* /etc/cni/net.d
 		mkdir -p /run/flannel

install.sh

@@ -51,7 +51,7 @@ function usage() {
 	echo
 	echo "  --start=UNIT        Enable and start the specified target after the installation, e.g. \"u7s.target\". Set to an empty to disable autostart. (Default: \"$start\")"
 	echo "  --cri=RUNTIME       Specify CRI runtime, \"containerd\" or \"crio\". (Default: \"$cri\")"
-	echo '  --cni=RUNTIME       Specify CNI, an empty string (none) or "flannel". (Default: none)'
+	echo '  --cni=RUNTIME       Specify CNI, an empty string (none), \"calico\" or "flannel". (Default: none)'
 	echo "  -p, --publish=PORT  Publish ports in RootlessKit's network namespace, e.g. \"0.0.0.0:10250:10250/tcp\". Can be specified multiple times. (Default: \"${publish_default}\")"
 	echo "  --cidr=CIDR         Specify CIDR of RootlessKit's network namespace, e.g. \"10.0.100.0/24\". (Default: \"$cidr\")"
 	echo
@@ -107,7 +107,7 @@ while true; do
 	--cni)
 		cni="$2"
 		case "$cni" in
-		"" | "flannel") ;;
+		"" | "flannel" | "calico") ;;
 
 		*)
 			ERROR "Unknown CNI \"$cni\". Supported values: \"\" (default) \"flannel\" ."
@@ -429,6 +429,20 @@ EOF
 	fi
 fi
 
+# Need to enable calico before starting rootlesskit
+if [ "$cni" = "calico" ]; then
+	cat <<EOF >>${config_dir}/usernetes/env
+U7S_CALICO=1
+EOF
+fi
+
+### Secret encryption
+if [ ! -f ${config_dir}/usernetes/master/secrets-encryption.yaml.template ]; then
+	INFO "Enabling secrets encryption"
+	export ENCRYPTION_SECRET=$(cat /dev/urandom | head -c 32 | base64 -w 0)
+	(envsubst '$ENCRYPTION_SECRET' < manifests/secrets-encryption.yaml.template) > ${config_dir}/usernetes/master/secrets-encryption.yml
+fi
+
 ### Finish installation
 systemctl --user daemon-reload
 if [ -z $start ]; then
@@ -447,6 +461,51 @@ if systemctl --user -q is-active u7s-master.target; then
 	PATH="${base}/bin:$PATH"
 	KUBECONFIG="${config_dir}/usernetes/master/admin-localhost.kubeconfig"
 	export PATH KUBECONFIG
+	INFO "Granting all permissions to kubectl"
+	cat <<EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:kube-apiserver-to-kubelet
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/proxy
+      - nodes/stats
+      - nodes/log
+      - nodes/spec
+      - nodes/metrics
+    verbs:
+      - "*"
+EOF
+	cat <<EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:kube-apiserver
+  namespace: ""
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-apiserver-to-kubelet
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: kubernetes
+EOF
+	if [ "$cni" = "calico" ]; then
+		INFO "Installing calico"
+		set -x
+		kubectl create -f ${base}/manifests/calico/tigera-operator.yaml
+		${base}/manifests/calico/custom-resources.sh
+		set +x
+		sleep 30
+	fi
 	INFO "Installing CoreDNS"
 	set -x
 	# sleep for waiting the node to be available
@@ -457,7 +516,7 @@ if systemctl --user -q is-active u7s-master.target; then
 	INFO "Waiting for CoreDNS pods to be available"
 	set -x
 	# sleep for waiting the pod object to be created
-	sleep 3
+	sleep 10
 	kubectl -n kube-system wait --for=condition=ready pod -l k8s-app=kube-dns
 	kubectl get pods -A -o wide
 	set +x

manifests/calico/custom-resources.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+export U7S_BASE_DIR=$(realpath $(dirname $0)/../..)
+source $U7S_BASE_DIR/common/common.inc.sh
+
+mkdir -p $XDG_RUNTIME_DIR/usernetes/calico
+cat >$XDG_RUNTIME_DIR/usernetes/calico/custom-resources.yaml <<EOF
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    # Note: The ipPools section cannot be modified post-install.
+    ipPools:
+    - blockSize: 26
+      cidr: 10.88.0.0/16
+      encapsulation: VXLANCrossSubnet
+      natOutgoing: Enabled
+      nodeSelector: all()
+  kubeletVolumePluginPath: "$XDG_DATA_HOME/usernetes/kubelet"
+  nonPrivileged: Enabled
+  flexVolumePath: "$XDG_DATA_HOME/usernetes/kubelet-plugins-exec"
+
+---
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}
+EOF
+
+exec $U7S_BASE_DIR/boot/nsenter.sh kubectl \
+	create -f "$XDG_RUNTIME_DIR/usernetes/calico/custom-resources.yaml"

manifests/coredns.yaml

@@ -28,6 +28,13 @@ rules:
   - nodes
   verbs:
   - get
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

manifests/secrets-encryption.yaml.template

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: EncryptionConfig
+resources:
+  - resources:
+      - secrets
+    providers:
+      - secretbox:
+          keys:
+            - name: secrets_key
+              secret: $ENCRYPTION_SECRET