vagrant-lab-k8s/provision/rke2-server.sh

#!/bin/bash
set -euo pipefail
#
# Helper functions
#
declare -i term_width=80

h1() {
    declare border padding text
    border='\e[1;34m'"$(printf '=%.0s' $(seq 1 "$term_width"))"'\e[0m'
    padding="$(printf ' %.0s' $(seq 1 $(((term_width - $(wc -m <<<"$*")) / 2))))"
    text="\\e[1m$*\\e[0m"
    echo -e "$border"
    echo -e "${padding}${text}${padding}"
    echo -e "$border"
}

h2() {
    printf '\e[1;33m==>\e[37;1m %s\e[0m\n' "$*"
}

rke2_command="$1"; shift
rke2_channel="${1:-latest}"; shift
rke2_version="${1:-v1.21.5+rke2r1}"; shift
ip_address="$1"; shift
krew_version="${1:-v0.4.1}"; shift || true # NB see https://github.com/kubernetes-sigs/krew
fqdn="$(hostname --fqdn)"
rke2_url="https://server.$(hostname --domain):9345"

h1 "Install rke2 server"
h2 "Version: ${rke2_version}"
h2 "Server url: ${rke2_url}"

cat >/etc/motd <<'EOF'
      _        ____
 _ __| | _____|___ \   ___  ___ _ ____   _____ _ __
| '__| |/ / _ \ __) | / __|/ _ \ '__\ \ / / _ \ '__|
| |  |   <  __// __/ _\__ \  __/ |   \ V /  __/ |
|_|  |_|\_\___|_____(_)___/\___|_|    \_/ \___|_|

EOF

h1 "Configure rke2 server"
# configure the rke2 server.
# see https://docs.rke2.io/install/install_options/install_options/
# see https://docs.rke2.io/install/install_options/server_config/
install -d -m 700 /etc/rancher/rke2
install /dev/null -m 600 /etc/rancher/rke2/config.yaml
if [ "$rke2_command" != 'cluster-init' ]; then
  cat >>/etc/rancher/rke2/config.yaml <<EOF
server: $rke2_url
token: $(cat /vagrant/tmp/node-token)
EOF
fi
cat >>/etc/rancher/rke2/config.yaml <<EOF
node-ip: $ip_address
node-taint: CriticalAddonsOnly=true:NoExecute
tls-san:
 - server.$(hostname --domain)
 - $fqdn
cni: calico
cluster-cidr: 10.12.0.0/16
service-cidr: 10.13.0.0/16
cluster-dns: 10.13.0.10
cluster-domain: cluster.local
EOF

h1 "Install NFS"
apt-get install -y nfs-server
mkdir /data
chmod -cR 777 /data
cat << EOF >> /etc/exports
/data ${ip_address}(rw,no_subtree_check,no_root_squash)
EOF
systemctl enable --now nfs-server
exportfs -ar

h1 "Install rke2 server"
h2 "Version: ${rke2_version}"
# install rke2 server.
# see https://docs.rke2.io/install/install_options/install_options/
# see https://docs.rke2.io/install/install_options/server_config/
curl -sfL https://raw.githubusercontent.com/rancher/rke2/$rke2_version/install.sh \
  | \
    INSTALL_RKE2_CHANNEL="$rke2_channel" \
    INSTALL_RKE2_VERSION="$rke2_version" \
    INSTALL_RKE2_TYPE="server" \
    sh -

h2 "Start rke2 server"
# start the rke2-server service.
systemctl cat rke2-server
systemctl enable rke2-server.service
systemctl start rke2-server.service

h2 "Configure system path for rke2"
# symlink the utilities and setup the environment variables to use them.
ln -fs /var/lib/rancher/rke2/bin/{kubectl,crictl,ctr} /usr/local/bin/
cat >/etc/profile.d/01-rke2.sh <<'EOF'
export CONTAINERD_ADDRESS=/run/k3s/containerd/containerd.sock
export CONTAINERD_NAMESPACE=k8s.io
export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
EOF
source /etc/profile.d/01-rke2.sh


h1 "wait for this node to be Ready."
# e.g. server     Ready    control-plane,etcd,master   3m    v1.21.5+rke2r1
$SHELL -c 'node_name=$(hostname); echo "waiting for node $node_name to be ready..."; while [ -z "$(kubectl get nodes $node_name | grep -E "$node_name\s+Ready\s+")" ]; do sleep 3; done; echo "node ready!"'

h1 "wait for the kube-dns pod to be Running."
# e.g. rke2-coredns-rke2-coredns-7bb4f446c-jksvq   1/1     Running   0          33m
$SHELL -c 'while [ -z "$(kubectl get pods --selector k8s-app=kube-dns --namespace kube-system | grep -E "\s+Running\s+")" ]; do sleep 3; done'

if [ "$rke2_command" == 'cluster-init' ]; then
  h2 "Copy server token to shared directory"
  install -d /vagrant/tmp
  cp /var/lib/rancher/rke2/server/node-token /vagrant/tmp/node-token
fi

h1 "Install krew package manager"
h2 "Version: ${krew_version}"
apt-get install -y --no-install-recommends git
wget -qO- "https://github.com/kubernetes-sigs/krew/releases/download/$krew_version/krew.tar.gz" | tar xzf - ./krew-linux_amd64
wget -q "https://github.com/kubernetes-sigs/krew/releases/download/$krew_version/krew.yaml"
./krew-linux_amd64 install --manifest=krew.yaml
rm krew-linux_amd64
cat >/etc/profile.d/krew.sh <<'EOF'
export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH"
EOF
source /etc/profile.d/krew.sh
kubectl krew version

# install the bash completion scripts.
crictl completion bash >/usr/share/bash-completion/completions/crictl
kubectl completion bash >/usr/share/bash-completion/completions/kubectl

# save kubeconfig in the host.
if [ "$rke2_command" == 'cluster-init' ]; then
  mkdir -p /vagrant/tmp
  python3 - <<EOF
import base64
import yaml

d = yaml.load(open('/etc/rancher/rke2/rke2.yaml', 'r'))

# save cluster ca certificate.
for c in d['clusters']:
    open(f"/vagrant/tmp/{c['name']}-ca-crt.pem", 'wb').write(base64.b64decode(c['cluster']['certificate-authority-data']))

# save user client certificates.
for u in d['users']:
    open(f"/vagrant/tmp/{u['name']}-crt.pem", 'wb').write(base64.b64decode(u['user']['client-certificate-data']))
    open(f"/vagrant/tmp/{u['name']}-key.pem", 'wb').write(base64.b64decode(u['user']['client-key-data']))
    print(f"Kubernetes API Server https://$ip_address:6443 user {u['name']} client certificate in tmp/{u['name']}-*.pem")

# set the server ip.
for c in d['clusters']:
    c['cluster']['server'] = 'https://$ip_address:6443'

yaml.dump(d, open('/vagrant/tmp/admin.conf', 'w'), default_flow_style=False)
EOF
fi

# show cluster-info.
kubectl cluster-info

# list etcd members.
etcdctl --write-out table member list

# show the endpoint status.
etcdctl --write-out table endpoint status

# list nodes.
kubectl get nodes -o wide

# rbac info.
kubectl get serviceaccount --all-namespaces
kubectl get role --all-namespaces
kubectl get rolebinding --all-namespaces
kubectl get rolebinding --all-namespaces -o json | jq .items[].subjects
kubectl get clusterrole --all-namespaces
kubectl get clusterrolebinding --all-namespaces
kubectl get clusterrolebinding --all-namespaces -o json | jq .items[].subjects

# rbac access matrix.
# see https://github.com/corneliusweig/rakkess/blob/master/doc/USAGE.md
#kubectl krew install access-matrix
#kubectl access-matrix version --full
#kubectl access-matrix # at cluster scope.
#kubectl access-matrix --namespace default
#kubectl access-matrix --sa kubernetes-dashboard --namespace kubernetes-dashboard

# list system secrets.
kubectl -n kube-system get secret

# list all objects.
# NB without this hugly redirect the kubectl output will be all messed
#    when used from a vagrant session.
kubectl get all --all-namespaces

# really get all objects.
# see https://github.com/corneliusweig/ketall/blob/master/doc/USAGE.md
kubectl krew install get-all
kubectl get-all

# list services.
kubectl get svc

# list running pods.
kubectl get pods --all-namespaces -o wide

# list runnnig pods.
crictl pods

# list running containers.
crictl ps
ctr containers ls

# show listening ports.
ss -n --tcp --listening --processes

# show network routes.
ip route

# show memory info.
free

# show versions.
kubectl version
crictl version
ctr version