commit ef753a3c7e2be09a44cd94ca60499eb495feb709 Author: Aleksander Cynarski Date: Sun Mar 13 13:29:53 2022 +0100 init commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..53b3473 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.vagrant/ +tmp/ diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..feabf6f --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,101 @@ +# to make sure the nodes are created in order, we +# have to force a --no-parallel execution. +ENV['VAGRANT_NO_PARALLEL'] = 'yes' + +require 'ipaddr' + +# see https://update.rke2.io/v1-release/channels +# see https://github.com/rancher/rke2/releases +rke2_channel = 'latest' +rke2_version = 'v1.23.4+rke2r1' +# see https://github.com/etcd-io/etcd/releases +etcdctl_version = 'v3.5.2' +# see https://github.com/derailed/k9s/releases +k9s_version = 'v0.25.18' +# see https://github.com/kubernetes-sigs/krew/releases +krew_version = 'v0.4.3' + +number_of_server_nodes = 1 +number_of_agent_nodes = 2 + +first_server_node_ip = '10.11.0.101' +first_agent_node_ip = '10.11.0.201' + +server_node_ip_address = IPAddr.new first_server_node_ip +agent_node_ip_address = IPAddr.new first_agent_node_ip + +domain = 'rke2.test' +rke2_server_domain = "server.#{domain}" +rke2_server_url = "https://#{rke2_server_domain}:9345" + +Vagrant.configure(2) do |config| + config.vm.box = 'generic/ubuntu2010' + + config.vm.provider 'libvirt' do |lv, config| + lv.cpus = 2 + lv.cpu_mode = 'host-passthrough' + lv.nested = true + lv.keymap = 'pt' + config.vm.synced_folder '.', '/vagrant', type: 'nfs', nfs_version: '4.2', nfs_udp: false + end + + (1..number_of_server_nodes).each do |n| + name = "server#{n}" + fqdn = "#{name}.#{domain}" + ip_address = server_node_ip_address.to_s; server_node_ip_address = server_node_ip_address.succ + + config.vm.define name do |config| + config.vm.provider 'libvirt' do |lv, config| + lv.memory = 2*1024 + end + config.vm.hostname = fqdn + config.vm.network :private_network, ip: ip_address, libvirt__forward_mode: 'none', libvirt__dhcp_enabled: false + config.vm.provision 'hosts' do |hosts| + hosts.autoconfigure = true + hosts.sync_hosts = true + hosts.add_localhost_hostnames = false + hosts.add_host first_server_node_ip, [rke2_server_domain] + end + config.vm.provision 'shell', path: 'provision/base.sh' + config.vm.provision 'shell', path: 'provision/etcdctl.sh', args: [etcdctl_version] + config.vm.provision 'shell', path: 'provision/k9s.sh', args: [k9s_version] + config.vm.provision 'shell', path: 'provision/rke2-server.sh', args: [ + n == 1 ? "cluster-init" : "cluster-join", + rke2_channel, + rke2_version, + ip_address, + krew_version + ] + if n == 1 + config.vm.provision 'shell', path: 'provision/example-app.sh' + end + end + end + + (1..number_of_agent_nodes).each do |n| + name = "agent#{n}" + fqdn = "#{name}.#{domain}" + ip_address = agent_node_ip_address.to_s; agent_node_ip_address = agent_node_ip_address.succ + + config.vm.define name do |config| + config.vm.provider 'libvirt' do |lv, config| + lv.memory = 2*1024 + end + config.vm.hostname = fqdn + config.vm.network :private_network, ip: ip_address, libvirt__forward_mode: 'none', libvirt__dhcp_enabled: false + config.vm.provision 'hosts' do |hosts| + hosts.autoconfigure = true + hosts.sync_hosts = true + hosts.add_localhost_hostnames = false + hosts.add_host first_server_node_ip, [rke2_server_domain] + end + config.vm.provision 'shell', path: 'provision/base.sh' + config.vm.provision 'shell', path: 'provision/rke2-agent.sh', args: [ + rke2_channel, + rke2_version, + rke2_server_url, + ip_address + ] + end + end +end diff --git a/provision/base.sh b/provision/base.sh new file mode 100644 index 0000000..d9e3186 --- /dev/null +++ b/provision/base.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -euo pipefail + +# +# Helper functions +# +declare -i term_width=80 + +h1() { + declare border padding text + border='\e[1;34m'"$(printf '=%.0s' $(seq 1 "$term_width"))"'\e[0m' + padding="$(printf ' %.0s' $(seq 1 $(((term_width - $(wc -m <<<"$*")) / 2))))" + text="\\e[1m$*\\e[0m" + echo -e "$border" + echo -e "${padding}${text}${padding}" + echo -e "$border" +} + +h2() { + printf '\e[1;33m==>\e[37;1m %s\e[0m\n' "$*" +} + + +export DEBIAN_FRONTEND=noninteractive + +# !!!!!!!!!!!!!!! +# Wyłącz swap!! +# swapoff -a +# sed -i -E 's,^([^#]+\sswap\s.+),#\1,' /etc/fstab + +h1 "Debug networking and system uuid" +ip addr +cat /sys/class/dmi/id/product_uuid + +h1 "Prepare base system" +h2 "Install applications and utils" +# update the package cache. +apt-get update +apt-get install -y --no-install-recommends jq curl bash-completion vim tcpdump traceroute iptables + +h2 "Configure vim" +cat >/etc/vim/vimrc.local <<'EOF' +syntax on +set background=dark +set esckeys +set ruler +set laststatus=2 +set nobackup +EOF + +h2 "Configure shell" +cat >/etc/profile.d/login.sh <<'EOF' +[[ "$-" != *i* ]] && return +export EDITOR=vim +export PAGER=less +alias l='ls -lF --color' +alias ll='l -a' +alias h='history 25' +alias j='jobs -l' +EOF + +cat >/etc/inputrc <<'EOF' +set input-meta on +set output-meta on +set show-all-if-ambiguous on +set completion-ignore-case on +"\e[A": history-search-backward +"\e[B": history-search-forward +"\eOD": backward-word +"\eOC": forward-word +EOF + diff --git a/provision/etcdctl.sh b/provision/etcdctl.sh new file mode 100644 index 0000000..e12d667 --- /dev/null +++ b/provision/etcdctl.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -euo pipefail + +# +# Helper functions +# +declare -i term_width=80 + +h1() { + declare border padding text + border='\e[1;34m'"$(printf '=%.0s' $(seq 1 "$term_width"))"'\e[0m' + padding="$(printf ' %.0s' $(seq 1 $(((term_width - $(wc -m <<<"$*")) / 2))))" + text="\\e[1m$*\\e[0m" + echo -e "$border" + echo -e "${padding}${text}${padding}" + echo -e "$border" +} + +h2() { + printf '\e[1;33m==>\e[37;1m %s\e[0m\n' "$*" +} + +etcd_version="${1:-v3.4.16}"; shift || true + +h1 "Install etcd" +h2 "Version: ${etcd_version}" + +# install the binaries. +url="https://github.com/etcd-io/etcd/releases/download/$etcd_version/etcd-$etcd_version-linux-amd64.tar.gz" +filename="$(basename "$url")" +wget -q "$url" +rm -rf etcd && mkdir etcd +tar xf "$filename" --strip-components 1 -C etcd +install etcd/etcdctl /usr/local/bin +rm -rf "$filename" etcd + +h2 "Configure envs to access etcd" +cat >/etc/profile.d/etcdctl.sh <<'EOF' +export ETCDCTL_CACERT=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt +export ETCDCTL_CERT=/var/lib/rancher/rke2/server/tls/etcd/server-client.crt +export ETCDCTL_KEY=/var/lib/rancher/rke2/server/tls/etcd/server-client.key +EOF diff --git a/provision/example-app.sh b/provision/example-app.sh new file mode 100644 index 0000000..4e4e4e1 --- /dev/null +++ b/provision/example-app.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -euo pipefail + +# +# Helper functions +# +declare -i term_width=80 + +h1() { + declare border padding text + border='\e[1;34m'"$(printf '=%.0s' $(seq 1 "$term_width"))"'\e[0m' + padding="$(printf ' %.0s' $(seq 1 $(((term_width - $(wc -m <<<"$*")) / 2))))" + text="\\e[1m$*\\e[0m" + echo -e "$border" + echo -e "${padding}${text}${padding}" + echo -e "$border" +} + +h2() { + printf '\e[1;33m==>\e[37;1m %s\e[0m\n' "$*" +} + + +domain="$(hostname --domain)" + + +h1 "Deploy example app" +# deploy. +kubectl apply -f - <\e[37;1m %s\e[0m\n' "$*" +} + +k9s_version="${1:-v0.24.15}"; shift || true + +h1 "Install k9s" +h2 "Version: ${k9s_version}" + +# download and install. +wget -qO- "https://github.com/derailed/k9s/releases/download/$k9s_version/k9s_Linux_x86_64.tar.gz" \ + | tar xzf - k9s +install -m 755 k9s /usr/local/bin/ +rm k9s + +# try it. +k9s version diff --git a/provision/rke2-agent.sh b/provision/rke2-agent.sh new file mode 100644 index 0000000..98133ea --- /dev/null +++ b/provision/rke2-agent.sh @@ -0,0 +1,104 @@ +#!/bin/bash +set -euo pipefail + +# +# Helper functions +# +declare -i term_width=80 + +h1() { + declare border padding text + border='\e[1;34m'"$(printf '=%.0s' $(seq 1 "$term_width"))"'\e[0m' + padding="$(printf ' %.0s' $(seq 1 $(((term_width - $(wc -m <<<"$*")) / 2))))" + text="\\e[1m$*\\e[0m" + echo -e "$border" + echo -e "${padding}${text}${padding}" + echo -e "$border" +} + +h2() { + printf '\e[1;33m==>\e[37;1m %s\e[0m\n' "$*" +} + + +rke2_channel="$1"; shift +rke2_version="$1"; shift +rke2_server_url="$1"; shift +ip_address="$1"; shift + +cat >/etc/motd <<'EOF' + _ ____ _ + _ __| | _____|___ \ __ _ __ _ ___ _ __ | |_ +| '__| |/ / _ \ __) | / _` |/ _` |/ _ \ '_ \| __| +| | | < __// __/ | (_| | (_| | __/ | | | |_ +|_| |_|\_\___|_____(_)__,_|\__, |\___|_| |_|\__| + |___/ +EOF + +h1 "Install rke2 agent" +h2 "Version: ${rke2_version}" +h2 "Server: ${rke2_server_url}" + +# install rke2 agent. +# see https://docs.rke2.io/install/install_options/install_options/ +# see https://docs.rke2.io/install/install_options/linux_agent_config/ +install -d -m 700 /etc/rancher/rke2 +install /dev/null -m 600 /etc/rancher/rke2/config.yaml +cat >>/etc/rancher/rke2/config.yaml </etc/profile.d/01-rke2.sh <<'EOF' +export CONTAINERD_ADDRESS=/run/k3s/containerd/containerd.sock +export CONTAINERD_NAMESPACE=k8s.io +export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml +EOF +source /etc/profile.d/01-rke2.sh + +# NB do not try to use kubectl on a agent node, as kubectl does not work on a +# agent node without a proper kubectl configuration (which you could copy +# from the server, but we do not do it here). + +# install the bash completion scripts. +h2 "Bash configure" +crictl completion bash >/usr/share/bash-completion/completions/crictl +kubectl completion bash >/usr/share/bash-completion/completions/kubectl + +# list runnnig pods. +crictl pods + +# list running containers. +crictl ps +ctr containers ls + +# show listening ports. +ss -n --tcp --listening --processes + +# show network routes. +ip route + +# show memory info. +free + +# show versions. +crictl version +ctr version diff --git a/provision/rke2-server.sh b/provision/rke2-server.sh new file mode 100644 index 0000000..e767089 --- /dev/null +++ b/provision/rke2-server.sh @@ -0,0 +1,223 @@ +#!/bin/bash +set -euo pipefail +# +# Helper functions +# +declare -i term_width=80 + +h1() { + declare border padding text + border='\e[1;34m'"$(printf '=%.0s' $(seq 1 "$term_width"))"'\e[0m' + padding="$(printf ' %.0s' $(seq 1 $(((term_width - $(wc -m <<<"$*")) / 2))))" + text="\\e[1m$*\\e[0m" + echo -e "$border" + echo -e "${padding}${text}${padding}" + echo -e "$border" +} + +h2() { + printf '\e[1;33m==>\e[37;1m %s\e[0m\n' "$*" +} + +rke2_command="$1"; shift +rke2_channel="${1:-latest}"; shift +rke2_version="${1:-v1.21.5+rke2r1}"; shift +ip_address="$1"; shift +krew_version="${1:-v0.4.1}"; shift || true # NB see https://github.com/kubernetes-sigs/krew +fqdn="$(hostname --fqdn)" +rke2_url="https://server.$(hostname --domain):9345" + +h1 "Install rke2 server" +h2 "Version: ${rke2_version}" +h2 "Server url: ${rke2_url}" + +cat >/etc/motd <<'EOF' + _ ____ + _ __| | _____|___ \ ___ ___ _ ____ _____ _ __ +| '__| |/ / _ \ __) | / __|/ _ \ '__\ \ / / _ \ '__| +| | | < __// __/ _\__ \ __/ | \ V / __/ | +|_| |_|\_\___|_____(_)___/\___|_| \_/ \___|_| + +EOF + +# configure the rke2 server. +# see https://docs.rke2.io/install/install_options/install_options/ +# see https://docs.rke2.io/install/install_options/server_config/ +install -d -m 700 /etc/rancher/rke2 +install /dev/null -m 600 /etc/rancher/rke2/config.yaml +if [ "$rke2_command" != 'cluster-init' ]; then + cat >>/etc/rancher/rke2/config.yaml <>/etc/rancher/rke2/config.yaml </etc/profile.d/01-rke2.sh <<'EOF' +export CONTAINERD_ADDRESS=/run/k3s/containerd/containerd.sock +export CONTAINERD_NAMESPACE=k8s.io +export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +EOF +source /etc/profile.d/01-rke2.sh + +# wait for this node to be Ready. +# e.g. server Ready control-plane,etcd,master 3m v1.21.5+rke2r1 +$SHELL -c 'node_name=$(hostname); echo "waiting for node $node_name to be ready..."; while [ -z "$(kubectl get nodes $node_name | grep -E "$node_name\s+Ready\s+")" ]; do sleep 3; done; echo "node ready!"' + +# wait for the kube-dns pod to be Running. +# e.g. rke2-coredns-rke2-coredns-7bb4f446c-jksvq 1/1 Running 0 33m +$SHELL -c 'while [ -z "$(kubectl get pods --selector k8s-app=kube-dns --namespace kube-system | grep -E "\s+Running\s+")" ]; do sleep 3; done' + +# save the node-token in the host. +# NB do not create a token yourself as a simple hex random string, as that will +# not include the Cluster CA which means the joining nodes will not +# verify the server certificate. rke2 warns about this as: +# Cluster CA certificate is not trusted by the host CA bundle, but the +# token does not include a CA hash. Use the full token from the server's +# node-token file to enable Cluster CA validation +if [ "$rke2_command" == 'cluster-init' ]; then + install -d /vagrant/tmp + cp /var/lib/rancher/rke2/server/node-token /vagrant/tmp/node-token +fi + +# install the krew kubectl package manager. +echo "installing the krew $krew_version kubectl package manager..." +apt-get install -y --no-install-recommends git +wget -qO- "https://github.com/kubernetes-sigs/krew/releases/download/$krew_version/krew.tar.gz" | tar xzf - ./krew-linux_amd64 +wget -q "https://github.com/kubernetes-sigs/krew/releases/download/$krew_version/krew.yaml" +./krew-linux_amd64 install --manifest=krew.yaml +rm krew-linux_amd64 +cat >/etc/profile.d/krew.sh <<'EOF' +export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH" +EOF +source /etc/profile.d/krew.sh +kubectl krew version + +# install the bash completion scripts. +crictl completion bash >/usr/share/bash-completion/completions/crictl +kubectl completion bash >/usr/share/bash-completion/completions/kubectl + +# save kubeconfig in the host. +if [ "$rke2_command" == 'cluster-init' ]; then + mkdir -p /vagrant/tmp + python3 - <