🏷️ Manage the lifecycle of Kubernetes clusters

Allow control-plane node to schedule non-system pod

ᐅ kb describe no | grep Taints
Taints:             node-role.kubernetes.io/control-plane:NoSchedule
Taints:             <none>
ᐅ kb taint node cp-01 node-role.kubernetes.io/control-plane-
node/cp-01 untainted
ᐅ kb describe no | grep Taints
Taints:             <none>
Taints:             <none>

ᐅ kb get no cp-01 -o jsonpath='{.spec.taints}' | jq '.[]'

Revert.

ᐅ kb taint node cp-01 node-role.kubernetes.io/control-plane:NoSchedule
node/cp-01 tainted

Backup etcd database

Find data directory.

ᐅ sudo grep data-dir /etc/kubernetes/manifests/etcd.yaml
    - --data-dir=/var/lib/etcd

Find and connect etcd pod.

ᐅ kb get po -l component=etcd -n kube-system
ᐅ kb exec -it etcd-cp-01 -n kube-system -- sh
sh-5.2# alias etcdct='etcdctl --cacert="/etc/kubernetes/pki/etcd/ca.crt" --cert="/etc/kubernetes/pki/etcd/server.crt" --key="/etc/kubernetes/pki/etcd/server.key"'
sh-5.2# etcdct member list -w table
+------------------+---------+------------+----------------------------+----------------------------+------------+
|        ID        | STATUS  |    NAME    |         PEER ADDRS         |        CLIENT ADDRS        | IS LEARNER |
+------------------+---------+------------+----------------------------+----------------------------+------------+
| aa5b4d1a0319ca2d | started | cp-01 | https://192.168.94.73:2380 | https://192.168.94.73:2379 |      false |
+------------------+---------+------------+----------------------------+----------------------------+------------+

sh-5.2# etcdct endpoint status -w table
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|    ENDPOINT    |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 127.0.0.1:2379 | aa5b4d1a0319ca2d |  3.5.21 |   24 MB |      true |      false |         2 |      15133 |              15133 |        |
+----------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
sh-5.2# etcdct snapshot save /var/lib/etcd/snapshot.db

Create backup directory.

ᐅ sudo cp -p /var/lib/etcd/snapshot.db $HOME/backup/snapshot_$(date +%F).db
ᐅ sudo cp $HOME/kubeadm-config.yaml backup
ᐅ k -n kube-system get cm kubeadm-config -o yaml | \
yq -r '.data.ClusterConfiguration' > kubeadm-config-backup.yaml
ᐅ sudo cp -rp /etc/kubernetes/pki/etcd backup
ᐅ sudo cp -rp /etc/kubernetes/admin.conf backup/kubeconfig

backup
├── etcd
│   ├── ca.crt
│   ├── ca.key
│   ├── healthcheck-client.crt
│   ├── healthcheck-client.key
│   ├── peer.crt
│   ├── peer.key
│   ├── server.crt
│   └── server.key
├── kubeadm-config.yaml
├── kubeconfig
└── snapshot_2025-10-07.db

Upgrade the cluster

On a control-plane node.

# Change k8s minor version in source list file
ᐅ sudo sed -i 's/33/34/g' /etc/apt/sources.list.d/kubernetes.list
# Update and upgrade kubeadm
ᐅ sudo apt update
ᐅ sudo apt-mark unhold kubeadm
ᐅ sudo apt-cache madison kubeadm
ᐅ sudo apt install kubeadm=1.34.1-1.1 -y
ᐅ sudo apt-mark hold kubeadm
ᐅ sudo kubeadm version
# Upgrade the cluster with upgraded kubeadm
ᐅ kb drain cp-01 --ignore-daemonsets
ᐅ sudo kubeadm upgrade plan
ᐅ sudo kubeadm upgrade apply v1.34.1
# Upgrade kubelet and kubectl, also restart kubelet service
ᐅ sudo apt install kubelet=1.34.1-1.1 kubectl=1.34.1-1.1 -y
ᐅ sudo apt-mark hold kubelet kubectl
ᐅ sudo systemctl restart kubelet.service
# Check kubelet logs and uncordon the upgraded node
ᐅ sudo journalctl -u kubelet.service -f
ᐅ kb uncordon cp-01

On other node, same as control-plane but just sudo kubeadm upgrade node.

kubectl cache

Get cluster entry.

ᐅ kb config view -o jsonpath='{.clusters[0].cluster.server}'
https://cp-01:6443

See cluster API groups and their objects in the kubectl cache.

ᐅ ls -tr1 .kube/cache/discovery/kube_cp_01_6443
flowcontrol.apiserver.k8s.io
certificates.k8s.io
authentication.k8s.io
networking.k8s.io
authorization.k8s.io
apps
storage.k8s.io
scheduling.k8s.io
rbac.authorization.k8s.io
node.k8s.io
discovery.k8s.io
apiextensions.k8s.io
admissionregistration.k8s.io
policy
events.k8s.io
coordination.k8s.io
batch
autoscaling
apiregistration.k8s.io
cilium.io
metrics.k8s.io
resource.k8s.io
servergroups.json
v1

Configure APIs access

with certs extracted from kubeconfig file

Extract certificates then access api-server with curl.

# Extract key and certs, also get the cluster url from kubeconfig
ᐅ awk '/client-key-data/{print $2}' .kube/config | base64 -d > key.pem
ᐅ awk '/client-certificate-data/{print $2}' .kube/config | base64 -d > cert.pem
ᐅ awk '/certificate-authority-data/{print $2}' .kube/config | base64 -d > cacert.pem
ᐅ k8s_url=$(kb config view -o jsonpath='{.clusters[0].cluster.server}')
# Request some resources
ᐅ curl --cert cert.pem --key key.pem --cacert cacert.pem $k8s_url/api/v1/pods
ᐅ curl --cert cert.pem --key key.pem --cacert cacert.pem $k8s_url/api/v1/endpoints
ᐅ curl --cert cert.pem --key key.pem --cacert cacert.pem $k8s_url/api/v1/nodes
# Create a pod
ᐅ curl --cert cert.pem --key key.pem --cacert cacert.pem $k8s_url/api/v1/namespaces/default/pods \
-XPOST -H 'Content-Type: application/json' -d@curlpod.json

curlpod.json

{
    "kind": "Pod",
    "apiVersion": "v1",
    "metadata":{
        "name": "curlpod",
        "namespace": "default",
        "labels": {
            "name": "examplepod"
        }
    },
    "spec": {
        "containers": [{
            "name": "nginx",
            "image": "nginx",
            "ports": [{"containerPort": 80}]
        }]
    }
}

ᐅ kb get po
NAME      READY   STATUS    RESTARTS        AGE
curlpod   1/1     Running   1 (6h36m ago)   23h
ᐅ b delete pods curlpod

with token

Create a token

ᐅ k8s_url=$(kb config view -o jsonpath='{.clusters[0].cluster.server}')
ᐅ token="Authorization: Bearer $(kb create token default)"
ᐅ curl --header $token $k8s_url/apis -k

Check the APIs groups.

ᐅ curl -s --header $token $k8s_url/apis -k | jq '.groups[].name'
"apiregistration.k8s.io"
"apps"
"events.k8s.io"
"authentication.k8s.io"
"authorization.k8s.io"
"autoscaling"
"batch"
"certificates.k8s.io"
"networking.k8s.io"
"policy"
"rbac.authorization.k8s.io"
"storage.k8s.io"
"admissionregistration.k8s.io"
"apiextensions.k8s.io"
"scheduling.k8s.io"
"coordination.k8s.io"
"node.k8s.io"
"discovery.k8s.io"
"resource.k8s.io"
"flowcontrol.apiserver.k8s.io"
"cilium.io"
"metrics.k8s.io"

Attention

Connection with token is not a full APIs access.

ᐅ curl -s --header $token $k8s_url/api/v1/namespaces -k
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "namespaces is forbidden: User \"system:serviceaccount:default:default\" cannot list resource \"namespaces\" in API group \"\" at the cluster scope",
  "reason": "Forbidden",
  "details": {
    "kind": "namespaces"
  },
  "code": 403
}

with kube proxy

Create a kube proxy on a node.

ᐅ kb proxy --api-prefix=/
Starting to serve on 127.0.0.1:8001

Access the APIs via a local proxy (full permissions…as long as your kubeconfig has full access).

ᐅ curl -sk http://127.0.0.1:8001/api/v1/namespaces/ | jq '.items[].metadata.name'
"cilium-secrets"
"default"
"kube-node-lease"
"kube-public"
"kube-system"
"low"