[root@k8s-master ~]$ kubectl get node 
Unable to connect to the server: x509: certificate has expired or is not yet valid: current time 2022-09-15T10:27:07+08:00 is after 2022-09-15T02:17:20Z
[root@k8s-master ~]$ kubeadm certs renew all  # 更新证书，更新完后会提示重启kube-apiserver, kube-controller-manager, kube-scheduler and etcd等服务
[root@k8s-master ~]$ docker ps | grep -E 'kube-apiserver|kube-controller-manager|kube-scheduler|etcd'  # 找到提示的容器
[root@k8s-master ~]$ docker restart 497ec4931ad7 fc3ae3eea7d4 c986f6cf68a9 d93b96fc47ee 3ea76fcdfa43 94e2499de017  # 直接重启即可
[root@k8s-master ~]$ echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile  # 将最新的配置更新到master，使其能连接到集群
[root@k8s-master ~]$ source ~/.bash_profile
[root@k8s-master ~]$ cp $HOME/.kube/config $HOME/.kube/config.bak  # 备份
[root@k8s-master ~]$ cp -i /etc/kubernetes/admin.conf $HOME/.kube/config  # 更新kubectl

[root@k8s-master ~]$ wget https://github.com/etcd-io/etcd/releases/download/v3.5.6/etcd-v3.5.6-linux-amd64.tar.gz
[root@k8s-master ~]$ tar -zxvf etcd-v3.5.6-linux-amd64.tar.gz
[root@k8s-master ~]$ cd etcd-v3.5.6-linux-amd64
[root@k8s-master ~]$ cp etcdctl /usr/bin
[root@k8s-master ~]$ vim /usr/local/wlhiot/scripts/backup_etcd.sh  # 编写备份脚本,bash执行即可，备份文件保存在/usr/local/wlhiot/backup/目录
#! /bin/bash
# 配置获取证书的目录
ETCD_PATH=/etc/kubernetes/pki
CACERT=$ETCD_PATH/etcd/ca.crt
CERT=$ETCD_PATH/etcd/server.crt
KEY=$ETCD_PATH/etcd/server.key
# 本机的etcd地址
ENDPOINTS=https://192.168.254.23:2379
ETCDCTL_API=3 etcdctl \
--cacert="${CACERT}" --cert="${CERT}" --key="${KEY}" \
--endpoints=${ENDPOINTS} \
snapshot save /usr/local/wlhiot/backup/etcd-snapshot-`date +%Y%m%d%H%M%S`.db

[root@k8s-master ~]$ kubectl get node
NAME         STATUS   ROLES                  AGE    VERSION
k8s-master   Ready    control-plane,master   446d   v1.22.1
k8s-node01   Ready    <none>                 446d   v1.22.1
k8s-node02   Ready    <none>                 446d   v1.22.1
k8s-node03   Ready    <none>                 446d   v1.22.1
k8s-node04   Ready    <none>                 446d   v1.22.1
k8s-node05   Ready    <none>                 290d   v1.22.1
k8s-node06   Ready    <none>                 289d   v1.22.1
k8s-node07   Ready    <none>                 200d   v1.22.1
[root@k8s-master ~]$ yum list --showduplicates kubeadm  # 查找版本号列表，我当前版本为1.22.1，所以推荐升级到1.23.*的最高版本。虽然列表中只有1.23.10-0，但是实际上1.23版本中最新的是1.23.14(当然，这是我通过kubeadm upgrade plan命令才知道的)
[root@k8s-master ~]$ yum install -y kubeadm-1.23.14-0 --disableexcludes=kubernetes
[root@k8s-master ~]$ kubectl drain k8s-master --ignore-daemonsets  # 驱逐Pod
[root@k8s-master ~]$ kubeadm upgrade plan  # 查看升级计划
[root@k8s-master ~]$ kubeadm upgrade apply v1.23.14-0  # 升级
[root@k8s-master ~]$ kubectl uncordon k8s-master  # 恢复调度
[root@k8s-master ~]$ yum install -y kubectl-1.23.14-0 kubelet-1.23.14-0 --disableexcludes=kubernetes  # 安装kubectl及kubelet
[root@k8s-master ~]$ systemctl daemon-reload
[root@k8s-master ~]$ systemctl restart kubelet
[root@k8s-master ~]$ kubectl get pod -o wide  # 此时k8s-master的状态应该已经Ready(但实际我并没有，报错network plugin is not ready: cni config uninitialized)
[root@k8s-master ~]$ wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
# 该网络配置是由于公司内网网段冲突另外修改，所以在升级的时候不能照搬官网的kube-flannel.yml文件
  net-conf.json: |
    {
      "Network": "10.20.0.0/16",
      "Backend": {
        "Type": "host-gw"
      }
    }
[root@k8s-master ~]$ kubectl apply  -f /usr/local/wlhiot/yaml/kubernetes/basic/kube-flannel.yml  # 重新更新一下flannel网络插件

[root@k8s-master ~]$ kubectl drain [node_name] --ignore-daemonsets
[root@k8s-node03 ~]$ yum install -y kubectl-1.23.14-0 kubelet-1.23.14-0
[root@k8s-node03 ~]$ systemctl daemon-reload
[root@k8s-node03 ~]$ systemctl restart kubelet
[root@k8s-master ~]$ kubectl uncordon [node_name]
[root@k8s-master ~]$ kubectl get node -o wide  # 发现工作节点状态为NoReady
[root@k8s-master ~]$ kubectl describe node k8s-node03  # 检查发现网络插件未就绪
[root@k8s-master ~]$ kubectl delete pod [flannel_pod_name] -n kube-flannel  # 将指定工作节点上的flannel pod删除让它重新生成即可
[root@k8s-master ~]$ kubectl delete pod [coredns-pod_name]  -n kube-system  # 如果已有Prometheus且无法监控集群节点则删除coredns让其自动生成。因为发现Prometheus的自动发现采集类似https://kubernetes.default.svc/api/v1/nodes/k8s-node07/proxy/metrics的URL，通过node-exporter暴露的服务器均正常采集，所以确定是Prometheus server不能解析该URL，而K8s内部解析是通过coredns的，所以删除dns服务器让其自动生成即可。

[root@k8s-master ~]$ kubectl get ns kube-flannel -o json > delete.json
[root@k8s-master ~]$ netstat -tulnp | grep 8080  # 查看8080端口是否被占用
[root@k8s-master ~]$ kubectl proxy --port=8080
[root@k8s-master ~]$ curl http://localhost:8080/api/  # 测试是否有效
[root@k8s-master ~]$ curl -k -H "Content-Type: application/json" -X PUT --data-binary @delete.json http://127.0.0.1:8080/api/v1/namespaces/[namespace_name]/finalize

[root@k8s-master ~]$ kubectl drain [node_name] --force --ignore-daemonsets --delete-local-data  # 驱逐指定Node上的所有Pod,--force表示强制删除，--delete-local-data会删除本地数据，例如挂载至Pod的本地卷，表达形式如下：
volumes:
- name: data
  emptyDir: {}
[root@k8s-master ~]$ kubectl uncordon [node_name]  # 恢复调度