k8s 集群部署流程

hostnamectl set-hostname k8s-master-01-C-201
...

cat >> /etc/hosts <<EOF
192.168.2.200 api-server
192.168.2.201 k8s-master-01-C-201
192.168.2.202 k8s-master-02-R-202
192.168.2.203 k8s-master-03-U-203
192.168.2.211 k8s-node-01-C-211
192.168.2.212 k8s-node-02-C-212
192.168.2.213 k8s-node-03-C-213
192.168.2.214 k8s-node-04-R-214
192.168.2.215 k8s-node-05-R-215
192.168.2.216 k8s-node-06-R-216
192.168.2.217 k8s-node-07-U-217
192.168.2.218 k8s-node-08-U-218
192.168.2.219 k8s-node-09-U-219
EOF

# 修改操作系统的 /etc/selinux/config 或 /etc/sysconfig/selinux 文件中的 SELINUX=enforcing 为 SELINUX=disabled
# /etc/selinux/config中的配置优先级高于/etc/sysconfig/selinux。如果两个文件中存在相同的配置项，系统会优先采用/etc/selinux/config中的设置
sed -i 's#enforcing#disabled#g' /etc/selinux/config

# 临时关闭 SELinux
setenforce 0

# 在防火墙配置 k8s 各个组件的相互通信的端口号

# 在安全的网络环境下可以关闭防火墙
systemctl disable firewall
systemctl stop firewall

# 临时关闭
swapoff -a

# 永久关闭
cp /etc/fstab /etc/fstab_bak
sed -i '/swap/d' /etc/fstab


# ubuntu系统除上述步骤还需要以下步骤
systemctl list-unit-files --type=swap 

UNIT FILE     STATE     VENDOR PRESET
dev-sda3.swap generated -            

1 unit files listed.

systemctl mask dev-sda3.swap
reboot

# 时间同步方案：采用 k8s-master-01-C-201 为内网集群 ntp 服务端，与公网 ntp 服务器同步时间。其他节点与 k8s-master-01-C-201 同步时间
# ps: 如果遇到服务器时间不同步无法安装软件，可以先手动设置时间
date -s "2024-11-30 12:17:00"

# 安装 chrony
yum -y install chrony
# ubuntu 系统安装，配置文件路径 /etc/chrony/chrony.conf
apt -y install chrony


# 修改配置文件
## centos：
mv /etc/chrony.conf /etc/chrony.conf.bak
cat > /etc/chrony.conf << EOF
server ntp1.aliyun.com iburst minpoll 4 maxpoll 10
server ntp2.aliyun.com iburst minpoll 4 maxpoll 10
server ntp3.aliyun.com iburst minpoll 4 maxpoll 10
server ntp4.aliyun.com iburst minpoll 4 maxpoll 10
server ntp5.aliyun.com iburst minpoll 4 maxpoll 10
server ntp6.aliyun.com iburst minpoll 4 maxpoll 10
server ntp7.aliyun.com iburst minpoll 4 maxpoll 10
driftfile /var/lib/chrony/drift
makestep 10 3
rtcsync
allow 0.0.0.0/0
local stratum 10
keyfile /etc/chrony.keys
logdir /var/log/chrony
stratumweight 0.05
noclientlog
logchange 0.5
EOF

## ubuntu：
mv /etc/chrony/chrony.conf /etc/chrony/chrony.conf.bak
cat > /etc/chrony/chrony.conf << EOF
server ntp1.aliyun.com iburst minpoll 4 maxpoll 10
server ntp2.aliyun.com iburst minpoll 4 maxpoll 10
server ntp3.aliyun.com iburst minpoll 4 maxpoll 10
server ntp4.aliyun.com iburst minpoll 4 maxpoll 10
server ntp5.aliyun.com iburst minpoll 4 maxpoll 10
server ntp6.aliyun.com iburst minpoll 4 maxpoll 10
server ntp7.aliyun.com iburst minpoll 4 maxpoll 10
driftfile /var/lib/chrony/drift
makestep 10 3
rtcsync
allow 0.0.0.0/0
local stratum 10
keyfile /etc/chrony/chrony.keys
logdir /var/log/chrony
stratumweight 0.05
noclientlog
logchange 0.5
EOF

# 启动 chronyd 服务
systemctl restart chronyd.service # 最好重启，这样无论原来是否启动都可以重新加载配置
systemctl enable chronyd.service  # 设置开机自启
systemctl status chronyd.service



# 安装 chrony 客户端：在需要与外部同步时间的机器上安装，启动后会自动与你指定的服务端同步时间
# 安装 chrony
yum -y install chrony

# 需改客户端配置文件
## centos：
mv /etc/chrony.conf /etc/chrony.conf.bak
cat > /etc/chrony.conf << EOF
# server 服务端的ip地址或可解析的主机名 iburst
server k8s-master-01-C-201 iburst
driftfile /var/lib/chrony/drift
makestep 10 3
rtcsync
local stratum 10
keyfile /etc/chrony.key
logdir /var/log/chrony
stratumweight 0.05
noclientlog
logchange 0.5
EOF

## ubuntu：
mv /etc/chrony/chrony.conf /etc/chrony/chrony.conf.bak
cat > /etc/chrony/chrony.conf << EOF
# server 服务端的ip地址或可解析的主机名 iburst
server k8s-master-01-C-201 iburst
driftfile /var/lib/chrony/drift
makestep 10 3
rtcsync
local stratum 10
keyfile /etc/chrony/chrony.key
logdir /var/log/chrony
stratumweight 0.05
noclientlog
logchange 0.5
EOF

# 启动 chronyd 服务
systemctl restart chronyd.service
systemctl enable chronyd.service
systemctl status chronyd.service

# 验证
chronyc sources -v

sed -ri 's@^#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config 
sed -ri 's#^GSSAPIAuthentication yes#GSSAPIAuthentication no#g' /etc/ssh/sshd_config 
grep ^UseDNS /etc/ssh/sshd_config 
grep ^GSSAPIAuthentication /etc/ssh/sshd_config

systemctl restart sshd

# 生成 ssh 秘钥对，-f 指定生成路径和文件名，-N 设置秘钥对密码，这里“”表示密码为空
ssh-keygen -f ~/.ssh/id_rsa -N ""

# 推送公钥到对应主机
ssh-copy-id -i root@k8s-master-01-C-201
...

# centos：
## centos 7 已经停止更新维护，需要更换第三方 yum 源
## 安装阿里的 base 与 epel 源
curl -s -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo 
curl -s -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
## 清理旧缓存，并重新缓存
yum clean all 
yum makecache
yum update -y



# ubuntu:
apt update -y
# 完全升级系统（包括内核更新）
apt full-upgrade

sudo yum -y install expect wget jq psmisc vim net-tools telnet yum-utils devicemapper-persistent-data lvm2 git ntpdate chrony bind-utils rsync unzip git


sudo apt -y install expect wget jq psmisc vim net-tools telnet software-properties-common lvm2 git ntpdate chrony dnsutils rsync unzip

# centos 7 系统内核版本较低
# 系统内核最好 4.4 版本及以上
wget https://elrepo.org/linux/kernel/el7/x86_64/RPMS/kernel-lt-5.4.274-1.el7.elrepo.x86_64.rpm
wget https://elrepo.org/linux/kernel/el7/x86_64/RPMS/kernel-lt-devel-5.4.274-1.el7.elrepo.x86_64.rpm

# 将下载的内核传送到需要更新内核的主机上的 /opt 目录
for i in hosts_list ; do scp kernel-lt-* $i:/opt; done

# 安装
yum localinstall -y /opt/kernel-lt*
# 调到默认启动
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg  
# 查看当前默认启动的内核
grubby --default-kernel
 
# 重启系统
reboot

# “*” 表示适用于所有用户
cat > /etc/security/limits.d/k8s.conf <<'EOF' 
* soft nofile 1048576
* hard nofile 1048576
EOF 

# 查看当前用户的软限制和硬限制的文件描述符数
ulimit -Sn 
ulimit -Hn

cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp.keepaliv.probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp.max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp.max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.top_timestamps = 0
net.core.somaxconn = 16384
EOF

# 立即生效
sysctl --system

# 必须配置此项，否则 kubeadm init 时会直接失败

# centos：
# 这两行命令用于手动加载内核模块
modprobe br_netfilter
modprobe ip_conntrack

# 这个循环的目的是在系统启动时自动执行/etc/sysconfig/modules/目录下的所有模块加载脚本
cat >/etc/rc.sysinit<<EOF
#!/bin/bash
 
for file in /etc/sysconfig/modules/*.modules ; do
[ -x $file ] && $file
done
EOF

echo "modprobe br_netfilter" >/etc/sysconfig/modules/br_netfilter.modules
echo "modprobe ip_conntrack" >/etc/sysconfig/modules/ip_conntrack.modules
chmod 755 /etc/sysconfig/modules/br_netfilter.modules
chmod 755 /etc/sysconfig/modules/ip_conntrack.modules



# RockyLinux：
# 可以通过编辑/etc/modules-load.d/目录中的配置文件来实现在系统启动时自动加载内核模块
cat >/etc/modules-load.d/custom-modules.conf<<EOF
br_netfilter
ip_conntrack
EOF
# 重启系统



# ubuntu：
# 可以通过编辑/etc/modules文件来实现在系统启动时自动加载内核模块
cat >> /etc/modules <<EOF
br_netfilter
ip_conntrack
EOF
# 重启系统


# 查看内核模块是否加载成功
lsmod | grep br_netfilter

# centos：
# 安装 ipvsadm 等相关工具
yum -y install ipvsadm ipset sysstat conntrack libseccomp 

# 配置加载
cat > /etc/sysconfig/modules/ipvs.modules << "EOF"
#!/bin/bash 
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr 
ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack" 
for kernel_module in ${ipvs_modules}; 
do 
    /sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1 
    if [ $? -eq 0 ]; then 
        /sbin/modprobe ${kernel_module} 
    fi 
done 
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs



# RockyLinux：
yum -y install ipvsadm ipset sysstat conntrack libseccomp
cat > /etc/modules-load.d/ipvs.conf << "EOF"
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
nf_conntrack 
EOF



# ubuntu：
apt install -y ipvsadm ipset sysstat conntrack libseccomp2
cat >> /etc/modules <<EOF
# IPVS 相关模块
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
nf_conntrack
EOF

# centos7 默认的 libseccomp 的版本为 2.3.1 ，不满足 containerd 的需求，需要下载2.4以上的版本
# 如果不升级 libseccomp 的话，启动容器会报错
rpm -qa | grep libseccomp # 查询原来的版本
dpkg -l libseccomp2 # 在 ubuntu 系统中查询版本命令

rpm -e libseccomp-2.3.1-4.el7.x86_64 --nodeps #卸载原来的版本
wget https://mirrors.aliyun.com/centos/8/BaseOS/x86_64/os/Packages/libseccomp-2.5.1-1.el8.x86_64.rpm
rpm -ivh libseccomp-2.5.1-1.el8.x86_64.rpm

# 如果有，先卸载之前的旧版本 docker
yum remove docker docker-ce containerd docker-common docker-selinux docker-engine -y 

# 准备 repo
cd /etc/yum.repos.d/
wget http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
wget http://mirrors.aliyun.com/docker-ce/linux/rhel/docker-ce.repo
# 安装
yum install containerd* -y



# ubuntu:
apt install containerd* -y

mkdir -pv /etc/containerd 
# 为 containerd 生成配置文件
containerd config default > /etc/containerd/config.toml  

# 替换默认 pause 镜像地址
# 这一步非常非常非常非常重要，国内的镜像地址可能导致下载失败，最终 kubeadm 安装失败
grep sandbox_image /etc/containerd/config.toml
# 替换镜像地址
sed -i 's/registry.k8s.io/registry.cn-hangzhou.aliyuncs.com\/google_containers/' /etc/containerd/config.toml 
grep sandbox_image /etc/containerd/config.toml

# 配置 systemd 作为容器的 cgroup driver
grep SystemdCgroup /etc/containerd/config.toml
sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/' /etc/containerd/config.toml
grep SystemdCgroup /etc/containerd/config.toml

# 配置加速器（必须配置，否则后续安装cni网络插件时无法从docker.io里下载镜像）
# 参考：
https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry
configuration
# 添加 config_path = "/etc/containerd/certs.d"
sed -i 's/config_path\ =.*/config_path = \"\/etc\/containerd\/certs.d\"/g' /etc/containerd/config.toml

# docker hub 镜像加速
mkdir -p /etc/containerd/certs.d/docker.io

cat > /etc/containerd/certs.d/docker.io/hosts.toml << 'EOF' 
server = "https://docker.io"
[host."https://docker.m.daocloud.io"]
capabilities = ["pull", "resolve"]

[host."https://docker.udayun.com"]
capabilities = ["pull", "resolve"]

[host."https://noohub.ru"]
capabilities = ["pull", "resolve"]

[host."https://huecker.io"]
capabilities = ["pull", "resolve"]

[host."https://dockerhub.timeweb.cloud"]
capabilities = ["pull", "resolve"]
EOF


# 配置 containerd 开机自启动
# 启动 containerd 服务并配置开机自启动
systemctl daemon-reload && systemctl restart containerd
systemctl enable --now containerd
# 查看 containerd 状态
systemctl status containerd
# 查看containerd的版本
ctr version

# 添加 repo 源
cat > /etc/yum.repos.d/nginx.repo << "EOF"
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
EOF

# 安装
yum install nginx -y
apt install nginx -y

# 配置
cat >> /etc/nginx/nginx.conf <<'EOF'

stream {
    upstream kube-servers {
        hash $remote_addr consistent;
        server k8s-master-01-C-201:6443 weight=5 max_fails=1 fail_timeout=3s;
        server k8s-master-02-R-202:6443 weight=5 max_fails=1 fail_timeout=3s;
        server k8s-master-03-U-203:6443 weight=5 max_fails=1 fail_timeout=3s;
    }
    server {
        listen 8443 reuseport;  # 监听8443端口
        proxy_connect_timeout 3s;
        proxy_timeout 3000s;
        proxy_pass kube-servers;
    }
}
EOF

# 启动
systemctl restart nginx 
systemctl enable nginx 
systemctl status nginx 

# 安装
yum  -y install  keepalived
apt  -y install  keepalived

# 修改配置
mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
# 各个 master节点修改 router_id 和 mcast_src_ip 的值即可
# ==================================> k8s-master-01-C-201
cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived

global_defs {
    router_id 192.168.2.201  
}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 8443" 
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 100
    priority 100
    advert_int 1
    mcast_src_ip 192.168.2.201
    # nopreempt # 这行注释掉，否则即使一个具有更高优先级的备份节点出现，当前的 MASTER 也不会被抢占，直至 MASTER 失效。
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.2.200
    }
}
EOF

# ==================================> k8s-master-02-R-202
cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived

global_defs {
    router_id 192.168.2.202 
}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 8443" 
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens160
    virtual_router_id 100
    priority 100
    advert_int 1
    mcast_src_ip 192.168.2.202
    # nopreempt # 这行注释掉，否则即使一个具有更高优先级的备份节点出现，当前的 MASTER 也不会被抢占，直至 MASTER 失效。
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.2.200
    }
}
EOF

# ==================================> k8s-master-03-U-203
cat > /etc/keepalived/keepalived.conf <<EOF
! Configuration File for keepalived

global_defs {
    router_id 192.168.2.203
}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 8443" 
    interval 2
    weight -20
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 100
    priority 100
    advert_int 1
    mcast_src_ip 192.168.2.203
    # nopreempt # 这行注释掉，否则即使一个具有更高优先级的备份节点出现，当前的 MASTER 也不会被抢占，直至 MASTER 失效。
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.2.200
    }
}
EOF





# 所有 master 节点上创建健康检查脚本
cat > /etc/keepalived/check_port.sh << 'EOF'
#!/bin/bash

# 设置环境变量，确保所有必要的命令路径正确
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
    PORT_PROCESS=$(ss -lt | grep ":$CHK_PORT" | wc -l)
    if [ $PORT_PROCESS -eq 0 ];then
        echo "Port $CHK_PORT Is Not Used,End."
        exit 1
    fi
else
    echo "Check Port Cant Be Empty!"
fi
EOF

chmod +x /etc/keepalived/check_port.sh



# 启动
systemctl restart keepalived            
systemctl enable keepalived         
systemctl status keepalived


# 在一个 master 节点可以查看到 vip
ip addr show dev ens33
# 测试
# 停掉拥有 VIP 的节点的 Nginx，VIP 转移到其他节点

# 动态查看 keepalived 日志
 journalctl -u keepalived -f

# 阿里云文档参考：https://developer.aliyun.com/mirror/kubernetes/

# centos:
# 配置镜像源
cat > /etc/yum.repos.d/kubernetes.repo << "EOF" 
[kubernetes] 
name=Kubernetes 
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/ 
enabled=1 
gpgcheck=1 
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/repodata/repomd.xml.key 
EOF

# 安装（所有节点）
yum install -y kubelet-1.30* kubeadm-1.30* kubectl-1.30*
systemctl enable kubelet && systemctl start kubelet && systemctl status kubelet


# ubuntu:
apt-get install -y apt-transport-https
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
apt-get update
apt-get install -y kubelet kubeadm kubectl

# 初始化 master节点(仅在 k8s-master-01-C-201节点上执行)： 
# 可以 kubeadm config images list 查看镜像列表


# 先生成配置文件，编辑修改后，再部署
kubeadm config print init-defaults > kubeadm.yaml

# 配置文件示例
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 0.0.0.0  # 统一监听在0.0.0.0即可
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: k8s-master-01-C-201   # 当前的主机名
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd   # 内部 etcd 服务就直接指定本地文件夹就行
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers  # 换成阿里云镜像仓库地址
kind: ClusterConfiguration
kubernetesVersion: 1.30.7  # 指定 k8s 版本
controlPlaneEndpoint: "api-server:8443"  # 指定你的vip地址192.168.2.200与负载均衡暴漏的端口，建议用主机名
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12  # 指定 Service 网段
  podSubnet: 10.244.0.0/16  # 增加一行，指定pod网段
scheduler: {}
# 在文件最后,挿入以下内容,(复制时,要帯着--):
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs   # 表示kube-proxy代理模式是ipvs，如果不指定ipvs，会默认使用iptables，但是iptables效率低，所以我们生产环境建议开启ipvs，阿里云和华为云托管的K8s，也提供ipvs模式
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

# 部署
kubeadm init --config=kubeadm.yaml --ignore-preflight-errors=SystemVerification --ignore-preflight-errors=Swap

# 部署成功显示结果
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join api-server:8443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:efb37c407e4eaef751d402eed838e44b2defeb4c45e03b6d2151e62ca915e0f7 --control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join api-server:8443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:efb37c407e4eaef751d402eed838e44b2defeb4c45e03b6d2151e62ca915e0f7
	
# 为管理用户添加凭证
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 
kubectl -n kube-system get cm kubeadm-config -o yaml

# 仅在 k8s-master-01-C-201节点上执行
kubeadm init phase upload-certs --upload-certs # 你也可以在安装的时候 kubeadm init 的默认加上选项--upload-certs来上传证书，那这一步就不需要了

# 得到该值，后续添加master需要用到它
[upload-certs] Using certificate key:
c4d33271a56cd44e4df98f32e387fa74a77aaf8ce619782537ab5d822f48ee00

# 所有 master节点加入集群
kubeadm join api-server:8443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:efb37c407e4eaef751d402eed838e44b2defeb4c45e03b6d2151e62ca915e0f7 --control-plane --certificate-key c4d33271a56cd44e4df98f32e387fa74a77aaf8ce619782537ab5d822f48ee00   


# 成功后按照提示执行
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 加入工作节点
kubeadm join api-server:8443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:efb37c407e4eaef751d402eed838e44b2defeb4c45e03b6d2151e62ca915e0f7


# 在 master 节点查看所有节点
kubectl get nodes
# 查看静态 pod
kubectl -n kube-system get pods

# 下载网络插件
wget https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

vim kube-flannel.yml 
apiVersion: v1
data:
  ...
  net-conf.json: |
    {
      "Network": "10.244.0.0/16", # 与--pod-network-cidr保持一致
      "Backend": {
        "Type": "vxlan"
      }
    }
    

# 在 k8s-master-01-C-201上部署
kubectl apply -f kube-flannel.yml 
kubectl -n kube-flannel get pods
kubectl -n kube-flannel get pods -w

# 安装 Bash 自动补全功能
sudo yum install bash-completion* -y
sudo apt install bash-completion* -y

# 生成 kubectl 命令的 Bash 自动补全配置，ubuntu 修改 $HOME/.bash_profile为$HOME/.bprofile
kubectl completion bash > ~/.kube/completion.bash.inc
echo "source '$HOME/.kube/completion.bash.inc'" >> $HOME/.bash_profile
# 立即生效
source $HOME/.bash_profile