设置hostnamehostnamectl set-hostname k8s-master#编辑hosts文件添加hostsvim/etc/hosts172.21.16.7 k8s-master关闭防火墙systemctl stop firewalld systemctl disable firewalld关闭selinuxsetenforce0sed-is/SELINUXenforcing/SELINUXdisabled/g/etc/selinux/config关闭swapoff分区swapoff-ased-ris/.*swap.*/#//etc/fstab安装ipvsadmyuminstallipvsadm ipset sysstat conntrack libseccomp-ycat/etc/modules-load.d/ipvs.confEOF ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack ip_tables ip_set xt_set ipt_set ipt_rpfilter ipt_REJECT ipip EOFsystemctl restart systemd-modules-load.service配置前置需求#配置前置需求catEOF|sudotee/etc/modules-load.d/k8s.confoverlay br_netfilter EOFsudomodprobe overlaysudomodprobe br_netfilter systemctl restart systemd-modules-load.service# sysctl params required by setup, params persist across rebootscatEOF|sudotee/etc/sysctl.d/k8s.confnet.bridge.bridge-nf-call-iptables 1 net.bridge.bridge-nf-call-ip6tables 1 net.ipv4.ip_forward 1 EOF# Apply sysctl params without rebootsudosysctl--system##保险执行一下命令sysctl-wnet.ipv4.ip_forward1安装 container runtimecontainerdsudodnfinstall-ydnf-plugins-coresudodnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.reposudodnfinstallcontainerd.io#初始化默认配置containerd config default|tee/etc/containerd/config.toml#修改containerd配置更改cgroup, #修改镜像源sudosed-i-Es#^([[:space:]]*sandbox[[:space:]]*).*#\1registry.aliyuncs.com/google_containers/pause:3.10.2#/etc/containerd/config.tomlsudosed-is/SystemdCgroup false/SystemdCgroup true//etc/containerd/config.toml#安装cni ,docker 源安装 containerd 需要手动安装cniwgethttps://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgzmkdir-p/opt/cni/bintarCxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.1.1.tgz systemctl daemon-reload systemctlenable--nowcontainerd#安装crictlVERSIONv1.24.1wgethttps://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-amd64.tar.gzsudotarzxvf crictl-$VERSION-linux-amd64.tar.gz-C/usr/local/binrm-fcrictl-$VERSION-linux-amd64.tar.gz#配置crictlcatEOF|tee/etc/crictl.yamlruntime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false pull-image-on-create: false disable-pull-on-run: false EOFsystemctl restart containerd crictl infocri-docker#安装配置dockersudodnf removedockerdocker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-enginesudodnfinstall-yyum-utils device-mapper-persistent-data lvm2sudodnf-yinstalldnf-plugins-coresudodnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.reposudodnfinstalldocker-ce docker-ce-cli containerd.iosudomkdir-p/etc/dockersudotee/etc/docker/daemon.json-EOF { exec-opts: [native.cgroupdriversystemd], log-driver: json-file, log-opts: { max-size: 100m }, experimental: false, debug: false, max-concurrent-downloads: 10, registry-mirrors: [https://a7h8080e.mirror.aliyuncs.com] } EOFsudosystemctl daemon-reloadsudosystemctlenabledockersudosystemctl restartdocker#安装 golangdnfinstallgolang#安装 gitdnfinstallgit#安装 cri-dockergitclone https://github.com/Mirantis/cri-dockerd.gitcdcri-dockerdmkdirbin go build-obin/cri-dockerdmkdir-p/usr/local/bininstall-oroot-groot-m0755 bin/cri-dockerd /usr/local/bin/cri-dockerd#更改 packaging/systemd 目录下 cri-docker.service 和 cri-docker.socket 文件ExecStart/usr/bin/cri-dockerd --container-runtime-endpoint fd:// 改为ExecStart/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugincni --pod-infra-container-imageregistry.aliyuncs.com/google_containers/pause:3.8 ListenStream 的值 改为ListenStream/var/run/cri-dockerd.sockcp-apackaging/systemd/* /etc/systemd/systemsed-i-es,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,/etc/systemd/system/cri-docker.service systemctl daemon-reload systemctlenablecri-docker.service systemctl restart cri-docker.socket crictl config runtime-endpoint unix:///run/cri-dockerd.sock crictl config image-endpoint unix:///run/cri-dockerd.sock#或者编辑 vim /etc/crictl.yaml安装kubeadm ,kubectl ,kubelet# 添加kubernetes yum软件源cat/etc/yum.repos.d/kubernetes.repoEOF [kubernetes] nameKubernetes baseurlhttps://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 enabled1 gpgcheck0 repo_gpgcheck0 gpgkeyhttps://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOFsudoyuminstall-ykubelet kubeadm kubectl--disableexcludeskubernetessudosystemctlenable--nowkubelet# 命令补全yuminstall-ybash-completionsource(crictl completionbash)crictl completionbash/etc/bash_completion.d/crictlsource(kubectl completionbash)kubectl completionbash/etc/bash_completion.d/kubectlsource/usr/share/bash-completion/bash_completion配置kubeadmkubeadm config print init-defaultskubeadm.yamlapiVersion:kubeadm.k8s.io/v1beta3bootstrapTokens:-groups:-system:bootstrappers:kubeadm:default-node-tokentoken:abcdef.0123456789abcdefttl:24h0m0susages:-signing-authenticationkind:InitConfigurationlocalAPIEndpoint:advertiseAddress:172.21.16.7bindPort:6443nodeRegistration:# 如果用cri-docker#criSocket: unix:///var/run/cri-dockerd.sockcriSocket:unix:///var/run/containerd/containerd.sockimagePullPolicy:IfNotPresentname:k8s-mastertaints:null---apiServer:timeoutForControlPlane:4m0sapiVersion:kubeadm.k8s.io/v1beta3certificatesDir:/etc/kubernetes/pkiclusterName:kubernetescontrollerManager:{}dns:{}etcd:local:dataDir:/var/lib/etcdimageRepository:registry.aliyuncs.com/google_containerskind:ClusterConfigurationkubernetesVersion:1.24.0networking:dnsDomain:cluster.localserviceSubnet:10.96.0.0/12podSubnet:10.244.0.0/16scheduler:{}---apiVersion:kubeproxy.config.k8s.io/v1alpha1kind:KubeProxyConfigurationmode:ipvs---apiVersion:kubelet.config.k8s.io/v1beta1kind:KubeletConfigurationcgroupDriver:systemd创建集群kubeadm init--configkubeadm.yamlmkdir-p$HOME/.kube\cp-i/etc/kubernetes/admin.conf$HOME/.kube/config\chown$(id-u):$(id-g)$HOME/.kube/config# 下载calico 网络配置curl-Ohttps://docs.projectcalico.org/manifests/calico.yaml#初始化网络配置kubectl apply-fcalico.yaml#设置端口范围 vim /etc/kubernetes/manifests/kube-apiserver.yaml 添加- --service-node-port-range1-65535单节点去除master节点的污点kubectl taintnodek8s-master node-role.kubernetes.io/control-plane-补充k8s 1.24.0版本基于nfs-client-provisioner的StorageClass PVC一直 pending无法创建旧版本的k8s 解决方案是更改/etc/kubernetes/manifests/kube-apiserver.yaml,添加- --feature-gatesRemoveSelfLinkfalse新版本后(1.24)更改此配置导致kubelet无法启动使用新版本的nfs-client-provisioner 即可解决此问题#添加repohelm repoaddnfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/#拉去资源helm pull nfs-subdir-external-provisioner/nfs-subdir-external-provisioner更改配置# 由于国内网络拉取不到谷歌仓库这里在docker hub上找的资源image:repository:docker.io/willdockerhub/nfs-subdir-external-provisioner...nfs:server:xx.xx.xx.xxxpath:xx...Tip1: kubernetes 1.24之后创建serviceaccount不会创建secrets了所以需要service-account-token来做bearer token得情况应该使用TokenRequest的方式来获取token1. 创建一个serviceaccount kubectl create sa admin2. 给sa绑定一个clusterrole kubectl create clusterrolebinding admin--clusterrolecluster-admin--serviceaccounttest:admin3. 从sa创建个token kubectl create token admin