kubernetes 1.17.2 kubeadm部署 证书修改为100年

[root@hs-k8s-master01 ~]# cd /data/
[root@hs-k8s-master01 data]# ls
docker
[root@hs-k8s-master01 data]# mkdir k8s
[root@hs-k8s-master01 data]# cd k8s/
[root@hs-k8s-master01 k8s]# ls
[root@hs-k8s-master01 k8s]# mkdir source_code
[root@hs-k8s-master01 k8s]# cd source_code/
[root@hs-k8s-master01 source_code]# rz

[root@hs-k8s-master01 source_code]# tar xf kubernetes-1.17.2.tar.gz 
[root@hs-k8s-master01 source_code]# ls
kubernetes-1.17.2  kubernetes-1.17.2.tar.gz
[root@hs-k8s-master01 source_code]# cd kubernetes-1.17.2/
[root@hs-k8s-master01 kubernetes-1.17.2]# ls
api                cluster             Godeps   logo                      pkg                SUPPORT.md    WORKSPACE
build              cmd                 go.mod   Makefile                  plugin             test
BUILD.bazel        code-of-conduct.md  go.sum   Makefile.generated_files  README.md          third_party
CHANGELOG-1.17.md  CONTRIBUTING.md     hack     OWNERS                    SECURITY_CONTACTS  translations
CHANGELOG.md       docs                LICENSE  OWNERS_ALIASES            staging            vendor
[root@hs-k8s-master01 kubernetes-1.17.2]# 
[root@hs-k8s-master01 kubernetes-1.17.2]# vim ./staging/src/k8s.io/c
client-go/           cloud-provider/      code-generator/      cri-api/             
cli-runtime/         cluster-bootstrap/   component-base/      csi-translation-lib/ 
[root@hs-k8s-master01 kubernetes-1.17.2]# vim ./staging/src/k8s.io/cli
client-go/   cli-runtime/ 
[root@hs-k8s-master01 kubernetes-1.17.2]# vim ./staging/src/k8s.io/client-go/util/cert
cert/        certificate/ 
[root@hs-k8s-master01 kubernetes-1.17.2]# vim ./staging/src/k8s.io/client-go/util/cert/cert.go 
[root@hs-k8s-master01 kubernetes-1.17.2]# vim ./cmd/kubeadm/app/util/pkiutil/pki_helpers.go 
[root@hs-k8s-master01 kubernetes-1.17.2]# vim ./cmd/kubeadm/app/constants/constants.go 
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull mirrorgooglecontainers/kube-cross:v1.12.10-1
Error response from daemon: Get https://registry-1.docker.io/v2/: dial tcp: lookup registry-1.docker.io on 223.5.5.5:53: read udp 10.0.0.200:37338->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull mirrorgooglecontainers/kube-cross:v1.12.10-1
Error response from daemon: Get https://registry-1.docker.io/v2/: dial tcp: lookup registry-1.docker.io on 223.5.5.5:53: read udp 10.0.0.200:4029->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull gcrcontainer/kube-cross:v1.13.5-1
Error response from daemon: Get https://registry-1.docker.io/v2/: dial tcp: lookup registry-1.docker.io on 223.5.5.5:53: read udp 10.0.0.200:59440->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-cross:v1.13.5-1
Error response from daemon: Get https://registry.cn-hangzhou.aliyuncs.com/v2/: dial tcp: lookup registry.cn-hangzhou.aliyuncs.com on 223.5.5.5:53: read udp 10.0.0.200:42909->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# dig @114.114.114.114 registry-1.docker.io

; <<>> DiG 9.11.4-P2-RedHat-9.11.4-9.P2.el7 <<>> @114.114.114.114 registry-1.docker.io
; (1 server found)
;; global options: +cmd
;; connection timed out; no servers could be reached
[root@hs-k8s-master01 kubernetes-1.17.2]# docker version
Client: Docker Engine - Community
 Version:           19.03.5
 API version:       1.40
 Go version:        go1.12.12
 Git commit:        633a0ea
 Built:             Wed Nov 13 07:25:41 2019
 OS/Arch:           linux/amd64
 Experimental:      false

Server: Docker Engine - Community
 Engine:
  Version:          19.03.3
  API version:      1.40 (minimum version 1.12)
  Go version:       go1.12.10
  Git commit:       a872fc2f86
  Built:            Tue Oct  8 00:56:46 2019
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.2.10
  GitCommit:        b34a5c8af56e510852c35414db4c1f4fa6172339
 runc:
  Version:          1.0.0-rc8+dev
  GitCommit:        3e425f80a8c931f88e6d94a8c831b9d5aa481657
 docker-init:
  Version:          0.18.0
  GitCommit:        fec3683
[root@hs-k8s-master01 kubernetes-1.17.2]# docker image ls
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
[root@hs-k8s-master01 kubernetes-1.17.2]# 
[root@hs-k8s-master01 kubernetes-1.17.2]# docekr search nginx
-bash: docekr: 未找到命令
[root@hs-k8s-master01 kubernetes-1.17.2]# docker search nginx
Error response from daemon: Get https://index.docker.io/v1/search?q=nginx&n=25: dial tcp: lookup index.docker.io on 223.5.5.5:53: read udp 10.0.0.200:15999->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# mv /etc/sysconfig/network-scripts/ifcfg-eth1 /tmp/
[root@hs-k8s-master01 kubernetes-1.17.2]# systemctl restart network
[root@hs-k8s-master01 kubernetes-1.17.2]# hostname -I
20.0.0.200 172.17.0.1 
[root@hs-k8s-master01 kubernetes-1.17.2]# docker search nginx
Error response from daemon: Get https://index.docker.io/v1/search?q=nginx&n=25: dial tcp: lookup index.docker.io on 223.5.5.5:53: read udp 20.0.0.200:45441->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull nginx
Using default tag: latest
latest: Pulling from library/nginx
bc51dd8edc1b: Downloading [=>                                                 ]  542.7kB/27.09MB
66ba67045f57: Downloading [=>                                                 ]  717.7kB/23.88MB
bf317aa10aa5: Download complete 
^C
[root@hs-k8s-master01 kubernetes-1.17.2]# docker image ls
REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
[root@hs-k8s-master01 kubernetes-1.17.2]# 
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull gccontainer/kube-cross:v1.13.5-1
Error response from daemon: Get https://registry-1.docker.io/v2/: dial tcp: lookup registry-1.docker.io on 223.5.5.5:53: read udp 20.0.0.200:61687->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# dig @114.114.114.114 registry-1.docker.io

; <<>> DiG 9.11.4-P2-RedHat-9.11.4-9.P2.el7 <<>> @114.114.114.114 registry-1.docker.io
; (1 server found)
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 7712
;; flags: qr rd ra; QUERY: 1, ANSWER: 8, AUTHORITY: 0, ADDITIONAL: 1

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 512
;; QUESTION SECTION:
;registry-1.docker.io.        IN    A

;; ANSWER SECTION:
registry-1.docker.io.    34    IN    A    34.197.189.129
registry-1.docker.io.    34    IN    A    34.228.211.243
registry-1.docker.io.    34    IN    A    34.199.77.19
registry-1.docker.io.    34    IN    A    3.226.66.79
registry-1.docker.io.    34    IN    A    34.201.196.144
registry-1.docker.io.    34    IN    A    34.232.31.24
registry-1.docker.io.    34    IN    A    34.199.40.84
registry-1.docker.io.    34    IN    A    3.224.75.242

;; Query time: 15 msec
;; SERVER: 114.114.114.114#53(114.114.114.114)
;; WHEN: 一 2月 03 11:43:57 CST 2020
;; MSG SIZE  rcvd: 177

[root@hs-k8s-master01 kubernetes-1.17.2]# vim /etc/hosts
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull gccontainer/kube-cross:v1.13.5-1
Error response from daemon: Get https://registry-1.docker.io/v2/gccontainer/kube-cross/manifests/v1.13.5-1: Get https://auth.docker.io/token?scope=repository%3Agccontainer%2Fkube-cross%3Apull&service=registry.docker.io: dial tcp: lookup auth.docker.io on 223.5.5.5:53: read udp 20.0.0.200:31167->223.5.5.5:53: i/o timeout
[root@hs-k8s-master01 kubernetes-1.17.2]# vim /etc/sysconfig/network-scripts/ifcfg-eth0 
[root@hs-k8s-master01 kubernetes-1.17.2]# systemctl restart network
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull gccontainer/kube-cross:v1.13.5-1
Error response from daemon: pull access denied for gccontainer/kube-cross, repository does not exist or may require ‘docker login‘: denied: requested access to the resource is denied
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull gccontainer/kube-cross:v1.13.5
Error response from daemon: pull access denied for gccontainer/kube-cross, repository does not exist or may require ‘docker login‘: denied: requested access to the resource is denied
[root@hs-k8s-master01 kubernetes-1.17.2]# docker pull gcrcontainer/kube-cross:v1.13.5-1 


查看网上的资料主要有两个地方需要修改

vim ./staging/src/k8s.io/client-go/util/cert/cert.go
# 这个方法里面NotAfter:              now.Add(duration365d * 10).UTC()
# 默认有效期就是10年，改成100年
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
        now := time.Now()
        tmpl := x509.Certificate{
                SerialNumber: new(big.Int).SetInt64(0),
                Subject: pkix.Name{
                        CommonName:   cfg.CommonName,
                        Organization: cfg.Organization,
                },
                NotBefore:             now.UTC(),
                // NotAfter:              now.Add(duration365d * 10).UTC(),
                NotAfter:              now.Add(duration365d * 100).UTC(),
                KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
                BasicConstraintsValid: true,
                IsCA:                  true,
        }
 
        certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
        if err != nil {
                return nil, err
        }
        return x509.ParseCertificate(certDERBytes)
}
 

vim ./cmd/kubeadm/app/util/pkiutil/pki_helpers.go
# 这个方法里面看到NotAfter:     time.Now().Add(kubeadmconstants.CertificateValidity).UTC()
# 参数里面是一个常量kubeadmconstants.CertificateValidity
# 所以这里可以不修改，我去看看源码能不能找到这个常量的赋值位置
func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {        serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64))
        if err != nil {
                return nil, err
        }       
        if len(cfg.CommonName) == 0 {
                return nil, errors.New("must specify a CommonName")
        }       
        if len(cfg.Usages) == 0 {
                return nil, errors.New("must specify at least one ExtKeyUsage")
        }       
        
        certTmpl := x509.Certificate{
                Subject: pkix.Name{
                        CommonName:   cfg.CommonName,
                        Organization: cfg.Organization,
                },      
                DNSNames:     cfg.AltNames.DNSNames,
                IPAddresses:  cfg.AltNames.IPs,
                SerialNumber: serial,
                NotBefore:    caCert.NotBefore,
                NotAfter:     time.Now().Add(kubeadmconstants.CertificateValidity).UTC(),
                KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
                ExtKeyUsage:  cfg.Usages,
        }       
        certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
        if err != nil {
                return nil, err
        }       
        return x509.ParseCertificate(certDERBytes)
} 
结果在这里找到kubeadmconstants.CertificateValidity的定义

vim ./cmd/kubeadm/app/constants/constants.go
// 就是这个常量定义CertificateValidity，我改成*100年
const (
        // KubernetesDir is the directory Kubernetes owns for storing various configuration files
        KubernetesDir = "/etc/kubernetes"
        // ManifestsSubDirName defines directory name to store manifests
        ManifestsSubDirName = "manifests"
        // TempDirForKubeadm defines temporary directory for kubeadm
        // should be joined with KubernetesDir.
        TempDirForKubeadm = "tmp"
 
        // CertificateValidity defines the validity for all the signed certificates generated by kubeadm
        // CertificateValidity = time.Hour * 24 * 365
        CertificateValidity = time.Hour * 24 * 365 * 100
 
        // CACertAndKeyBaseName defines certificate authority base name
        CACertAndKeyBaseName = "ca"
        // CACertName defines certificate name
        CACertName = "ca.crt"
        // CAKeyName defines certificate name
        CAKeyName = "ca.key"
源代码改好了，接下来就是编译kubeadm了

[root@hs-k8s-master01 ~]# kubeadm  alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml‘

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Feb 02, 2021 07:17 UTC   364d                                    no      
apiserver                  Feb 02, 2021 07:17 UTC   364d            ca                      no      
apiserver-etcd-client      Feb 02, 2021 07:17 UTC   364d            etcd-ca                 no      
apiserver-kubelet-client   Feb 02, 2021 07:17 UTC   364d            ca                      no      
controller-manager.conf    Feb 02, 2021 07:17 UTC   364d                                    no      
etcd-healthcheck-client    Feb 02, 2021 07:17 UTC   364d            etcd-ca                 no      
etcd-peer                  Feb 02, 2021 07:17 UTC   364d            etcd-ca                 no      
etcd-server                Feb 02, 2021 07:17 UTC   364d            etcd-ca                 no      
front-proxy-client         Feb 02, 2021 07:17 UTC   364d            front-proxy-ca          no      
scheduler.conf             Feb 02, 2021 07:17 UTC   364d                                    no      

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Jan 31, 2030 07:17 UTC   9y              no      
etcd-ca                 Jan 31, 2030 07:17 UTC   9y              no      
front-proxy-ca          Jan 31, 2030 07:17 UTC   9y              no  

[root@hs-k8s-master01 ~]# cd /data/k8s/
[root@hs-k8s-master01 k8s]# ls
source_code  yaml
[root@hs-k8s-master01 k8s]# cd source_code/
[root@hs-k8s-master01 source_code]# ls
kubernetes-1.17.2  kubernetes-1.17.2.tar.gz
[root@hs-k8s-master01 source_code]# cd kubernetes-1.17.2/
[root@hs-k8s-master01 kubernetes-1.17.2]# ls
api                cluster             Godeps   logo                      OWNERS_ALIASES     staging       vendor
build              cmd                 go.mod   Makefile                  pkg                SUPPORT.md    WORKSPACE
BUILD.bazel        code-of-conduct.md  go.sum   Makefile.generated_files  plugin             test
CHANGELOG-1.17.md  CONTRIBUTING.md     hack     _output                   README.md          third_party
CHANGELOG.md       docs                LICENSE  OWNERS                    SECURITY_CONTACTS  translations
[root@hs-k8s-master01 kubernetes-1.17.2]# cd _output/
[root@hs-k8s-master01 _output]# ls
APIEXTENSIONS_violations.report  bin  CODEGEN_violations.report  KUBE_violations.report  local  SAMPLEAPISERVER_violations.report
[root@hs-k8s-master01 _output]# ll
总用量 88
-rw-r--r-- 1 root root  3669 2月   3 12:08 APIEXTENSIONS_violations.report
lrwxrwxrwx 1 root root    55 2月   3 12:09 bin -> /go/src/k8s.io/kubernetes/_output/local/bin/linux/amd64
-rw-r--r-- 1 root root  4256 2月   3 12:08 CODEGEN_violations.report
-rw-r--r-- 1 root root 73192 2月   3 12:08 KUBE_violations.report
drwxr-xr-x 4 root root    27 2月   3 12:07 local
-rw-r--r-- 1 root root  3999 2月   3 12:08 SAMPLEAPISERVER_violations.report
[root@hs-k8s-master01 _output]# cd local/
[root@hs-k8s-master01 local]# ls
bin  go
[root@hs-k8s-master01 local]# cd bin/
[root@hs-k8s-master01 bin]# ls
linux
[root@hs-k8s-master01 bin]# cd linux/
[root@hs-k8s-master01 linux]# ls
amd64
[root@hs-k8s-master01 linux]# cd amd64/
[root@hs-k8s-master01 amd64]# ls
conversion-gen  deepcopy-gen  defaulter-gen  go2make  go-bindata  kubeadm  openapi-gen
[root@hs-k8s-master01 amd64]# 
[root@hs-k8s-master01 amd64]# cd ../../
[root@hs-k8s-master01 bin]# ls
linux
[root@hs-k8s-master01 bin]# cd ../
[root@hs-k8s-master01 local]# ls
bin  go
[root@hs-k8s-master01 local]# cd ..
[root@hs-k8s-master01 _output]# ls
APIEXTENSIONS_violations.report  bin  CODEGEN_violations.report  KUBE_violations.report  local  SAMPLEAPISERVER_violations.report
[root@hs-k8s-master01 _output]# cd ..
[root@hs-k8s-master01 kubernetes-1.17.2]# ls
api                cluster             Godeps   logo                      OWNERS_ALIASES     staging       vendor
build              cmd                 go.mod   Makefile                  pkg                SUPPORT.md    WORKSPACE
BUILD.bazel        code-of-conduct.md  go.sum   Makefile.generated_files  plugin             test
CHANGELOG-1.17.md  CONTRIBUTING.md     hack     _output                   README.md          third_party
CHANGELOG.md       docs                LICENSE  OWNERS                    SECURITY_CONTACTS  translations
[root@hs-k8s-master01 kubernetes-1.17.2]# cp /usr/bin/kubeadm{,.bak}
[root@hs-k8s-master01 kubernetes-1.17.2]# cp _output/local/bin/linux/amd64/kubeadm 
[root@hs-k8s-master01 kubernetes-1.17.2]# cp _output/local/bin/linux/amd64/kubeadm /usr/bin/kubeadm
cp：是否覆盖"/usr/bin/kubeadm"？ y
[root@hs-k8s-master01 kubernetes-1.17.2]# cd /etc/kubernetes/pki/
[root@hs-k8s-master01 pki]# ls
apiserver.crt              apiserver.key                 ca.crt  front-proxy-ca.crt      front-proxy-client.key
apiserver-etcd-client.crt  apiserver-kubelet-client.crt  ca.key  front-proxy-ca.key      sa.key
apiserver-etcd-client.key  apiserver-kubelet-client.key  etcd    front-proxy-client.crt  sa.pub
[root@hs-k8s-master01 pki]# cd ..
[root@hs-k8s-master01 kubernetes]# ls
admin.conf  controller-manager.conf  gcrcontainer-kube-cross:v1.13.5-1.tar  kubelet.conf  manifests  pki  scheduler.conf
[root@hs-k8s-master01 kubernetes]# ll
总用量 1875756
-rw------- 1 root root       5450 2月   3 15:17 admin.conf
-rw------- 1 root root       5482 2月   3 15:17 controller-manager.conf
-rw-r--r-- 1 root root 1920737792 2月   3 12:20 gcrcontainer-kube-cross:v1.13.5-1.tar
-rw------- 1 root root       1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root        113 2月   3 15:17 manifests
drwxr-xr-x 3 root root       4096 2月   3 15:17 pki
-rw------- 1 root root       5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# rm -f gcrcontainer-kube-cross\:v1.13.5-1.tar 
[root@hs-k8s-master01 kubernetes]# ls
admin.conf  controller-manager.conf  kubelet.conf  manifests  pki  scheduler.conf
[root@hs-k8s-master01 kubernetes]# 
[root@hs-k8s-master01 kubernetes]# ll
总用量 32
-rw------- 1 root root 5450 2月   3 15:17 admin.conf
-rw------- 1 root root 5482 2月   3 15:17 controller-manager.conf
-rw------- 1 root root 1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root  113 2月   3 15:17 manifests
drwxr-xr-x 3 root root 4096 2月   3 15:17 pki
-rw------- 1 root root 5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# mkdir pki.bak
[root@hs-k8s-master01 kubernetes]# ll
总用量 32
-rw------- 1 root root 5450 2月   3 15:17 admin.conf
-rw------- 1 root root 5482 2月   3 15:17 controller-manager.conf
-rw------- 1 root root 1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root  113 2月   3 15:17 manifests
drwxr-xr-x 3 root root 4096 2月   3 15:17 pki
drwxr-xr-x 2 root root    6 2月   3 16:57 pki.bak
-rw------- 1 root root 5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# vm pki/* pki.bak/
-bash: vm: 未找到命令
[root@hs-k8s-master01 kubernetes]# mv pki/* pki.bak/
[root@hs-k8s-master01 kubernetes]# ll
总用量 32
-rw------- 1 root root 5450 2月   3 15:17 admin.conf
-rw------- 1 root root 5482 2月   3 15:17 controller-manager.conf
-rw------- 1 root root 1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root  113 2月   3 15:17 manifests
drwxr-xr-x 2 root root    6 2月   3 16:57 pki
drwxr-xr-x 3 root root 4096 2月   3 16:57 pki.bak
-rw------- 1 root root 5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# 
[root@hs-k8s-master01 kubernetes]# cd pki
[root@hs-k8s-master01 pki]# ls
[root@hs-k8s-master01 pki]# cd ..
[root@hs-k8s-master01 kubernetes]# kubeadm alpha certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml‘

Error checking external CA condition for ca certificate authority: failure loading certificate for CA: couldn‘t load the certificate file /etc/kubernetes/pki/ca.crt: open /etc/kubernetes/pki/ca.crt: no such file or directory
To see the stack trace of this error execute with --v=5 or higher
[root@hs-k8s-master01 kubernetes]# ll
总用量 32
-rw------- 1 root root 5450 2月   3 15:17 admin.conf
-rw------- 1 root root 5482 2月   3 15:17 controller-manager.conf
-rw------- 1 root root 1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root  113 2月   3 15:17 manifests
drwxr-xr-x 2 root root    6 2月   3 16:57 pki
drwxr-xr-x 3 root root 4096 2月   3 16:57 pki.bak
-rw------- 1 root root 5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# cp pki.bak/* pki/
cp: 略过目录"pki.bak/etcd"
[root@hs-k8s-master01 kubernetes]# ll
总用量 36
-rw------- 1 root root 5450 2月   3 15:17 admin.conf
-rw------- 1 root root 5482 2月   3 15:17 controller-manager.conf
-rw------- 1 root root 1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root  113 2月   3 15:17 manifests
drwxr-xr-x 2 root root 4096 2月   3 16:58 pki
drwxr-xr-x 3 root root 4096 2月   3 16:57 pki.bak
-rw------- 1 root root 5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# cd pki
[root@hs-k8s-master01 pki]# ls
apiserver.crt              apiserver.key                 ca.crt              front-proxy-ca.key      sa.key
apiserver-etcd-client.crt  apiserver-kubelet-client.crt  ca.key              front-proxy-client.crt  sa.pub
apiserver-etcd-client.key  apiserver-kubelet-client.key  front-proxy-ca.crt  front-proxy-client.key
[root@hs-k8s-master01 pki]# cd ..
[root@hs-k8s-master01 kubernetes]# ls
admin.conf  controller-manager.conf  kubelet.conf  manifests  pki  pki.bak  scheduler.conf
[root@hs-k8s-master01 kubernetes]# cd pki.bak/
[root@hs-k8s-master01 pki.bak]# ls
apiserver.crt              apiserver.key                 ca.crt  front-proxy-ca.crt      front-proxy-client.key
apiserver-etcd-client.crt  apiserver-kubelet-client.crt  ca.key  front-proxy-ca.key      sa.key
apiserver-etcd-client.key  apiserver-kubelet-client.key  etcd    front-proxy-client.crt  sa.pub
[root@hs-k8s-master01 pki.bak]# cd etcd/
[root@hs-k8s-master01 etcd]# ls
ca.crt  ca.key  healthcheck-client.crt  healthcheck-client.key  peer.crt  peer.key  server.crt  server.key
[root@hs-k8s-master01 etcd]# cd ..
[root@hs-k8s-master01 pki.bak]# cd ..
[root@hs-k8s-master01 kubernetes]# cd pki
[root@hs-k8s-master01 pki]# ll
总用量 56
-rw-r--r-- 1 root root 1241 2月   3 16:58 apiserver.crt
-rw-r--r-- 1 root root 1090 2月   3 16:58 apiserver-etcd-client.crt
-rw------- 1 root root 1675 2月   3 16:58 apiserver-etcd-client.key
-rw------- 1 root root 1675 2月   3 16:58 apiserver.key
-rw-r--r-- 1 root root 1099 2月   3 16:58 apiserver-kubelet-client.crt
-rw------- 1 root root 1675 2月   3 16:58 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1025 2月   3 16:58 ca.crt
-rw------- 1 root root 1675 2月   3 16:58 ca.key
-rw-r--r-- 1 root root 1038 2月   3 16:58 front-proxy-ca.crt
-rw------- 1 root root 1679 2月   3 16:58 front-proxy-ca.key
-rw-r--r-- 1 root root 1058 2月   3 16:58 front-proxy-client.crt
-rw------- 1 root root 1679 2月   3 16:58 front-proxy-client.key
-rw------- 1 root root 1675 2月   3 16:58 sa.key
-rw------- 1 root root  451 2月   3 16:58 sa.pub
[root@hs-k8s-master01 pki]# mkdir etcd
[root@hs-k8s-master01 pki]# cd ..
[root@hs-k8s-master01 kubernetes]# cd pki.bak/
[root@hs-k8s-master01 pki.bak]# mv etcd/* ../pki/etcd/
[root@hs-k8s-master01 pki.bak]# cd ..
[root@hs-k8s-master01 kubernetes]# ll
总用量 36
-rw------- 1 root root 5450 2月   3 15:17 admin.conf
-rw------- 1 root root 5482 2月   3 15:17 controller-manager.conf
-rw------- 1 root root 1894 2月   3 15:17 kubelet.conf
drwxr-xr-x 2 root root  113 2月   3 15:17 manifests
drwxr-xr-x 3 root root 4096 2月   3 16:59 pki
drwxr-xr-x 3 root root 4096 2月   3 16:57 pki.bak
-rw------- 1 root root 5430 2月   3 15:17 scheduler.conf
[root@hs-k8s-master01 kubernetes]# cd pki
[root@hs-k8s-master01 pki]# ll
总用量 56
-rw-r--r-- 1 root root 1241 2月   3 16:58 apiserver.crt
-rw-r--r-- 1 root root 1090 2月   3 16:58 apiserver-etcd-client.crt
-rw------- 1 root root 1675 2月   3 16:58 apiserver-etcd-client.key
-rw------- 1 root root 1675 2月   3 16:58 apiserver.key
-rw-r--r-- 1 root root 1099 2月   3 16:58 apiserver-kubelet-client.crt
-rw------- 1 root root 1675 2月   3 16:58 apiserver-kubelet-client.key
-rw-r--r-- 1 root root 1025 2月   3 16:58 ca.crt
-rw------- 1 root root 1675 2月   3 16:58 ca.key
drwxr-xr-x 2 root root  162 2月   3 16:59 etcd
-rw-r--r-- 1 root root 1038 2月   3 16:58 front-proxy-ca.crt
-rw------- 1 root root 1679 2月   3 16:58 front-proxy-ca.key
-rw-r--r-- 1 root root 1058 2月   3 16:58 front-proxy-client.crt
-rw------- 1 root root 1679 2月   3 16:58 front-proxy-client.key
-rw------- 1 root root 1675 2月   3 16:58 sa.key
-rw------- 1 root root  451 2月   3 16:58 sa.pub
[root@hs-k8s-master01 pki]# kubeadm alpha certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml‘

certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
[root@hs-k8s-master01 pki]# kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml‘

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Jan 10, 2120 08:59 UTC   99y                                     no      
apiserver                  Jan 10, 2120 08:59 UTC   99y             ca                      no      
apiserver-etcd-client      Jan 10, 2120 08:59 UTC   99y             etcd-ca                 no      
apiserver-kubelet-client   Jan 10, 2120 08:59 UTC   99y             ca                      no      
controller-manager.conf    Jan 10, 2120 08:59 UTC   99y                                     no      
etcd-healthcheck-client    Jan 10, 2120 08:59 UTC   99y             etcd-ca                 no      
etcd-peer                  Jan 10, 2120 08:59 UTC   99y             etcd-ca                 no      
etcd-server                Jan 10, 2120 08:59 UTC   99y             etcd-ca                 no      
front-proxy-client         Jan 10, 2120 08:59 UTC   99y             front-proxy-ca          no      
scheduler.conf             Jan 10, 2120 08:59 UTC   99y                                     no      

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Jan 31, 2030 07:17 UTC   9y              no      
etcd-ca                 Jan 31, 2030 07:17 UTC   9y              no      
front-proxy-ca          Jan 31, 2030 07:17 UTC   9y              no      


[root@bs-k8s-master02 ~]# cp /usr/bin/kubeadm{,.bak} 
[root@hs-k8s-master01 pki]# scp /usr/bin/kubeadm 20.0.0.201:/usr/bin/kubeadm
[root@bs-k8s-master02 ~]# kubeadm alpha certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml‘

certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
[root@bs-k8s-master02 ~]# kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with ‘kubectl -n kube-system get cm kubeadm-config -oyaml‘

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Jan 10, 2120 09:03 UTC   99y                                     no      
apiserver                  Jan 10, 2120 09:03 UTC   99y             ca                      no      
apiserver-etcd-client      Jan 10, 2120 09:03 UTC   99y             etcd-ca                 no      
apiserver-kubelet-client   Jan 10, 2120 09:03 UTC   99y             ca                      no      
controller-manager.conf    Jan 10, 2120 09:03 UTC   99y                                     no      
etcd-healthcheck-client    Jan 10, 2120 09:03 UTC   99y             etcd-ca                 no      
etcd-peer                  Jan 10, 2120 09:04 UTC   99y             etcd-ca                 no      
etcd-server                Jan 10, 2120 09:04 UTC   99y             etcd-ca                 no      
front-proxy-client         Jan 10, 2120 09:04 UTC   99y             front-proxy-ca          no      
scheduler.conf             Jan 10, 2120 09:04 UTC   99y                                     no      

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Jan 31, 2030 07:17 UTC   9y              no      
etcd-ca                 Jan 31, 2030 07:17 UTC   9y              no      
front-proxy-ca          Jan 31, 2030 07:17 UTC   9y              no      

同理 master03