189 8069 5689
备注：红色字迹为需要在shell中输入的内容：
创新互联公司是专业的普定网站建设公司，普定接单;提供网站建设、成都网站建设,网页设计,网站设计,建网站,PHP网站建设等专业做网站服务;采用PHP框架,可快速的进行普定网站开发网页制作和功能扩展;专业做搜索引擎喜爱的网站,专业的做网站团队,希望更多企业前来合作!
机器规划：
开机自动关闭防火墙
[root@k8s-etcd1 ~]# systemctl disable firewalld.service
关闭防火墙
[root@k8s-etcd1 ~]# systemctl stop firewalld.service
计划任务同步时间：
0 */1 * * * /usr/sbin/ntpdate cn.pool.ntp.org > /dev/null 2>&1
关闭selinux
[root@k8s-etcd1 ~]# sed -i "s#SELINUX=enforcing#SELINUX=disabled#g" /etc/selinux/config
[root@k8s-etcd1 ~]# sed -i "s#SELINUX=enforcing#SELINUX=disabled#g" /etc/sysconfig/selinux
修改主机名
[root@k8s-etcd1 ~]# hostnamectl set-hostname k8s-etcd1
同步时间
[root@k8s-etcd1 ~]# sudo timedatectl set-timezone Asia/Shanghai
[root@k8s-etcd1 ~]# date
[root@k8s-etcd1 ~]# sudo timedatectl set-local-rtc 0
[root@k8s-etcd1 ~]# sudo systemctl restart rsyslog
[root@k8s-etcd1 ~]# sudo systemctl restart crond
关闭swap分区
[root@k8s-etcd1 ~]# sudo swapoff -a
[root@k8s-etcd1 ~]# free -m
total used free shared buff/cache available
Mem: 3770 136 3510 11 124 3449
Swap: 0 0 0 确认swap分区已关闭
开机自动关闭swap分区
[root@k8s-etcd1 ~]# sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
将主机解析到hosts文件里面
[root@k8s-etcd1 ~]# cat >> /etc/hosts << EOF
192.168.1.121 k8s-etcd1
192.168.1.122 k8s-etcd2
192.168.1.123 k8s-etcd3
192.168.1.124 k8s-master1
192.168.1.125 k8s-master2
192.168.1.126 k8s-master3
192.168.1.127 k8s-node1
192.168.1.128 k8s-node2
192.168.1.129 k8s-node3
192.168.1.130 LVS1
192.168.1.131 LVS2
EOF
环境准备：
软件准备：
[root@k8s-etcd1 src]# ls
etcd.tar.gz HA.zip k8s-master.tar.gz k8s-node.tar.gz TLS.tar.gz
[root@k8s-etcd1 src]# tar xf etcd.tar.gz
[root@k8s-etcd1 src]# ls
etcd etcd.service etcd.tar.gz HA.zip k8s-master.tar.gz k8s-node.tar.gz TLS.tar.gz
[root@k8s-etcd1 src]# mv etcd /opt/
[root@k8s-etcd1 src]# ls
etcd.service etcd.tar.gz HA.zip k8s-master.tar.gz k8s-node.tar.gz TLS.tar.gz
[root@k8s-etcd1 src]# cp etcd.service /usr/lib/systemd/system/
[root@k8s-etcd1 src]# ls
etcd.service etcd.tar.gz HA.zip k8s-master.tar.gz k8s-node.tar.gz TLS.tar.gz
[root@k8s-etcd1 src]# tar xf TLS.tar.gz
[root@k8s-etcd1 src]# ls
etcd.service etcd.tar.gz HA.zip k8s-master.tar.gz k8s-node.tar.gz TLS TLS.tar.gz
[root@k8s-etcd1 src]# cd TLS
[root@k8s-etcd1 TLS]# ls
cfssl cfssl-certinfo cfssljson cfssl.sh etcd k8s
[root@k8s-etcd1 TLS]# vim cfssl.sh
#curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
#curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
#curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin
chmod +x /usr/local/bin/cfssl*
[root@k8s-etcd1 TLS]# cd etcd/
[root@k8s-etcd1 etcd]# ls
ca-config.json ca-csr.json generate_etcd_cert.sh server-csr.json
[root@k8s-etcd1 etcd]# vim server-csr.json
{
"CN": "etcd",
"hosts": [
"192.168.1.121",
"192.168.1.122",
"192.168.1.123"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
[root@k8s-etcd1 etcd]# cat generate_etcd_cert.sh
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
[root@k8s-etcd1 TLS]# bash -x cfssl.sh
+ cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin
+ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssl-certinfo /usr/local/bin/cfssljson
[root@k8s-etcd1 TLS]# cd etcd/
[root@k8s-etcd1 etcd]# ls
ca-config.json ca-csr.json generate_etcd_cert.sh server-csr.json
[root@k8s-etcd1 etcd]# ./generate_etcd_cert.sh
2020/01/29 17:16:06 [INFO] generating a new CA key and certificate from CSR
2020/01/29 17:16:06 [INFO] generate received request
2020/01/29 17:16:06 [INFO] received CSR
2020/01/29 17:16:06 [INFO] generating key: rsa-2048
2020/01/29 17:16:07 [INFO] encoded CSR
2020/01/29 17:16:07 [INFO] signed certificate with serial number 269437542636059925584584985232691741175950103574
2020/01/29 17:16:08 [INFO] generate received request
2020/01/29 17:16:08 [INFO] received CSR
2020/01/29 17:16:08 [INFO] generating key: rsa-2048
2020/01/29 17:16:08 [INFO] encoded CSR
2020/01/29 17:16:09 [INFO] signed certificate with serial number 193231136641378501576889547197718117578468352944
2020/01/29 17:16:09 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-etcd1 etcd]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem generate_etcd_cert.sh server.csr server-csr.json server-key.pem server.pem
[root@k8s-etcd1 etcd]# cp ca.pem server-key.pem server.pem /opt/etcd/ssl/
cp: overwrite ‘/opt/etcd/ssl/ca.pem’? y
cp: overwrite ‘/opt/etcd/ssl/server-key.pem’? y
cp: overwrite ‘/opt/etcd/ssl/server.pem’? y
[root@k8s-etcd1 ssl]# vim /opt/etcd/cfg/etcd.conf
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.121:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.121:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.121:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.121:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
"/opt/etcd/cfg/etcd.conf" 13L, 510C written
复制配置文件和etcd程序到另外两个节点：
[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.123:/opt/
The authenticity of host '192.168.1.123 (192.168.1.123)' can't be established.
ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.
ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.123' (ECDSA) to the list of known hosts.
root@192.168.1.123's password:
etcd 100% 16MB 10.3MB/s 00:01
etcdctl 100% 13MB 12.9MB/s 00:01
etcd.conf 100% 510 88.3KB/s 00:00
ca.pem 100% 1265 263.3KB/s 00:00
server.pem 100% 1338 267.5KB/s 00:00
server-key.pem 100% 1675 51.1KB/s 00:00
[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.122:/opt/
root@192.168.1.122's password:
etcd 100% 16MB 11.7MB/s 00:01
etcdctl 100% 13MB 9.0MB/s 00:01
etcd.conf 100% 510 94.0KB/s 00:00
ca.pem 100% 1265 264.9KB/s 00:00
server.pem 100% 1338 295.0KB/s 00:00
server-key.pem 100% 1675 140.1KB/s 00:00
[root@k8s-etcd2 cfg]# vim etcd.conf
#[Member]
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.122:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.122:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.122:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.122:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@k8s-etcd3cfg]# cat etcd.conf
#[Member]
ETCD_NAME="etcd-3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.123:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.123:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.123:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.123:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@k8s-etcd1 src]# scp -r etcd.service root@192.168.1.122:/usr/lib/systemd/system/
root@192.168.1.122's password:
etcd.service 100% 1078 202.6KB/s 00:00
[root@k8s-etcd1 src]# scp -r etcd.service root@192.168.1.123:/usr/lib/systemd/system/
root@192.168.1.123's password:
etcd.service 100% 1078 209.8KB/s 00:00
启动etcd服务并加入开机自启动：
[root@k8s-etcd1 src]# systemctl restart etcd.service
[root@k8s-etcd1 src]# systemctl status etcd
● etcd.service - Etcd Server
Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 17:33:30 CST; 11s ago
Main PID: 13531 (etcd)
CGroup: /system.slice/etcd.service
└─13531 /opt/etcd/bin/etcd --name=etcd-1 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.1.121:2380 --listen-client-urls=https://1...
Jan 29 17:33:30 k8s-etcd1 etcd[13531]: enabled capabilities for version 3.0
Jan 29 17:33:32 k8s-etcd1 etcd[13531]: peer c48307bcc0ac155e became active
Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 reader)
Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 writer)
Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message writer)
Jan 29 17:33:32 k8s-etcd1 etcd[13531]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message reader)
Jan 29 17:33:34 k8s-etcd1 etcd[13531]: updating the cluster version from 3.0 to 3.3
Jan 29 17:33:35 k8s-etcd1 etcd[13531]: updated the cluster version from 3.0 to 3.3
Jan 29 17:33:35 k8s-etcd1 etcd[13531]: enabled capabilities for version 3.3
Jan 29 17:33:35 k8s-etcd1 etcd[13531]: health check for peer c48307bcc0ac155e could not connect: dial tcp 192.168.1.123:2380: connect: connection refused (p..._MESSAGE")
Hint: Some lines were ellipsized, use -l to show in full.
[root@k8s-etcd1 src]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
[root@k8s-etcd2 cfg]# systemctl restart etcd.service
[root@k8s-etcd2 cfg]# systemctl status etcd
● etcd.service - Etcd Server
Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 17:33:30 CST; 34s ago
Main PID: 13101 (etcd)
CGroup: /system.slice/etcd.service
└─13101 /opt/etcd/bin/etcd --name=etcd-2 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.1.122:2380 --listen-client-urls=https://1...
Jan 29 17:33:30 k8s-etcd2 etcd[13101]: set the initial cluster version to 3.0
Jan 29 17:33:30 k8s-etcd2 etcd[13101]: enabled capabilities for version 3.0
Jan 29 17:33:32 k8s-etcd2 etcd[13101]: peer c48307bcc0ac155e became active
Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 writer)
Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message writer)
Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream MsgApp v2 reader)
Jan 29 17:33:32 k8s-etcd2 etcd[13101]: b5900dafc7a3b5f9 initialzed peer connection; fast-forwarding 8 ticks (election ticks 10) with 2 active peer(s)
Jan 29 17:33:32 k8s-etcd2 etcd[13101]: established a TCP streaming connection with peer c48307bcc0ac155e (stream Message reader)
Jan 29 17:33:35 k8s-etcd2 etcd[13101]: updated the cluster version from 3.0 to 3.3
Jan 29 17:33:35 k8s-etcd2 etcd[13101]: enabled capabilities for version 3.3
[root@k8s-etcd2 cfg]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
[root@k8s-etcd3 cfg]# systemctl restart etcd.service
[root@k8s-etcd3 cfg]# systemctl status etcd
● etcd.service - Etcd Server
Loaded: loaded (/usr/lib/systemd/system/etcd.service; disabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 17:33:32 CST; 36s ago
Main PID: 13071 (etcd)
CGroup: /system.slice/etcd.service
└─13071 /opt/etcd/bin/etcd --name=etcd-3 --data-dir=/var/lib/etcd/default.etcd --listen-peer-urls=https://192.168.1.123:2380 --listen-client-urls=https://1...
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: set the initial cluster version to 3.0
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: enabled capabilities for version 3.0
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: published {Name:etcd-3 ClientURLs:[https://192.168.1.123:2379]} to cluster ff7100b1c25258f1
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: ready to serve client requests
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: serving insecure client requests on 127.0.0.1:2379, this is strongly discouraged!
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: ready to serve client requests
Jan 29 17:33:32 k8s-etcd3 systemd[1]: Started Etcd Server.
Jan 29 17:33:32 k8s-etcd3 etcd[13071]: serving client requests on 192.168.1.123:2379
Jan 29 17:33:35 k8s-etcd3 etcd[13071]: updated the cluster version from 3.0 to 3.3
Jan 29 17:33:35 k8s-etcd3 etcd[13071]: enabled capabilities for version 3.3
[root@k8s-etcd3 cfg]# systemctl enable etcd
Created symlink from /etc/systemd/system/multi-user.target.wants/etcd.service to /usr/lib/systemd/system/etcd.service.
至此，etcd集群安装完毕
安装k8s-master集群开始：
将k8s-master.tar.gz软件包复制到三个master节点：
[root@k8s-etcd1 src]# scp -r k8s-master.tar.gz root@192.168.1.124:/usr/local/src/
The authenticity of host '192.168.1.124 (192.168.1.124)' can't be established.
ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.
ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.124' (ECDSA) to the list of known hosts.
root@192.168.1.124's password:
k8s-master.tar.gz 100% 87MB 19.2MB/s 00:04
[root@k8s-etcd1 src]# scp -r k8s-master.tar.gz root@192.168.1.125:/usr/local/src/
The authenticity of host '192.168.1.125 (192.168.1.125)' can't be established.
ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.
ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.125' (ECDSA) to the list of known hosts.
root@192.168.1.125's password:
k8s-master.tar.gz 100% 87MB 17.3MB/s 00:05
[root@k8s-etcd1 src]# scp -r k8s-master.tar.gz root@192.168.1.126:/usr/local/src/
The authenticity of host '192.168.1.126 (192.168.1.126)' can't be established.
ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.
ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.126' (ECDSA) to the list of known hosts.
root@192.168.1.126's password:
k8s-master.tar.gz 100% 87MB 15.2MB/s 00:05
[root@k8s-etcd1 src]# scp -r TLS.tar.gz root@192.168.1.124:/usr/local/src/
root@192.168.1.124's password:
TLS.tar.gz 100% 5715KB 15.5MB/s 00:00
[root@k8s-master1 src]# tar xf TLS.tar.gz
[root@k8s-master1 TLS]# ls
cfssl cfssl-certinfo cfssljson cfssl.sh etcd k8s
[root@k8s-master1TLS]# cd k8s/
[root@k8s-master1k8s]# ls
ca-config.json ca-csr.json generate_k8s_cert.sh kube-proxy-csr.json server-csr.json
[root@k8s-master1 src]# cat TLS/k8s/server-csr.json
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"192.168.1.120",
"192.168.1.121",
"192.168.1.122",
"192.168.1.123",
"192.168.1.124",
"192.168.1.125",
"192.168.1.126",
"192.168.1.127",
"192.168.1.128",
"192.168.1.129",
"192.168.1.130",
"192.168.1.131",
"192.168.1.132",
"192.168.1.133"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
[root@k8s-master1 src]# cd TLS
[root@k8s-master1 TLS]# ls
cfssl cfssl-certinfo cfssljson cfssl.sh etcd k8s
[root@k8s-master1 TLS]# bash -x cfssl.sh
+ cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin
+ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssl-certinfo /usr/local/bin/cfssljson
[root@k8s-master1 TLS]# cd k8s/
[root@k8s-master1 k8s]# ls
ca-config.json ca-csr.json generate_k8s_cert.sh kube-proxy-csr.json server-csr.json
[root@k8s-master1 k8s]# ./generate_k8s_cert.sh
2020/01/29 19:48:44 [INFO] generating a new CA key and certificate from CSR
2020/01/29 19:48:44 [INFO] generate received request
2020/01/29 19:48:44 [INFO] received CSR
2020/01/29 19:48:44 [INFO] generating key: rsa-2048
2020/01/29 19:48:46 [INFO] encoded CSR
2020/01/29 19:48:46 [INFO] signed certificate with serial number 644734139928112787467618196045542040345765034825
2020/01/29 19:48:46 [INFO] generate received request
2020/01/29 19:48:46 [INFO] received CSR
2020/01/29 19:48:46 [INFO] generating key: rsa-2048
2020/01/29 19:48:49 [INFO] encoded CSR
2020/01/29 19:48:49 [INFO] signed certificate with serial number 641751437813294970674608941868096756464221965513
2020/01/29 19:48:49 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
2020/01/29 19:48:49 [INFO] generate received request
2020/01/29 19:48:49 [INFO] received CSR
2020/01/29 19:48:49 [INFO] generating key: rsa-2048
2020/01/29 19:48:51 [INFO] encoded CSR
2020/01/29 19:48:51 [INFO] signed certificate with serial number 573406333537228053382286849722926605568836533693
2020/01/29 19:48:51 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-master1 k8s]# ls
ca-config.json ca-csr.json ca.pem kube-proxy.csr kube-proxy-key.pem server.csr server-key.pem
ca.csr ca-key.pem generate_k8s_cert.sh kube-proxy-csr.json kube-proxy.pem server-csr.json server.pem
k8s-master集群需要用到etcd的ssl验证文件。所以要将etcd节点上面的etcd目录拷贝到k8s-master下面的/opt目录下，这是有kubernetes的配置文件决定的。
[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.124:/opt/
root@192.168.1.124's password:
etcd 100% 16MB 16.3MB/s 00:00
etcdctl 100% 13MB 19.6MB/s 00:00
etcd.conf 100% 510 103.5KB/s 00:00
ca.pem 100% 1265 424.0KB/s 00:00
server.pem 100% 1338 426.6KB/s 00:00
server-key.pem 100% 1675 441.4KB/s 00:00
[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.125:/opt/
root@192.168.1.125's password:
etcd 100% 16MB 16.4MB/s 00:00
etcdctl 100% 13MB 12.9MB/s 00:01
etcd.conf 100% 510 92.4KB/s 00:00
ca.pem 100% 1265 273.2KB/s 00:00
server.pem 100% 1338 202.1KB/s 00:00
server-key.pem 100% 1675 394.4KB/s 00:00
[root@k8s-etcd1 opt]# scp -r etcd root@192.168.1.126:/opt/
root@192.168.1.126's password:
etcd 100% 16MB 16.1MB/s 00:01
etcdctl 100% 13MB 12.9MB/s 00:01
etcd.conf 100% 510 84.2KB/s 00:00
ca.pem 100% 1265 348.9KB/s 00:00
server.pem 100% 1338 251.2KB/s 00:00
server-key.pem 100% 1675 213.2KB/s 00:00
[root@k8s-master1 src]# ls
k8s-master.tar.gz kube-apiserver.service kube-controller-manager.service kubernetes kube-scheduler.service
[root@k8s-master1 src]# \cp -r kubernetes /opt/
将生成的配置文件拷贝到/opt/kubernetes/ssl/目录下
[root@k8s-master1 k8s]# cp ca-key.pem ca.pem server-key.pem server.pem /opt/kubernetes/ssl/
: overwrite ‘/opt/kubernetes/ssl/ca-key.pem’? y
cp: overwrite ‘/opt/kubernetes/ssl/ca.pem’? y
[root@k8s-master1 k8s]# cd /opt/kubernetes/
[root@k8s-master1 kubernetes]# ls
bin cfg logs ssl
[root@k8s-master1 kubernetes]# cd ssl/
[root@k8s-master1 ssl]# ls
ca-key.pem ca.pem server-key.pem server.pem
[root@k8s-master1 ssl]# ll
total 16
-rw------- 1 root root 1679 Jan 29 20:00 ca-key.pem
-rw-r--r-- 1 root root 1359 Jan 29 20:00 ca.pem
-rw------- 1 root root 1679 Jan 29 20:00 server-key.pem
-rw-r--r-- 1 root root 1716 Jan 29 20:00 server.pem
修改Kubernetes的配置文件kube-apiserver.conf
[root@k8s-master1 cfg]# cat kube-apiserver.conf
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 \ #etcd集群的地址和端口
--bind-address=192.168.1.124 \ #本机地址
--secure-port=6443 \
--advertise-address=192.168.1.124 \ #本机地址
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \
--tls-cert-file=/opt/kubernetes/ssl/server.pem \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
[root@k8s-master1 opt]# scp -r kubernetes root@192.168.1.125:/opt/
root@192.168.1.125's password:
kubectl 100% 45MB 16.1MB/s 00:02
kube-apiserver 100% 167MB 18.5MB/s 00:09
kube-controller-manager 100% 115MB 14.4MB/s 00:08
kube-scheduler 100% 43MB 12.5MB/s 00:03
token.csv 100% 83 20.2KB/s 00:00
kube-controller-manager.conf 100% 546 98.2KB/s 00:00
kube-scheduler.conf 100% 148 20.4KB/s 00:00
kube-apiserver.conf 100% 1193 252.7KB/s 00:00
ca-key.pem 100% 1679 337.2KB/s 00:00
ca.pem 100% 1359 297.2KB/s 00:00
server-key.pem 100% 1679 289.3KB/s 00:00
server.pem 100% 1716 292.5KB/s 00:00
[root@k8s-master1 opt]# scp -r kubernetes root@192.168.1.126:/opt/
root@192.168.1.126's password:
kubectl 100% 45MB 14.8MB/s 00:03
kube-apiserver 100% 167MB 16.7MB/s 00:10
kube-controller-manager 100% 115MB 14.4MB/s 00:08
kube-scheduler 100% 43MB 14.4MB/s 00:02
token.csv 100% 83 22.3KB/s 00:00
kube-controller-manager.conf 100% 546 95.2KB/s 00:00
kube-scheduler.conf 100% 148 35.0KB/s 00:00
kube-apiserver.conf 100% 1193 176.9KB/s 00:00
ca-key.pem 100% 1679 340.2KB/s 00:00
ca.pem 100% 1359 266.1KB/s 00:00
server-key.pem 100% 1679 298.4KB/s 00:00
server.pem 100% 1716 331.6KB/s 00:00
修改k8s-master2 的配置文件kube-apiserver.conf
[root@k8s-master2 cfg]# cat kube-apiserver.conf
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 \
--bind-address=192.168.1.125 \
--secure-port=6443 \
--advertise-address=192.168.1.125 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \
--tls-cert-file=/opt/kubernetes/ssl/server.pem \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
修改k8s-master3 的配置文件kube-apiserver.conf
[root@k8s-master3 cfg]# cat kube-apiserver.conf
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/opt/kubernetes/logs \
--etcd-servers=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 \
--bind-address=192.168.1.126 \
--secure-port=6443 \
--advertise-address=192.168.1.126 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \
--tls-cert-file=/opt/kubernetes/ssl/server.pem \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
[root@k8s-master1 src]# cp *.service /usr/lib/systemd/system/
[root@k8s-master1 src]# scp -r *.service root@192.168.1.125:/usr/lib/systemd/system/
The authenticity of host '192.168.1.125 (192.168.1.125)' can't be established.
ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.
ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.125' (ECDSA) to the list of known hosts.
root@192.168.1.125's password:
kube-apiserver.service 100% 286 67.9KB/s 00:00
kube-controller-manager.service 100% 321 68.1KB/s 00:00
kube-scheduler.service 100% 285 46.3KB/s 00:00
[root@k8s-master1 src]# scp -r *.service root@192.168.1.126:/usr/lib/systemd/system/
The authenticity of host '192.168.1.126 (192.168.1.126)' can't be established.
ECDSA key fingerprint is SHA256:7NT7mqzyv1a+w46h8OQMxZELZaWGOsEHDOBVjCKt6Jc.
ECDSA key fingerprint is MD5:b5:05:57:c3:48:bc:95:9e:28:dc:60:9f:44:96:88:81.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.1.126' (ECDSA) to the list of known hosts.
root@192.168.1.126's password:
kube-apiserver.service 100% 286 57.5KB/s 00:00
kube-controller-manager.service 100% 321 64.6KB/s 00:00
kube-scheduler.service 100% 285 71.7KB/s 00:00
3台机器都将服务加入开机自启动：
[root@k8s-master1 opt]# systemctl enable kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.
[root@k8s-master1 opt]# systemctl enable kube-controller-manager
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.
[root@k8s-master1 opt]# systemctl enable kube-scheduler
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.
[root@k8s-master2 opt]# systemctl enable kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.
[root@k8s-master2 opt]# systemctl enable kube-controller-manager
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.
[root@k8s-master2 opt]# systemctl enable kube-scheduler
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.
[root@k8s-master3 opt]# systemctl enable kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.
[root@k8s-master3 opt]# systemctl enable kube-controller-manager
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.
[root@k8s-master3 opt]# systemctl enable kube-scheduler
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.
[root@k8s-master1 cfg]# cp /opt/kubernetes/bin/kubectl /usr/local/bin/
[root@k8s-master2 cfg]# cp /opt/kubernetes/bin/kubectl /usr/local/bin/
[root@k8s-master3 cfg]# cp /opt/kubernetes/bin/kubectl /usr/local/bin/
启动服务并验证是否启动成功，如出现active (running)则代表启动成功
[root@k8s-master1 opt]# systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler
[root@k8s-master1 opt]# systemctl status kube-apiserver && systemctl status kube-controller-manager && systemctl status kube-scheduler
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 20:26:01 CST; 6s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 12430 (kube-apiserver)
CGroup: /system.slice/kube-apiserver.service
└─12430 /opt/kubernetes/bin/kube-apiserver --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --etcd-servers=https://192.168.1.121:2379,https://192.1...
Jan 29 20:26:01 k8s-master1 systemd[1]: Started Kubernetes API Server.
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 12437 (kube-controller)
CGroup: /system.slice/kube-controller-manager.service
└─12437 /opt/kubernetes/bin/kube-controller-manager --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect=true --master=127.0.0.1:8080 --...
Jan 29 20:26:01 k8s-master1 systemd[1]: Started Kubernetes Controller Manager.
Jan 29 20:26:02 k8s-master1 kube-controller-manager[12437]: Flag --address has been deprecated, see --bind-address instead.
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 12450 (kube-scheduler)
CGroup: /system.slice/kube-scheduler.service
└─12450 /opt/kubernetes/bin/kube-scheduler --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect --master=127.0.0.1:8080 --address=127.0....
Jan 29 20:26:01 k8s-master1 systemd[1]: Started Kubernetes Scheduler.
[root@k8s-master2 opt]# systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler
[root@k8s-master2 opt]# systemctl status kube-apiserver && systemctl status kube-controller-manager && systemctl status kube-scheduler
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 20:26:01 CST; 6s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 12430 (kube-apiserver)
CGroup: /system.slice/kube-apiserver.service
└─12430 /opt/kubernetes/bin/kube-apiserver --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --etcd-servers=https://192.168.1.121:2379,https://192.1...
Jan 29 20:26:01 k8s-master2 systemd[1]: Started Kubernetes API Server.
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 12437 (kube-controller)
CGroup: /system.slice/kube-controller-manager.service
└─12437 /opt/kubernetes/bin/kube-controller-manager --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect=true --master=127.0.0.1:8080 --...
Jan 29 20:26:01 k8s-master2 systemd[1]: Started Kubernetes Controller Manager.
Jan 29 20:26:02 k8s-master2 kube-controller-manager[12437]: Flag --address has been deprecated, see --bind-address instead.
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-29 20:26:01 CST; 5s ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 12450 (kube-scheduler)
CGroup: /system.slice/kube-scheduler.service
└─12450 /opt/kubernetes/bin/kube-scheduler --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect --master=127.0.0.1:8080 --address=127.0....
Jan 29 20:26:01 k8s-master2 systemd[1]: Started Kubernetes Scheduler.
[root@k8s-master3 opt]# systemctl start kube-apiserver && systemctl start kube-controller-manager && systemctl start kube-scheduler
[root@k8s-master3 opt]# systemctl status kube-apiserver && systemctl status kube-controller-manager && systemctl status kube-scheduler
● kube-apiserver.service - Kubernetes API Server