momo's Blog.

ETCD集群安装

字数统计: 1.2k阅读时长: 7 min
2020/03/18 Share

一、 生成TLS证书

1.1 下载 cfssl

  • 请自行去github下载相同版本的cfsslcfssl
    1
    2
    3
    mv cfssl_1.4.1_linux_amd64 /bin/cfssl
    mv cfssljson_1.4.1_linux_amd64 /bin/cfssljson
    mv cfssl-certinfo_1.4.1_linux_amd64 /bin/cfssl-certinfo

systemctl disable firewalld && systemctl stop firewalld && sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

1.2 初始化证书颁发机构(CA)

  • CA中心配置

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    {
    "signing": {
    "default": {
    "expiry": "87600h"
    },
    "profiles": {
    "peer": {
    "usages": [
    "signing",
    "key encipherment",
    "server auth",
    "client auth"
    ],
    "expiry": "87600h"
    },
    "client": {
    "usages": [
    "signing",
    "key encipherment",
    "client auth"
    ],
    "expiry": "87600h"
    },
    "server": {
    "usages": [
    "signing",
    "key encipherment",
    "server auth"
    ],
    "expiry": "87600h"
    }
    }
    }
    }
  • ETCD CA配置 etcd-root-ca.json

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    {
    "CN": "etcd-root-ca",
    "key": {
    "algo": "rsa",
    "size": 2048
    },
    "names": [
    {
    "C": "CN",
    "ST": "Shanhai",
    "L": "Shanhai",
    "O": "Mu77Mu77",
    "OU": "ops"
    }
    ]
    }
  • ETCD双向认证CA etcd-peer-ca.json

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    {
    "CN": "etcd",
    "hosts": [
    "127.0.0.1",
    "192.168.1.20",
    "192.168.1.23",
    "192.168.1.26",
    "k001",
    "k002",
    "k003"
    ],
    "key": {
    "algo": "rsa",
    "size": 2048
    },
    "names": [
    {
    "C": "CN",
    "ST": "Shanhai",
    "L": "Shanhai",
    "O": "Mu77Mu77",
    "OU": "ops"
    }
    ]
    }
  • ETCD客户端证书 etcd-client-ca.json

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    {
    "CN": "etcd-client",
    "hosts": [
    "127.0.0.1",
    "192.168.1.20",
    "192.168.1.23",
    "192.168.1.26",
    "k001",
    "k002",
    "k003"
    ],
    "key": {
    "algo": "rsa",
    "size": 2048
    }
    }
  • 生成根证书

    1
    cfssl gencert  -initca etcd-root-ca.json | cfssljson -bare etcd-root-ca
  • 生成服务端对等证书

    1
    cfssl gencert --ca etcd-root-ca.pem --ca-key etcd-root-ca-key.pem --config ca-config.json -profile=peer etcd-peer-ca.json | cfssljson --bare etcd
  • 生成客户端证书

    1
    cfssl gencert --ca etcd-root-ca.pem --ca-key etcd-root-ca-key.pem --config ca-config.json -profile=client etcd-client-ca.json | cfssljson --bare etcd-client

二、安装ETCD

2.1 etcd 下载

  • 下载etcd
    1
    2
    3
    4
    5
    6
    7
    8
    ETCD_DEFAULT_VERSION="3.4.4"
    wget https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz

    # 复制二进制文件到bin下
    cp etcd-v${ETCD_VERSION}-linux-amd64/etcd* /usr/local/bin

    # 创建配置目录
    mkdir -p /etc/etcd/ssl

2.2 etcd配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# [member]
ETCD_NAME=etcd1
ETCD_DATA_DIR="/var/lib/etcd/data"
ETCD_WAL_DIR="/var/lib/etcd/wal"
ETCD_SNAPSHOT_COUNT="100"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="1000"
ETCD_LISTEN_PEER_URLS="https://192.168.1.20:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.20:2379,http://127.0.0.1:2379"
ETCD_MAX_SNAPSHOTS="5"
ETCD_MAX_WALS="5"
#ETCD_CORS=""

# [cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.20:2380"
# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.20:2380,etcd2=https://192.168.1.23:2380,etcd3=https://192.168.1.27:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.20:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_SRV=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_STRICT_RECONFIG_CHECK="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"

# [proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"

# [security]
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-root-ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-root-ca.pem"
ETCD_PEER_AUTO_TLS="true"

# [logging]
#ETCD_DEBUG="false"
# examples for -log-package-levels etcdserver=WARNING,security=DEBUG
#ETCD_LOG_PACKAGE_LEVELS=""
  • etcd.service

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    [Unit]
    Description=Etcd Server
    After=network.target
    After=network-online.target
    Wants=network-online.target

    [Service]
    Type=notify
    WorkingDirectory=/var/lib/etcd/
    EnvironmentFile=-/etc/etcd/config
    User=etcd
    # set GOMAXPROCS to number of processors
    ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/local/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
    Restart=on-failure
    LimitNOFILE=65536

    [Install]
    WantedBy=multi-user.target
  • 复制配置文件

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    # 复制之前生成的证书文件

    # 注意:此次开启了PEER_AUTO_TLS="true",则不在需要客户端证书
    cp *.pem /etc/etcd/ssl/

    # 复制配置文件
    vi /etc/etcd/config
    vi /lib/systemd/system/etcd.service
    systemctl daemon-reload
    systemctl status etcd

    # 创建ETCD用户
    groupadd -r etcd && useradd -r -g etcd -d /var/lib/etcd -s /sbin/nologin -c "etcd user" etcd

    # 创建数据目录,如需要修改请修改配置文件
    mkdir /var/lib/etcd && chown -R etcd:etcd /var/lib/etcd

    # 权限配置
    chown -R etcd:etcd /etc/etcd && chmod -R 755 /etc/etcd/ssl
  • 以上操作在其他2台节点上同步

1
2
3
4
5
6
7
8
9
10
11
12
13
# 创建用户,目录
for i in 23 27;do ssh root@192.168.1.${i} "mkdir -p /etc/etcd/ssl/ && mkdir /var/lib/etcd && chown -R etcd:etcd /var/lib/etcd";done;

# 复制etcd执行文件
for i in 23 27;do scp /usr/local/bin/etcd* root@192.168.1.${i}:/usr/local/bin/;done;

# 复制配置文件
for i in 23 27;do scp /etc/etcd/config root@192.168.1.${i}:/etc/etcd;done;
for i in 23 27;do scp /etc/etcd/ssl/* root@192.168.1.${i}:/etc/etcd/ssl;done;
for i in 23 27;do scp /lib/systemd/system/etcd.service root@192.168.1.${i}:/lib/systemd/system;done;

# 权限设置
for i in 23 27;do ssh root@192.168.1.${i} "chown -R etcd:etcd /etc/etcd && chmod -R 755 /etc/etcd/ssl";done;
  • 修改配置文件

    1
    修改 ${NAME} 以及对应的IP地址即可
  • 启动验证

IS LEARNER 字段是etcd 3.4 以后的新特性,后续节点将以学习者的身份加入集群,学习者没有投票权,直至同步完成leader所有的日志以后可以提权,官方文档链接: etcd.io

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
etcdctl --cacert /etc/etcd/ssl/ca.pem --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem --write-out=table member list
+------------------+---------+-------+---------------------------+---------------------------+------------+
| ID | STATUS | NAME | PEER ADDRS | CLIENT ADDRS | IS LEARNER |
+------------------+---------+-------+---------------------------+---------------------------+------------+
| 24de4a5dd6970656 | started | etcd3 | https://192.168.1.27:2380 | https://192.168.1.27:2379 | false |
| 5a224bcd35cc7d02 | started | etcd1 | https://192.168.1.20:2380 | https://192.168.1.20:2379 | false |
| 86ab7347ef3517f6 | started | etcd2 | https://192.168.1.23:2380 | https://192.168.1.23:2379 | false |
+------------------+---------+-------+---------------------------+---------------------------+------------+






etcdctl --cacert /etc/etcd/ssl/etcd-root-ca.pem --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem --endpoints=https://192.168.1.20:2379,https://192.168.1.23:2379,https://192.168.1.27:2379 endpoint health
https://192.168.1.27:2379 is healthy: successfully committed proposal: took = 6.002107ms
https://192.168.1.20:2379 is healthy: successfully committed proposal: took = 6.318034ms
https://192.168.1.23:2379 is healthy: successfully committed proposal: took = 7.365734ms
CATALOG
  1. 1. 一、 生成TLS证书
    1. 1.1. 1.1 下载 cfssl
    2. 1.2. 1.2 初始化证书颁发机构(CA)
  2. 2. 二、安装ETCD
    1. 2.1. 2.1 etcd 下载
    2. 2.2. 2.2 etcd配置