Kubernetes 1.14.1 搭建

一直想写,于是趁着有空写了

环境准备

安装环境为2台 Debian 9.9 KVM

  • 系统配置信息:
系统 内核 ip 类型
Debian 9.9 4.19.0-0.bpo.5-amd64 10.147.20.44 master
Debian 9.9 4.19.0-0.bpo.5-amd64 10.147.20.45 worker
  • docker版本都是18.09.6
curl -fsS https://get.docker.com | bash -s
apt-mark hold docker-ce
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "bip": "172.30.42.1/16",
  "max-concurrent-downloads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "20m",
    "max-file": "2"
  },
  "storage-driver": "overlay2",
  "metrics-addr" : "127.0.0.1:9323",
  "experimental" : true
}
EOF

docker run --rm -v /usr/local/bin:/sysdir spanda/pkg tar zxf /pkg.tgz -C /sysdir

安装kubelet等二进制工具

apt-get update && apt-get install -y apt-transport-https curl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
EOF
apt-get update
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl

etcd部署安装

可以通过etcd labs生成etcd相关配置

创建TLS

安装cfssl

rm -f /tmp/cfssl* && rm -rf /tmp/certs && mkdir -p /tmp/certs

curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /tmp/cfssl
chmod +x /tmp/cfssl
sudo mv /tmp/cfssl /usr/local/bin/cfssl

curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /tmp/cfssljson
chmod +x /tmp/cfssljson
sudo mv /tmp/cfssljson /usr/local/bin/cfssljson

/usr/local/bin/cfssl version
/usr/local/bin/cfssljson -h

mkdir -p /tmp/certs

生成自签名证书

mkdir -p /tmp/certs

cat > /tmp/certs/etcd-root-ca-csr.json <<EOF
{
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "etcd",
      "OU": "etcd Security",
      "L": "San Francisco",
      "ST": "California",
      "C": "USA"
    }
  ],
  "CN": "etcd-root-ca"
}
EOF
cfssl gencert --initca=true /tmp/certs/etcd-root-ca-csr.json | cfssljson --bare /tmp/certs/etcd-root-ca

# verify
openssl x509 -in /tmp/certs/etcd-root-ca.pem -text -noout

# cert-generation configuration
cat > /tmp/certs/etcd-gencert.json <<EOF
{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "876000h"
    }
  }
}
EOF

cat > /tmp/certs/etcd-ca-csr.json <<EOF
{
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "etcd",
      "OU": "etcd Security",
      "L": "San Francisco",
      "ST": "California",
      "C": "USA"
    }
  ],
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "localhost",
    "10.147.20.44",
    "etcd.ns.godu.dev",
    "*.etcd.ns.godu.dev"
  ]
}
EOF
cfssl gencert \
  --ca /tmp/certs/etcd-root-ca.pem \
  --ca-key /tmp/certs/etcd-root-ca-key.pem \
  --config /tmp/certs/etcd-gencert.json \
  /tmp/certs/etcd-ca-csr.json | cfssljson --bare /tmp/certs/etcd

# verify
openssl x509 -in /tmp/certs/etcd.pem -text -noout

运行部署etcd

mkdir -p /etc/etcd/ssl
cp /tmp/certs/* /etc/etcd/ssl

cat > /etc/systemd/system/etcd.service <<EOF
[Unit]
Description=etcd with Docker
Documentation=https://github.com/coreos/etcd

[Service]
Restart=always
RestartSec=5s
TimeoutStartSec=0
LimitNOFILE=40000

ExecStart=/usr/bin/docker \
  run \
  --rm \
  --net=host \
  --name etcd \
  --volume=/data/etcd:/etcd-data \
  --volume=/etc/etcd/ssl:/etcd-ssl-certs-dir \
  gcr.io/etcd-development/etcd:v3.3.8 \
  /usr/local/bin/etcd \
  --name etcd \
  --data-dir /etcd-data \
  --listen-client-urls https://10.147.20.44:2379,http://127.0.0.1:2379 \
  --advertise-client-urls https://10.147.20.44:2379 \
  --listen-peer-urls https://10.147.20.44:2380 \
  --initial-advertise-peer-urls https://10.147.20.44:2380 \
  --initial-cluster etcd=https://10.147.20.44:2380 \
  --initial-cluster-token godu \
  --initial-cluster-state new \
  --auto-compaction-retention 1 \
  --client-cert-auth \
  --trusted-ca-file /etcd-ssl-certs-dir/etcd-root-ca.pem \
  --cert-file /etcd-ssl-certs-dir/etcd.pem \
  --key-file /etcd-ssl-certs-dir/etcd-key.pem \
  --peer-client-cert-auth \
  --peer-trusted-ca-file /etcd-ssl-certs-dir/etcd-root-ca.pem \
  --peer-cert-file /etcd-ssl-certs-dir/etcd.pem \
  --peer-key-file /etcd-ssl-certs-dir/etcd-key.pem \
  --enable-pprof \
  --debug

ExecStop=/usr/bin/docker stop etcd

[Install]
WantedBy=multi-user.target
EOF

# start service
systemctl daemon-reload
systemctl enable etcd.service
systemctl start etcd.service

验证

ETCDCTL_API=3 etcdctl endpoint health

ETCDCTL_API=3 etcdctl --endpoints https://10.147.20.44:2379  --cacert /etc/etcd/ssl/etcd-root-ca.pem --cert /etc/etcd/ssl/etcd.pem --key /etc/etcd/ssl/etcd-key.pem endpoint health

安装 Kubernetes

创建TLS

mkdir -pv /tmp/k8s/ssl
# CA
cat > /tmp/k8s/ssl/k8s-root-ca-csr.json <<EOF
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 4096
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "kubernetes",
            "OU": "System"
        }
    ],
    "ca": {
        "expiry": "876000h"
    }
}
EOF

cat > /tmp/k8s/ssl/k8s-gencert.json <<EOF
{
    "signing": {
        "default": {
            "expiry": "876000h"
        },
        "profiles": {
            "kubernetes": {
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ],
                "expiry": "876000h"
            }
        }
    }
}
EOF

cat > /tmp/k8s/ssl/kube-apiserver-csr.json <<EOF
{
    "CN": "kubernetes",
    "hosts": [
        "127.0.0.1",
        "10.254.0.1",
        "10.147.20.44",
        "localhost",
        "k8s.ns.godu.dev",
        "*.k8s.ns.godu.dev",
        "kubernetes",
        "kubernetes.default",
        "kubernetes.default.svc",
        "kubernetes.default.svc.cluster",
        "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "kubernetes",
            "OU": "System"
        }
    ]
}
EOF

cat > /tmp/k8s/ssl/kube-controller-manager-csr.json <<EOF
{
  "CN": "system:kube-controller-manager",
  "hosts": [
    "127.0.0.1",
    "localhost",
    "10.147.20.44",
    "k8s.ns.godu.dev",
    "*.k8s.ns.godu.dev"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:kube-controller-manager",
      "OU": "System"
    }
  ]
}
EOF

cat > /tmp/k8s/ssl/kube-scheduler-csr.json <<EOF
{
  "CN": "system:kube-scheduler",
  "hosts": [
    "127.0.0.1",
    "localhost",
    "10.147.20.44",
    "k8s.ns.godu.dev",
    "*.k8s.ns.godu.dev"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "system:kube-scheduler",
      "OU": "System"
    }
  ]
}
EOF

cat > /tmp/k8s/ssl/kube-proxy-csr.json <<EOF
{
    "CN": "system:kube-proxy",
    "hosts": [],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "system:kube-proxy",
            "OU": "System"
        }
    ]
}
EOF

cat > /tmp/k8s/ssl/kubelet-api-admin-csr.json <<EOF
{
    "CN": "system:kubelet-api-admin",
    "hosts": [],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "system:kubelet-api-admin",
            "OU": "System"
        }
    ]
}
EOF

cat > /tmp/k8s/ssl/admin-csr.json <<EOF
{
    "CN": "system:masters",
    "hosts": [],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "system:masters",
            "OU": "System"
        }
    ]
}
EOF

cd /tmp/k8s/ssl

cfssl gencert --initca=true k8s-root-ca-csr.json | cfssljson --bare k8s-root-ca

for targetName in kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet-api-admin admin; do
    cfssl gencert --ca k8s-root-ca.pem --ca-key k8s-root-ca-key.pem --config k8s-gencert.json --profile kubernetes $targetName-csr.json | cfssljson --bare $targetName
done

后面有空在写, 机器有问题 😢😢

推荐阅读