#!/usr/bin/env bash DAYS=${DAYS:-"3650"} ADMIN_TOKEN=${ADMIN_TOKEN:-""} RSA_KEY_BITS=${RSA_KEY_BITS:-"2048"} KUBERNETES_PATH=${KUBERNETES_PATH:-"/etc/kubernetes"} NODE_OUTPUT=${NODE_OUTPUT:-"/tmp/kuberntes_node"} K8S_CA_CRT_PATH="${KUBERNETES_PATH}/pki/ca.crt" K8S_CA_KEY_PATH="${KUBERNETES_PATH}/pki/ca.key" ETCD_CA_CRT_PATH="${KUBERNETES_PATH}/pki/etcd/ca.crt" ETCD_CA_KEY_PATH="${KUBERNETES_PATH}/pki/etcd/ca.key" FRONT_PROXY_CA_CRT_PATH="${KUBERNETES_PATH}/pki/front-proxy-ca.crt" FRONT_PROXY_CA_KEY_PATH="${KUBERNETES_PATH}/pki/front-proxy-ca.key" VIP=${VIP:-""} KUBELET_CLIENT_CURRENT_PEM_PATH="/var/lib/kubelet/pki/kubelet-client-current.pem" function usage() { echo " $1 master # generate certificates for master $1 node # generate certificates for node " exit -1 } CMD=$1 if [ "${CMD}" == "master" ]; then MASTER_IP=$2 if [ "x${MASTER_IP}" == "x" ]; then usage $0 fi else if [ "${CMD}" == "node" ]; then NODE_NAME=$2 if [ "x${NODE_NAME}" == "x" ]; then usage $0 fi else usage $0 fi fi TMP=$(mktemp -d) function basic_conf() { local usage=$1 echo " [req] req_extensions = v3_req distinguished_name = req_distinguished_name [req_distinguished_name] [v3_req] keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = ${usage} " } function gen() { local subj=$1 local crt_path=$2 local key_path=$3 local conf_path=$4 local ca_crt_path=$5 local ca_key_path=$6 local csr_path=$(mktemp) openssl genrsa -out "${key_path}" "${RSA_KEY_BITS}" 2>/dev/null openssl req -new -key "${key_path}" -subj "${subj}" -out "${csr_path}" -config "${conf_path}" openssl x509 -req -days "${DAYS}" -in "${csr_path}" -CA "${ca_crt_path}" -CAkey "${ca_key_path}" -CAcreateserial -extensions v3_req -extfile "${conf_path}" -out "${crt_path}" rm -rf "${csr_path}" } function gen_conf() { local user=$1 local org=$2 local cluster_name=$3 local endpoint=$4 local out_path=$5 local key_path=$(mktemp) local crt_path=$(mktemp) local csr_path=$(mktemp) local conf_path=$(mktemp) local subj="/CN=${user}" if [ "x${org}" != "x" ]; then subj="/O=${org}${subj}" fi basic_conf "TLS Web Client Authentication" > "${conf_path}" gen "${subj}" "${crt_path}" "${key_path}" "${conf_path}" "${K8S_CA_CRT_PATH}" "${K8S_CA_KEY_PATH}" echo " apiVersion: v1 clusters: - cluster: certificate-authority-data: $(cat ${K8S_CA_CRT_PATH} | base64 -w 0) server: https://${endpoint} name: ${cluster_name} contexts: - context: cluster: ${cluster_name} user: ${user} name: ${user}@${cluster_name} current-context: ${user}@${cluster_name} kind: Config preferences: {} users: - name: ${user} user: client-certificate-data: $(cat ${crt_path} | base64 -w 0) client-key-data: $(cat ${key_path} | base64 -w 0) " > "${out_path}" rm -rf "${csr_path}" "${key_path}" "${crt_path}" "${conf_path}" } function get_cert_sans() { local certs="" local line="" set +e kubectl --token "${ADMIN_TOKEN}" get cm -n kube-system kubeadm-config -o jsonpath={.data.ClusterConfiguration} 2>/dev/null | while read line; do if [ "${line:0:8}" == "certSANs" ]; then certs=on continue fi if [ "${certs}" == "on" ]; then if [ "${line:0:2}x" == "- x" ]; then echo "${line:2}" else certs=off fi fi done set -e } function gen_apiserver_crt_key() { local key_path="${KUBERNETES_PATH}/pki/apiserver.key" local crt_path="${KUBERNETES_PATH}/pki/apiserver.crt" local conf_path="${TMP}/kube-apiserver.conf" local alt_names=$(get_cert_sans) if [ -f "${crt_path}" ]; then local t=$(openssl x509 -in "${crt_path}" -noout -text | grep -A1 'Subject Alternative Name' | tail -n1 | tr -d ',' | sed 's|DNS:||g' | sed 's|IP Address:||g') local file_alt_names=$(for n in ${t}; do echo $n; done) alt_names=$(echo "${alt_names} ${file_alt_names} kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local localhost 127.0.0.1 " | sort -u) fi local dns="" local ip="" for n in $alt_names; do if [[ ${n} =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then ip="${ip} ${n}" continue fi if [[ ${n} =~ ":" ]]; then ip="${ip} ${n}" continue fi dns="${dns} ${n}" done basic_conf "TLS Web Server Authentication" > "${conf_path}" echo " subjectAltName = @alt_names [alt_names] " >> "${conf_path}" local i=1 for n in ${dns}; do echo "DNS.${i} = ${n}" >> "${conf_path}" i=$((${i}+1)) done i=1 for n in ${ip}; do echo "IP.${i} = ${n}" >> "${conf_path}" i=$((${i}+1)) done gen "/CN=kube-apiserver" "${crt_path}" "${key_path}" "${conf_path}" "${K8S_CA_CRT_PATH}" "${K8S_CA_KEY_PATH}" } function gen_apiserver_kubelet_client_crt_key() { local key_path="${KUBERNETES_PATH}/pki/apiserver-kubelet-client.key" local crt_path="${KUBERNETES_PATH}/pki/apiserver-kubelet-client.crt" local conf_path="${TMP}/apiserver-kubelet-client.conf" basic_conf "TLS Web Client Authentication" > "${conf_path}" gen "/O=system:masters/CN=kube-apiserver-kubelet-client" "${crt_path}" "${key_path}" "${conf_path}" "${K8S_CA_CRT_PATH}" "${K8S_CA_KEY_PATH}" } function gen_apiserver_etcd_client_crt_key() { local key_path="${KUBERNETES_PATH}/pki/apiserver-etcd-client.key" local crt_path="${KUBERNETES_PATH}/pki/apiserver-etcd-client.crt" local conf_path="${TMP}/apiserver-etcd-client.conf" basic_conf "TLS Web Client Authentication" > "${conf_path}" gen "/O=system:masters/CN=kube-apiserver-etcd-client" "${crt_path}" "${key_path}" "${conf_path}" "${ETCD_CA_CRT_PATH}" "${ETCD_CA_KEY_PATH}" } function gen_front_proxy_client_crt_key() { local key_path="${KUBERNETES_PATH}/pki/front-proxy-client.key" local crt_path="${KUBERNETES_PATH}/pki/front-proxy-client.crt" local conf_path="${TMP}/front-proxy-client.conf" basic_conf "TLS Web Client Authentication" > "${conf_path}" gen "/CN=front-proxy-client" "${crt_path}" "${key_path}" "${conf_path}" "${FRONT_PROXY_CA_CRT_PATH}" "${FRONT_PROXY_CA_KEY_PATH}" } function gen_etcd_healthcheck_client_crt_key() { local key_path="${KUBERNETES_PATH}/pki/etcd/healthcheck-client.key" local crt_path="${KUBERNETES_PATH}/pki/etcd/healthcheck-client.crt" local conf_path="${TMP}/healthcheck-client.conf" basic_conf "TLS Web Client Authentication" > "${conf_path}" gen "/O=system:masters/CN=kube-etcd-healthcheck-client" "${crt_path}" "${key_path}" "${conf_path}" "${ETCD_CA_CRT_PATH}" "${ETCD_CA_KEY_PATH}" } function gen_etcd_peer_crt_key() { local ip=$1 local key_path="${KUBERNETES_PATH}/pki/etcd/peer.key" local crt_path="${KUBERNETES_PATH}/pki/etcd/peer.crt" local conf_path="${TMP}/peer.conf" basic_conf "TLS Web Client Authentication, TLS Web Server Authentication" > "${conf_path}" echo " subjectAltName = @alt_names [alt_names] DNS.1 = ${ip} DNS.2 = localhost IP.1 = ${ip} IP.2 = 127.0.0.1 IP.3 = ::1 " >> "${conf_path}" gen "/CN=${ip}" "${crt_path}" "${key_path}" "${conf_path}" "${ETCD_CA_CRT_PATH}" "${ETCD_CA_KEY_PATH}" } function gen_etcd_server_crt_key() { local ip=$1 local key_path="${KUBERNETES_PATH}/pki/etcd/server.key" local crt_path="${KUBERNETES_PATH}/pki/etcd/server.crt" local conf_path="${TMP}/server.conf" local vip="${VIP}" if [ "x${vip}" == "x" ]; then vip=$(kubectl --token "${ADMIN_TOKEN}" get cm -n kube-system kubeadm-config -o jsonpath={.data.ClusterConfiguration} | grep controlPlaneEndpoint | tr ':' ' ' |awk '{print $2}') if [ "x${vip}" == "x" ]; then vip="127.0.0.1" fi fi basic_conf "TLS Web Server Authentication, TLS Web Client Authentication" > "${conf_path}" echo " subjectAltName = @alt_names [alt_names] DNS.1 = ${ip} DNS.2 = etcd DNS.3 = etcd.kube-system DNS.4 = localhost DNS.5 = ${vip} IP.1 = 127.0.0.1 IP.2 = ::1 IP.3 = ${ip} IP.4 = ${vip} " >> "${conf_path}" if [ "x${vip}" != "x" ]; then echo "IP.4 = ${vip}" >> "${conf_path}" fi gen "/CN=${ip}" "${crt_path}" "${key_path}" "${conf_path}" "${ETCD_CA_CRT_PATH}" "${ETCD_CA_KEY_PATH}" } function gen_controller_manager_conf() { local ip=$1 local out_path="${KUBERNETES_PATH}/controller-manager.conf" local cluster_name=$(cat "${out_path}" | grep " cluster:" | tr ':' ' ' | awk '{print $2}') if [ "x${cluster_name}" == "x" ]; then cluster_name="default" fi gen_conf "system:kube-controller-manager" "" "${cluster_name}" "${ip}:6443" "${out_path}" } function gen_scheduler_conf() { local ip=$1 local out_path="${KUBERNETES_PATH}/scheduler.conf" local cluster_name=$(cat "${out_path}" | grep " cluster:" | tr ':' ' ' | awk '{print $2}') if [ "x${cluster_name}" == "x" ]; then cluster_name="default" fi gen_conf "system:kube-scheduler" "" "${cluster_name}" "${ip}:6443" "${out_path}" } function gen_admin_conf() { local out_path="${KUBERNETES_PATH}/admin.conf" local endpoint=$(kubectl --token "${ADMIN_TOKEN}" get cm -n kube-system kubeadm-config -o jsonpath={.data.ClusterConfiguration} | grep controlPlaneEndpoint |awk '{print $2}') local cluster_name=$(cat "${out_path}" | grep " cluster:" | tr ':' ' ' | awk '{print $2}') if [ "x${cluster_name}" == "x" ]; then cluster_name="default" fi if [ "x${endpoint}" == "x" ]; then endpoint="127.0.0.1:6443" fi gen_conf "kubernetes-admin" "system:masters" "${cluster_name}" "${endpoint}" "${out_path}" local kubeconfig="${HOME}/.kube/config" mkdir -p $(dirname "${kubeconfig}") gen_conf "admin" "system:masters" "${cluster_name}" "${endpoint}" "${kubeconfig}" } #-------- node --------- function gen_kubelet_crt_key() { local node_name=$1 local node_ip=$2 local key_path="${NODE_OUTPUT}/${node_name}/kubelet.key" local crt_path="${NODE_OUTPUT}/${node_name}/kubelet.crt" local conf_path=$(mktemp) local alt_names=$(echo " ${node_name} ${node_ip} " | sort -u) basic_conf "TLS Web Server Authentication, TLS Web Client Authentication" > "${conf_path}" echo " subjectAltName = @alt_names [alt_names] " >> "${conf_path}" local dns_idx=1 local ip_idx=1 for n in $alt_names; do if [[ ${n} =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then echo "IP.${ip_idx} = ${n}" >> "${conf_path}" ip_idx=$((${ip_idx}+1)) continue fi if [[ ${n} =~ ":" ]]; then echo "IP.${ip_idx} = ${n}" >> "${conf_path}" ip_idx=$((${ip_idx}+1)) continue fi echo "DNS.${dns_idx} = ${n}" >> "${conf_path}" dns_idx=$((${dns_idx}+1)) done mkdir -p "${NODE_OUTPUT}/${node_name}" gen "/CN=kubelet" "${crt_path}" "${key_path}" "${conf_path}" "${K8S_CA_CRT_PATH}" "${K8S_CA_KEY_PATH}" rm -rf "${conf_path}" } function gen_kubelet_conf() { local node_name=$1 local key_path="$(mktemp)" local crt_path="$(mktemp)" local conf_path="$(mktemp)" local out_path="${NODE_OUTPUT}/${node_name}/kubelet.conf" local pem_path="${NODE_OUTPUT}/${node_name}/kubelet-client-current.pem" local endpoint=$(kubectl --token "${ADMIN_TOKEN}" get cm -n kube-system kubeadm-config -o jsonpath={.data.ClusterConfiguration} | grep controlPlaneEndpoint |awk '{print $2}') basic_conf "TLS Web Client Authentication" > "${conf_path}" mkdir -p "${NODE_OUTPUT}/${node_name}" gen "/O=system:nodes/CN=system:node:${node_name}" "${crt_path}" "${key_path}" "${conf_path}" "${K8S_CA_CRT_PATH}" "${K8S_CA_KEY_PATH}" cat "${crt_path}" > "${pem_path}" cat "${key_path}" >> ${pem_path} echo " apiVersion: v1 clusters: - cluster: certificate-authority-data: $(cat ${K8S_CA_CRT_PATH} | base64 -w 0) server: https://${endpoint} name: default-cluster contexts: - context: cluster: default-cluster namespace: default user: default-auth name: default-context current-context: default-context kind: Config preferences: {} users: - name: default-auth user: client-certificate: ${KUBELET_CLIENT_CURRENT_PEM_PATH} client-key: ${KUBELET_CLIENT_CURRENT_PEM_PATH} " > "${out_path}" rm -rf "${key_path}" "${crt_path}" "${conf_path}" } function backup() { local backup_dir="/tmp/kuernetes_backup_$(date +%Y%m%d_%H%M%S)" mkdir -p "${backup_dir}" cp -r "${KUBERNETES_PATH}/pki" "${backup_dir}/pki" cp -r "${KUBERNETES_PATH}"/*.conf "${backup_dir}/" echo "Backup Dir: ${backup_dir}" } function gen_master() { local ip=$1 set +e kubectl --token "${ADMIN_TOKEN}" get cm -n kube-system kubeadm-config -o jsonpath={.data.ClusterConfiguration} | grep controlPlaneEndpoint set -e gen_admin_conf gen_apiserver_crt_key gen_apiserver_kubelet_client_crt_key gen_apiserver_etcd_client_crt_key gen_front_proxy_client_crt_key gen_etcd_healthcheck_client_crt_key gen_etcd_peer_crt_key "${ip}" gen_etcd_server_crt_key "${ip}" gen_controller_manager_conf "${ip}" gen_scheduler_conf "${ip}" } function gen_node() { local node_name=$1 kubectl --token "${ADMIN_TOKEN}" get nodes "${node_name}" local address=$(kubectl --token "${ADMIN_TOKEN}" get nodes "${node_name}" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') mkdir -p "${NODE_OUTPUT}/${node_name}" gen_kubelet_conf "${node_name}" gen_kubelet_crt_key "${node_name}" "${address}" echo "Output Dir: ${NODE_OUTPUT}/${node_name}" echo " move kubelet.crt and kubelet.key to /etc/kubernetes/pki/ move kubelet.conf to /etc/kubernetes/ move kubelet-client-current.pem to /var/lib/kubelet/pki/ " > "${NODE_OUTPUT}/${node_name}/readme.txt" } if [ "${CMD}" == "master" ]; then ip address | grep "inet " | grep "${MASTER_IP}" >/dev/null if [ "$?" -ne "0" ]; then echo "cannot find ip ${MASTER_IP} on this host" exit -2 fi set -e backup gen_master "${MASTER_IP}" rm -rf "${TMP}" echo "Success!!!" fi if [ "${CMD}" == "node" ]; then set -e gen_node "${NODE_NAME}" echo "Success!!!" fi