手动部署 kubernetes 1.9 记录

[root@master01 ~]# cat /etc/kubernetes/apiserver # kubernetes system config## The following values are used to configure the kube-apiserver# # The address on the local server to listen to.KUBE_API_ADDRESS="--advertise-address=192.168.133.128 --insecure-bind-address=127.0.0.1 --bind-address=192.168.133.128" # The port on the local server to listen on.KUBE_API_PORT="--insecure-port=8080 --secure-port=6443" # Port minions listen on# KUBELET_PORT="--kubelet-port=10250" # Comma separated list of nodes in the etcd clusterKUBE_ETCD_SERVERS="--etcd-servers=https://192.168.133.128:2379,https://192.168.133.129:2379,https://192.168.133.130:2379" # Address range to use for servicesKUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16" # default admission control policiesKUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,NodeRestriction"# Add your own!KUBE_API_ARGS="--authorization-mode=RBAC,Node ???????????????--anonymous-auth=false ???????????????--kubelet-https=true ???????????????--enable-bootstrap-token-auth ???????????????--token-auth-file=/etc/kubernetes/ssl/token.csv ???????????????--service-node-port-range=30000-50000 ???????????????--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem ???????????????--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem ???????????????--client-ca-file=/etc/kubernetes/ssl/k8s-root-ca.pem ???????????????--service-account-key-file=/etc/kubernetes/ssl/k8s-root-ca.pem ???????????????--audit-policy-file=/etc/kubernetes/ssl/audit-policy.yaml ???????????????--etcd-quorum-read=true ???????????????--storage-backend=etcd3 ???????????????--etcd-cafile=/etc/etcd/ssl/etcd-root-ca.pem ???????????????--etcd-certfile=/etc/etcd/ssl/etcd.pem ???????????????--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem ???????????????--etcd-compaction-interval=5m0s ???????????????--enable-swagger-ui=true ???????????????--enable-garbage-collector ???????????????--enable-logs-handler ???????????????--kubelet-timeout=3s ???????????????--apiserver-count=3 ???????????????--audit-log-maxage=30 ???????????????--audit-log-maxbackup=3 ???????????????--audit-log-maxsize=100 ???????????????--audit-log-path=/var/log/kube-audit/audit.log ???????????????--event-ttl=1h ???????????????--enable-swagger-ui ???????????????--log-flush-frequency=5s"

# The following values are used to configure the kubernetes controller-manager# defaults from config and apiserver should be adequate# Add your own!KUBE_CONTROLLER_MANAGER_ARGS="--address=0.0.0.0 ??????????????????????????????--service-cluster-ip-range=10.254.0.0/16 ??????????????????????????????--cluster-name=kubernetes ??????????????????????????????--cluster-signing-cert-file=/etc/kubernetes/ssl/k8s-root-ca.pem ??????????????????????????????--cluster-signing-key-file=/etc/kubernetes/ssl/k8s-root-ca-key.pem ??????????????????????????????--service-account-private-key-file=/etc/kubernetes/ssl/k8s-root-ca-key.pem ??????????????????????????????--controllers=*,bootstrapsigner,tokencleaner ??????????????????????????????--deployment-controller-sync-period=10s ??????????????????????????????--experimental-cluster-signing-duration=86700h0m0s ??????????????????????????????--root-ca-file=/etc/kubernetes/ssl/k8s-root-ca.pem ??????????????????????????????--leader-elect=true ??????????????????????????????--node-monitor-grace-period=40s ??????????????????????????????--node-monitor-period=5s ??????????????????????????????--pod-eviction-timeout=5m0s ??????????????????????????????--feature-gates=RotateKubeletServerCertificate=true"

[root@master01 ~]# cat /etc/kubernetes/scheduler #### kubernetes scheduler config # default config should be adequate # Add your own!KUBE_SCHEDULER_ARGS="--leader-elect=true --address=0.0.0.0 ?????--algorithm-provider=DefaultProvider"

cat << EOF > /usr/lib/systemd/system/calico-node.service[Unit]Description=calico nodeAfter=docker.serviceRequires=docker.service[Service]User=rootEnvironment=ETCD_ENDPOINTS=https://172.16.204.131:2379PermissionsStartOnly=trueExecStart=/usr/bin/docker run ??--net=host --privileged --name=calico-node \ ???????????????????????????????-e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ ???????????????????????????????-e ETCD_CA_CERT_FILE=/etc/etcd/ssl/etcd-root-ca.pem \ ???????????????????????????????-e ETCD_CERT_FILE=/etc/etcd/ssl/etcd.pem \ ???????????????????????????????-e ETCD_KEY_FILE=/etc/etcd/ssl/etcd-key.pem \ ???????????????????????????????-e NODENAME=node01 \ ???????????????????????????????-e IP= \ ???????????????????????????????-e IP6= \ ???????????????????????????????-e NO_DEFAULT_POOLS= \ ???????????????????????????????-e AS= \ ???????????????????????????????-e CALICO_IPV4POOL_CIDR=10.20.0.0/16 \ ???????????????????????????????-e CALICO_IPV4POOL_IPIP=always \ ???????????????????????????????-e CALICO_LIBNETWORK_ENABLED=true \ ???????????????????????????????-e CALICO_NETWORKING_BACKEND=bird \ ???????????????????????????????-e CALICO_DISABLE_FILE_LOGGING=true \ ???????????????????????????????-e FELIX_IPV6SUPPORT=false \ ???????????????????????????????-e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ ???????????????????????????????-e FELIX_LOGSEVERITYSCREEN=info \ ???????????????????????????????-v /etc/etcd/ssl/etcd-root-ca.pem:/etc/etcd/ssl/etcd-root-ca.pem \ ???????????????????????????????-v /etc/etcd/ssl/etcd.pem:/etc/etcd/ssl/etcd.pem \ ???????????????????????????????-v /etc/etcd/ssl/etcd-key.pem:/etc/etcd/ssl/etcd-key.pem \ ???????????????????????????????-v /var/run/calico:/var/run/calico \ ???????????????????????????????-v /lib/modules:/lib/modules \ ???????????????????????????????-v /run/docker/plugins:/run/docker/plugins \ ???????????????????????????????-v /var/run/docker.sock:/var/run/docker.sock \ ???????????????????????????????-v /var/log/calico:/var/log/calico \ ???????????????????????????????calico/node:v2.6.9ExecStop=/usr/bin/docker rm -f calico-nodeRestart=alwaysRestartSec=10[Install]WantedBy=multi-user.targetEOF

systemctl daemon-reloadsystemctl start calico-node

wget https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/rbac.yamlwget https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml

## 更改为自己的etcd集群sed -i ‘s@.*etcd_endpoints:.*@\ \ etcd_endpoints:\ \"https://172.16.204.131:2379\"@gi‘ calico.yamlexport ETCD_CERT=`cat /etc/etcd/ssl/etcd.pem | base64 | tr -d ‘\n‘`export ETCD_KEY=`cat /etc/etcd/ssl/etcd-key.pem | base64 | tr -d ‘\n‘`export ETCD_CA=`cat /etc/etcd/ssl/etcd-root-ca.pem | base64 | tr -d ‘\n‘`sed -i "s@.*etcd-cert:.*@\ \ etcd-cert:\ ${ETCD_CERT}@gi" calico.yamlsed -i "s@.*etcd-key:.*@\ \ etcd-key:\ ${ETCD_KEY}@gi" calico.yamlsed -i "s@.*etcd-ca:.*@\ \ etcd-ca:\ ${ETCD_CA}@gi" calico.yamlsed -i ‘s@.*etcd_ca:.*@\ \ etcd_ca:\ "/calico-secrets/etcd-ca"@gi‘ calico.yamlsed -i ‘s@.*etcd_cert:.*@\ \ etcd_cert:\ "/calico-secrets/etcd-cert"@gi‘ calico.yamlsed -i ‘s@.*etcd_key:.*@\ \ etcd_key:\ "/calico-secrets/etcd-key"@gi‘ calico.yaml## 禁止kubernetes启动calico-node容器sed -i ‘106,197s@.*@#&@gi‘ calico.yaml

[root@node01 ~]# cat /etc/kubernetes/kubelet#### kubernetes kubelet (minion) config# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)KUBELET_ADDRESS="--address=172.16.204.132"# The port for the info server to serve on# KUBELET_PORT="--port=10250"# You may leave this blank to use the actual hostnameKUBELET_HOSTNAME="--hostname-override=172.16.204.132"# location of the api-server# KUBELET_API_SERVER="--api-servers=http://127.0.0.1:8080"# Add your own!# KUBELET_ARGS="--cgroup-driver=systemd"KUBELET_ARGS="--cgroup-driver=systemd ??????????????--network-plugin=cni ??????????????--cni-conf-dir=/etc/cni/net.d ??????????????--cni-bin-dir=/opt/cni/bin ??????????????--cluster-dns=10.254.0.2 ??????????????--resolv-conf=/etc/resolv.conf ??????????????--experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig ??????????????--kubeconfig=/etc/kubernetes/kubelet.kubeconfig ??????????????--fail-swap-on=false ??????????????--cert-dir=/etc/kubernetes/ssl ??????????????--cluster-domain=cluster.local. ??????????????--hairpin-mode=promiscuous-bridge ??????????????--serialize-image-pulls=false ??????????????--pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0"

systemctl daemon-reloadsystemctl restart kubelet

## 创建RBACkubectl apply -f rbac.yaml## 启动calico-cni以及kube-controller容器kubectl create -f calico.yaml

cat << EOF > demo.deploy.ymlapiVersion: apps/v1beta1kind: Deploymentmetadata: ?name: demo-tomcatspec: ?replicas: 3 ?template: ???metadata: ?????labels: ???????app: demo ???spec: ?????containers: ?????- name: demo ???????image: tomcat:9.0.7 ???????ports: ???????- containerPort: 80EOFkubectl create -f demo.deploy.ymlkubetcl get pods -o wide --all-namespaces

[root@master01 calico]# kubectl get pods --all-namespaces -o wideNAMESPACE ????NAME ??????????????????????????????????????READY ????STATUS ???RESTARTS ??AGE ??????IP ???????????????NODEdefault ??????demo-tomcat-56697dcc5b-2jv69 ??????????????1/1 ??????Running ??0 ?????????34s ??????10.20.196.136 ????192.168.133.129default ??????demo-tomcat-56697dcc5b-lmc2h ??????????????1/1 ??????Running ??0 ?????????35s ??????10.20.140.74 ?????192.168.133.130default ??????demo-tomcat-56697dcc5b-whbg7 ??????????????1/1 ??????Running ??0 ?????????34s ??????10.20.140.73 ?????192.168.133.130kube-system ??calico-kube-controllers-684fcf8587-66kxn ??1/1 ??????Running ??0 ?????????43m ??????192.168.133.129 ??192.168.133.129kube-system ??calico-node-hpr9c ?????????????????????????1/1 ??????Running ??0 ?????????43m ??????192.168.133.129 ??192.168.133.129kube-system ??calico-node-jvpf2 ?????????????????????????1/1 ??????Running ??0 ?????????43m ??????192.168.133.130 ??192.168.133.130[root@master01 calico]# kubectl exec -it demo-tomcat-56697dcc5b-2jv69 bashroot@demo-tomcat-56697dcc5b-2jv69:/usr/local/tomcat# pinpinentry ????????pinentry-curses ?ping ????????????ping6 ???????????pinky ???????????root@demo-tomcat-56697dcc5b-2jv69:/usr/local/tomcat# ping 10.20.140.74PING 10.20.140.74 (10.20.140.74): 56 data bytes64 bytes from 10.20.140.74: icmp_seq=0 ttl=62 time=0.673 ms64 bytes from 10.20.140.74: icmp_seq=1 ttl=62 time=0.398 ms^C--- 10.20.140.74 ping statistics ---2 packets transmitted, 2 packets received, 0% packet lossround-trip min/avg/max/stddev = 0.398/0.536/0.673/0.138 msroot@demo-tomcat-56697dcc5b-2jv69:/usr/local/tomcat# ping 10.20.140.73PING 10.20.140.73 (10.20.140.73): 56 data bytes64 bytes from 10.20.140.73: icmp_seq=0 ttl=62 time=0.844 ms64 bytes from 10.20.140.73: icmp_seq=1 ttl=62 time=0.348 ms^C--- 10.20.140.73 ping statistics ---2 packets transmitted, 2 packets received, 0% packet lossround-trip min/avg/max/stddev = 0.348/0.596/0.844/0.248 msroot@demo-tomcat-56697dcc5b-2jv69:/usr/local/tomcat# ping 10.20.196.136PING 10.20.196.136 (10.20.196.136): 56 data bytes64 bytes from 10.20.196.136: icmp_seq=0 ttl=64 time=0.120 ms64 bytes from 10.20.196.136: icmp_seq=1 ttl=64 time=0.068 ms^C--- 10.20.196.136 ping statistics ---2 packets transmitted, 2 packets received, 0% packet lossround-trip min/avg/max/stddev = 0.068/0.094/0.120/0.026 ms

./deploy.sh -r 10.254.0.0/16 -i 10.254.0.2 -d cluster.local | kubectl apply -f - 

[root@master01 coredns]# kubectl get pods --all-namespacesNAMESPACE ????NAME ??????????????????????????????????????READY ????STATUS ???RESTARTS ??AGEkube-system ??calico-kube-controllers-684fcf8587-5ndks ??1/1 ??????Running ??1 ?????????11dkube-system ??calico-node-4wskw ?????????????????????????1/1 ??????Running ??1 ?????????11dkube-system ??calico-node-sbngf ?????????????????????????1/1 ??????Running ??1 ?????????11dkube-system ??coredns-64b597b598-fmh85 ??????????????????1/1 ??????Running ??0 ?????????57skube-system ??coredns-64b597b598-jf88d ??????????????????1/1 ??????Running ??0 ?????????57s

cat > my-nginx.yaml << EOF apiVersion: extensions/v1beta1 kind: Deployment metadata: ??name: my-nginx spec: ??replicas: 2 ??template: ????metadata: ??????labels: ????????run: my-nginx ????spec: ??????containers: ??????- name: my-nginx ????????image: nginx:1.7.9 ????????ports: ????????- containerPort: 80EOFkubectl create -f my-nginx.yaml

##创建my-nginx pod servicekubectl expose deploy my-nginx##查看创建的service[root@master01 ~]# kubectl get services --all-namespacesNAMESPACE ????NAME ????????TYPE ???????CLUSTER-IP ????EXTERNAL-IP ??PORT(S) ????????AGEdefault ??????kubernetes ??ClusterIP ??10.254.0.1 ????<none> ???????443/TCP ????????12ddefault ??????my-nginx ????ClusterIP ??10.254.37.75 ??<none> ???????80/TCP ?????????13skube-system ??kube-dns ????ClusterIP ??10.254.0.2 ????<none> ???????53/UDP,53/TCP ??4m

[root@master01 ~]# kubectl exec -it my-nginx-56b48db847-g8fr2 /bin/bashroot@my-nginx-56b48db847-g8fr2:/# cat /etc/resolv.confnameserver 10.254.0.2search default.svc.cluster.local. svc.cluster.local. cluster.local.options ndots:5root@my-nginx-56b48db847-g8fr2:/# ping my-nginxPING my-nginx.default.svc.cluster.local (10.254.37.75): 48 data bytes^C--- my-nginx.default.svc.cluster.local ping statistics ---7 packets transmitted, 0 packets received, 100% packet lossroot@my-nginx-56b48db847-g8fr2:/# ping kubernetesPING kubernetes.default.svc.cluster.local (10.254.0.1): 48 data bytes^C--- kubernetes.default.svc.cluster.local ping statistics ---5 packets transmitted, 0 packets received, 100% packet lossroot@my-nginx-56b48db847-g8fr2:/# ping kube-dns.kube-system.svc.cluster.localPING kube-dns.kube-system.svc.cluster.local (10.254.0.2): 48 data bytes^C--- kube-dns.kube-system.svc.cluster.local ping statistics ---6 packets transmitted, 0 packets received, 100% packet lossroot@my-nginx-56b48db847-g8fr2:/# curl -I my-nginxHTTP/1.1 200 OKServer: nginx/1.7.9Date: Tue, 08 May 2018 07:27:13 GMTContent-Type: text/htmlContent-Length: 612Last-Modified: Tue, 23 Dec 2014 16:25:09 GMTConnection: keep-aliveETag: "54999765-264"Accept-Ranges: bytesroot@my-nginx-56b48db847-g8fr2:/# curl my-nginx.default.svc.cluster.local<!DOCTYPE html><html><head><title>Welcome to nginx!</title><style>body {width: 35em;margin: 0 auto;font-family: Tahoma, Verdana, Arial, sans-serif;}...省略其他...