注意：所有与监控有关的信息都放在namespace=monitor上
github:https://github.com/grafana/kubernetes-app
１、安装prometheus
prom-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRolemetadata: ?name: prometheus ?namespace: monitorrules:- apiGroups: [""] ?resources: ?- nodes ?- nodes/proxy ?- services ?- endpoints ?- pods ?verbs: ["get", "list", "watch"]- apiGroups: ?- extensions ?resources: ?- ingresses ?verbs: ["get", "list", "watch"]- nonResourceURLs: ["/metrics"] ?verbs: ["get"]---apiVersion: v1kind: ServiceAccountmetadata: ?name: prometheus ?namespace: monitor---apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRoleBindingmetadata: ?name: prometheusroleRef: ?apiGroup: rbac.authorization.k8s.io ?kind: ClusterRole ?name: prometheussubjects:- kind: ServiceAccount ?name: prometheus ?namespace: monitorcat prom-config-kubernetes.yaml
apiVersion: v1kind: ConfigMapmetadata: name: prometheus-config namespace: monitordata: prometheus.yml: | ?global: ?scrape_configs: ??- job_name: ‘kubernetes-kubelet‘ ????scheme: https ????tls_config: ??????ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt ??????insecure_skip_verify: true ????bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ????kubernetes_sd_configs: ????- role: node ????relabel_configs: ????- action: labelmap ??????regex: __meta_kubernetes_node_label_(.+) ????- target_label: __address__ ??????replacement: kubernetes.default.svc.cluster.local:443 ????- source_labels: [__meta_kubernetes_node_name] ??????regex: (.+) ??????target_label: __metrics_path__ ??????replacement: /api/v1/nodes/${1}/proxy/metrics ??- job_name: ‘kubernetes-cadvisor‘ ????scheme: https ????tls_config: ??????ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt ??????insecure_skip_verify: true ????bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ????kubernetes_sd_configs: ????- role: node ????relabel_configs: ????- action: labelmap ??????regex: __meta_kubernetes_node_label_(.+) ????- target_label: __address__ ??????replacement: kubernetes.default.svc.cluster.local:443 ????- source_labels: [__meta_kubernetes_node_name] ??????regex: (.+) ??????target_label: __metrics_path__ ??????replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor ??- job_name: ‘kubernetes-kube-state‘ ????kubernetes_sd_configs: ????- role: pod ????relabel_configs: ????- action: labelmap ??????regex: __meta_kubernetes_pod_label_(.+) ????- source_labels: [__meta_kubernetes_namespace] ??????action: replace ??????target_label: kubernetes_namespace ????- source_labels: [__meta_kubernetes_pod_name] ??????action: replace ??????target_label: kubernetes_pod_name ????- source_labels: [__meta_kubernetes_pod_label_grafanak8sapp] ??????regex: .*true.* ??????action: keep ????- source_labels: [‘__meta_kubernetes_pod_label_daemon‘, ‘__meta_kubernetes_pod_node_name‘] ??????regex: ‘node-exporter;(.*)‘ ??????action: replace ??????target_label: nodenamecat prom-install.yaml
apiVersion: v1kind: ConfigMapmetadata: name: prometheus-config-test namespace: monitordata: prometheus.yml: | ?global: ?scrape_configs: ??- job_name: ‘kubernetes-kubelet‘ ????scheme: https ????tls_config: ??????ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt ??????insecure_skip_verify: true ????bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ????kubernetes_sd_configs: ????- role: node ????relabel_configs: ????- action: labelmap ??????regex: __meta_kubernetes_node_label_(.+) ????- target_label: __address__ ??????replacement: kubernetes.default.svc.cluster.local:443 ????- source_labels: [__meta_kubernetes_node_name] ??????regex: (.+) ??????target_label: __metrics_path__ ??????replacement: /api/v1/nodes/${1}/proxy/metrics ??- job_name: ‘kubernetes-cadvisor‘ ????scheme: https ????tls_config: ??????ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt ??????insecure_skip_verify: true ????bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token ????kubernetes_sd_configs: ????- role: node ????relabel_configs: ????- action: labelmap ??????regex: __meta_kubernetes_node_label_(.+) ????- target_label: __address__ ??????replacement: kubernetes.default.svc.cluster.local:443 ????- source_labels: [__meta_kubernetes_node_name] ??????regex: (.+) ??????target_label: __metrics_path__ ??????replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor ??- job_name: ‘kubernetes-kube-state‘ ????kubernetes_sd_configs: ????- role: pod ????relabel_configs: ????- action: labelmap ??????regex: __meta_kubernetes_pod_label_(.+) ????- source_labels: [__meta_kubernetes_namespace] ??????action: replace ??????target_label: kubernetes_namespace ????- source_labels: [__meta_kubernetes_pod_name] ??????action: replace ??????target_label: kubernetes_pod_name ????- source_labels: [__meta_kubernetes_pod_label_grafanak8sapp] ??????regex: .*true.* ??????action: keep ????- source_labels: [‘__meta_kubernetes_pod_label_daemon‘, ‘__meta_kubernetes_pod_node_name‘] ??????regex: ‘node-exporter;(.*)‘ ??????action: replace ??????target_label: nodename ??- job_name: ‘tomcat-pods‘ ????kubernetes_sd_configs: ????- role: pod ????relabel_configs: ????- source_labels: [__meta_kubernetes_namespace] ??????regex: tomcat ??????action: keep ????- source_labels: [__meta_kubernetes_pod_container_port_number] ??????regex: 1234 ??????action: keep ????- source_labels: [__meta_kubernetes_pod_name] ??????action: replace ??????target_label: kubernetes_pod_name[root@k8s-master1 monitor]# cat prom/prom-install.yaml apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels: ?name: prometheus name: prometheus namespace: monitorspec: ?replicas: 1 ?template: ??metadata: ???labels: ?????app: prometheus-server ??spec: ???serviceAccountName: prometheus ???containers: ???- name: prometheus ?????image: hub.llzg.com.cn/monitor/prometheus:latest ?????imagePullPolicy: Always ?????ports: ???????- containerPort: 9090 ?????????protocol: TCP ?????volumeMounts: ???????- mountPath: "/etc/prometheus" ?????????name: config-prometheus ???imagePullSecrets: ???- name: authllzg ???volumes: ???- name: config-prometheus ?????configMap: ??????name: prometheus-configcat prom-svc.yaml
kind: ServiceapiVersion: v1metadata: ?name: prometheus-ingress-service ?namespace: monitorspec: ?selector: ????app: prometheus-server ?ports: ???- protocol: TCP ?????port: 9090 ?????name: prom---apiVersion: extensions/v1beta1kind: Ingressmetadata: ?name: prom-web-ui ?namespace: monitorspec: ?rules: ?- host: prom.name.com.cn ???http: ?????paths: ?????- path: / ???????backend: ?????????serviceName: prometheus-ingress-service ?????????servicePort: prom2、安装state-metrics
cat state-metrics-rbac.yaml
apiVersion: v1kind: ServiceAccountmetadata: ?name: kube-state-metrics ?namespace: monitor---apiVersion: rbac.authorization.k8s.io/v1kind: Rolemetadata: ?namespace: monitor ?name: kube-state-metrics-resizerrules:- apiGroups: [""] ?resources: ?- pods ?verbs: ["get"]- apiGroups: ["extensions"] ?resources: ?- deployments ?resourceNames: ["kube-state-metrics"] ?verbs: ["get", "update"]---apiVersion: rbac.authorization.k8s.io/v1kind: RoleBindingmetadata: ?name: kube-state-metrics ?namespace: monitorroleRef: ?apiGroup: rbac.authorization.k8s.io ?kind: Role ?name: kube-state-metrics-resizersubjects:- kind: ServiceAccount ?name: kube-state-metrics ?namespace: monitor---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRolemetadata: ?name: kube-state-metrics ?namespace: monitorrules:- apiGroups: [""] ?resources: ?- nodes ?- pods ?- services ?- resourcequotas ?- replicationcontrollers ?- limitranges ?- persistentvolumeclaims ?- persistentvolumes ?- namespaces ?- endpoints ?verbs: ["list", "watch"]- apiGroups: ["extensions"] ?resources: ?- daemonsets ?- deployments ?- replicasets ?verbs: ["list", "watch"]- apiGroups: ["apps"] ?resources: ?- statefulsets ?verbs: ["list", "watch"]- apiGroups: ["batch"] ?resources: ?- cronjobs ?- jobs ?verbs: ["list", "watch"]- apiGroups: ["autoscaling"] ?resources: ?- horizontalpodautoscalers ?verbs: ["list", "watch"]---apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBindingmetadata: ?name: kube-state-metrics ?namespace: monitorroleRef: ?apiGroup: rbac.authorization.k8s.io ?kind: ClusterRole ?name: kube-state-metricssubjects:- kind: ServiceAccount ?name: kube-state-metrics ?namespace: monitor---{ ?"apiVersion": "apps/v1beta1", ?"kind": "Deployment", ?"metadata": { ???"name": "kube-state-metrics", ???"namespace": "monitor" ?}, ?"spec": { ???"selector": { ?????"matchLabels": { ???????"k8s-app": "kube-state-metrics", ???????"grafanak8sapp": "true" ?????} ???}, ???"replicas": 1, ???"template": { ?????"metadata": { ???????"labels": { ?????????"k8s-app": "kube-state-metrics", ?????????"grafanak8sapp": "true" ???????} ?????}, ?????"spec": { ???????"serviceAccountName": "kube-state-metrics", ???????"containers": [ ?????????{ ???????????"name": "kube-state-metrics", ???????????"image": "hub.llzg.com.cn/monitor/kube-state-metrics:v1.1.0", ???????????"ports": [ ?????????????{ ???????????????"name": "http-metrics", ???????????????"containerPort": 8080 ?????????????} ???????????], ???????????"readinessProbe": { ?????????????"httpGet": { ???????????????"path": "/healthz", ???????????????"port": 8080 ?????????????}, ?????????????"initialDelaySeconds": 5, ?????????????"timeoutSeconds": 5 ???????????} ?????????} ???????], ???????"imagePullSecrets": [ ?????????{ ???????????"name": "authllzg" ?????????????} ???????????] ?????} ???} ?}}3、安装node-exporter
cat node-exporter.yaml
{ ?"kind": "DaemonSet", ?"apiVersion": "extensions/v1beta1", ?"metadata": { ???"name": "node-exporter", ???"namespace": "monitor" ?}, ?"spec": { ???"selector": { ?????"matchLabels": { ???????"daemon": "node-exporter", ???????"grafanak8sapp": "true" ?????} ???}, ???"template": { ?????"metadata": { ???????"name": "node-exporter", ???????"labels": { ?????????"daemon": "node-exporter", ?????????"grafanak8sapp": "true" ???????} ?????}, ?????"spec": { ???????"volumes": [ ?????????{ ???????????"name": "proc", ???????????"hostPath": { ?????????????"path": "/proc" ???????????} ?????????}, ?????????{ ???????????"name": "sys", ???????????"hostPath": { ?????????????"path": "/sys" ???????????} ?????????} ???????], ???????"containers": [ ?????????{ ???????????"name": "node-exporter", ???????????"image": "hub.llzg.com.cn/monitor/node-exporter:v0.15.0", ???????????"args": [ ?????????????"--path.procfs=/proc_host", ?????????????"--path.sysfs=/host_sys" ???????????], ???????????"ports": [ ?????????????{ ???????????????"name": "node-exporter", ???????????????"hostPort": 9100, ???????????????"containerPort": 9100 ?????????????} ???????????], ???????????"volumeMounts": [ ?????????????{ ???????????????"name": "sys", ???????????????"readOnly": true, ???????????????"mountPath": "/host_sys" ?????????????}, ?????????????{ ???????????????"name": "proc", ???????????????"readOnly": true, ???????????????"mountPath": "/proc_host" ?????????????} ???????????], ???????????"imagePullPolicy": "IfNotPresent" ?????????} ???????], ???????"restartPolicy": "Always", ???????"hostNetwork": true, ???????"hostPID": true ?????} ???} ?}}4、安装grafana(添加grafana-kubernetes-app插件)
cat grafana.yaml
apiVersion: extensions/v1beta1kind: Deploymentmetadata: labels: ?name: grafana-server name: grafana namespace: monitorspec: ?replicas: 1 ?template: ??metadata: ???labels: ?????app: grafana-server ??spec: ???serviceAccountName: prometheus ???containers: ???- name: grafana ?????image: hub.llzg.com.cn/monitor/grafana:latest ?????ports: ???????- containerPort: 3000 ?????????protocol: TCP ?????volumeMounts: ?????- mountPath: "/var/lib/grafana" ???????readOnly: false ???????name: grafana-nfs ?????env: ?????- name: GF_INSTALL_PLUGINS ???????value: "grafana-kubernetes-app" ???imagePullSecrets: ???- name: authllzg ???volumes: ???- name: grafana-nfs ?????nfs: ??????server: 192.168.10.28 ??????path: "/data/nfs/monitor/grafana"---kind: ServiceapiVersion: v1metadata: ?name: grafana-ingress-service ?namespace: monitorspec: ?selector: ???app: grafana-server ?ports: ???- protocol: TCP ?????port: 3000 ?????name: grafana---apiVersion: extensions/v1beta1kind: Ingressmetadata: ?name: grafana-ingress ?namespace: monitor ?annotations: ???kubernetes.io/ingress.class: traefikspec: ?rules: ?- host: grafana.llzg.com.cn ???http: ?????paths: ?????- path: / ???????backend: ?????????serviceName: grafana-ingress-service ?????????servicePort: grafana５、grafana+prometheus监控显示kubernetes
(1)grafana添加prometheus
(2)加载grafana-kubernetes-app插件
configuration---->plugins--------->kubernetes
(3)配置grafana-kubernetes-app插件
４、OK，去看图吧
kubernetes监控:grafana plugins IN kubernetes
原文地址：http://blog.51cto.com/11578535/2095343