kubernetes之收集集群的events，监控集群行为

input{ ??kafka{ ?????bootstrap_servers => ["kafka-0.kafka-svc.kafka.svc.cluster.local:9092,kafka-1.kafka-svc.kafka.svc.cluster.local:9092,kafka-2.kafka-svc.kafka.svc.cluster.local:9092"] ?????client_id => "eventrouter-prod" ?????#auto_offset_reset => "latest" ?????group_id => "eventrouter" ?????consumer_threads => 2 ?????#decorate_events ?=> true ?????id => "eventrouter" ?????topics => ["eventrouter"]}}filter { ?if [message] =~ ‘DNSConfigForming‘ { ????drop { } ?} ?json { ???source => "message" ?} ?mutate { ???remove_field => [ "message","old_event" ]}}output{ elasticsearch { ???????????????????????hosts => "10.4.9.28:9200" ???????????????????????index => "eventrouter-%{+YYYY-MM-dd}" ????????????????}}

version: ‘2‘services: ?elasticsearch: ???image: docker.elastic.co/elasticsearch/elasticsearch:6.5.1 ???container_name: elasticsearch ???environment: ?????- cluster.name=docker-cluster ?????- bootstrap.memory_lock=true ?????- "ES_JAVA_OPTS=-Xms4096m -Xmx4096m" ???ulimits: ?????memlock: ???????soft: -1 ???????hard: -1 ???volumes: ?????- /data/es1:/usr/share/elasticsearch/data ?????- /data/backups:/usr/share/elasticsearch/backups ?????- /data/longterm_backups:/usr/share/elasticsearch/longterm_backups ?????- ./config/jvm.options:/usr/share/elasticsearch/config/jvm.options ???ports: ?????- "9200:9200" ???networks: ?????- esnet# ?elasticsearch2:# ???image: docker.elastic.co/elasticsearch/elasticsearch:6.5.1# ???container_name: elasticsearch2# ???environment:# ?????- cluster.name=docker-cluster# ?????- bootstrap.memory_lock=true# ?????- "ES_JAVA_OPTS=-Xms512m -Xmx512m"# ?????- "discovery.zen.ping.unicast.hosts=elasticsearch"# ???ulimits:# ?????memlock:# ???????soft: -1# ???????hard: -1# ???volumes:# ?????- /data/es2:/usr/share/elasticsearch/data# ???networks:# ?????- esnet ?kibana: ???image: docker.elastic.co/kibana/kibana:6.5.1 ???container_name: kibana ???environment: ?????SERVER_NAME: kibana ?????SERVER_HOST: "0.0.0.0" ?????ELASTICSEARCH_URL: http://elasticsearch:9200 ?????XPACK_MONITORING_UI_CONATINER_ELASTICSEARCH_ENABLED: "true" ???volumes: ?????- /data/plugin:/usr/share/kibana/plugin ?????- /tmp/:/etc/archives ???ports: ?????- "5601:5601" ???networks: ?????- esnet ???depends_on: ?????- elasticsearchnetworks: esnet: ??driver: bridge

## JVM configuration################################################################## IMPORTANT: JVM heap size#################################################################### You should always set the min and max JVM heap## size to the same value. For example, to set## the heap to 4 GB, set:#### -Xms4g## -Xmx4g#### See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html## for more information################################################################### Xms represents the initial size of total heap space# Xmx represents the maximum size of total heap space-Xms2g-Xmx2g################################################################## Expert settings#################################################################### All settings below this section are considered## expert settings. Don‘t tamper with them unless## you understand what you are doing#################################################################### GC configuration-XX:+UseConcMarkSweepGC-XX:CMSInitiatingOccupancyFraction=75-XX:+UseCMSInitiatingOccupancyOnly## G1GC Configuration# NOTE: G1GC is only supported on JDK version 10 or later.# To use G1GC uncomment the lines below.# 10-:-XX:-UseConcMarkSweepGC# 10-:-XX:-UseCMSInitiatingOccupancyOnly# 10-:-XX:+UseG1GC# 10-:-XX:InitiatingHeapOccupancyPercent=75## optimizations# pre-touch memory pages used by the JVM during initialization-XX:+AlwaysPreTouch## basic# explicitly set the stack size-Xss1m# set to headless, just in case-Djava.awt.headless=true# ensure UTF-8 encoding by default (e.g. filenames)-Dfile.encoding=UTF-8# use our provided JNA always versus the system one-Djna.nosys=true# turn off a JDK optimization that throws away stack traces for common# exceptions because stack traces are important for debugging-XX:-OmitStackTraceInFastThrow# flags to configure Netty-Dio.netty.noUnsafe=true-Dio.netty.noKeySetOptimization=true-Dio.netty.recycler.maxCapacityPerThread=0# log4j 2-Dlog4j.shutdownHookEnabled=false-Dlog4j2.disable.jmx=true-Djava.io.tmpdir=${ES_TMPDIR}## heap dumps# generate a heap dump when an allocation from the Java heap fails# heap dumps are created in the working directory of the JVM-XX:+HeapDumpOnOutOfMemoryError# specify an alternative path for heap dumps; ensure the directory exists and# has sufficient space-XX:HeapDumpPath=data# specify an alternative path for JVM fatal error logs-XX:ErrorFile=logs/hs_err_pid%p.log## JDK 8 GC logging8:-XX:+PrintGCDetails8:-XX:+PrintGCDateStamps8:-XX:+PrintTenuringDistribution8:-XX:+PrintGCApplicationStoppedTime8:-Xloggc:logs/gc.log8:-XX:+UseGCLogFileRotation8:-XX:NumberOfGCLogFiles=328:-XX:GCLogFileSize=64m# JDK 9+ GC logging9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise# time/date parsing will break in an incompatible way for some date patterns and locals9-:-Djava.locale.providers=COMPAT# temporary workaround for C2 bug with JDK 10 on hardware with AVX-51210-:-XX:UseAVX=2

mkdir ?/etc/elastalert

cpoy ?elastalert/config.yaml.example ????/etc/elastalert/config.yaml

mkdir /etc/elastalert/rules

cp ?-r elastalert-dingtalk-plugin/elastalert_modules ??/etc/elastalert/elastalert

# Alert when the rate of events exceeds a threshold# (Optional)# Elasticsearch hostes_host: 10.2.9.28# (Optional)# Elasticsearch portes_port: 9200# (OptionaL) Connect with SSL to Elasticsearch#use_ssl: True# (Optional) basic-auth username and password for Elasticsearch#es_username: someusername#es_password: somepassword# (Required)# Rule name, must be uniquename: Other event frequency rule# (Required)# Type of alert.# the frequency rule type alerts when num_events events occur with timeframe timetype: frequency# (Required)# Index to search, wildcard supportedindex: eventrouter-*# (Required, frequency specific)# Alert when this many documents matching the query occur within a timeframenum_events: 5# (Required, frequency specific)# num_events must occur within this amount of time to trigger an alerttimeframe: ?#hours: 4 ?minutes: 15# (Required)# A list of Elasticsearch filters used for find events# These filters are joined with AND and nested in a filtered query# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.htmlfilter:#- term:# ???some_field: "some_value"- query: ???query_string: ?????query: "event.type: Warning NOT event.involvedObject.kind: Node"# (Required)# The alert is use when a match is found#smtp_host: smtp.exmail.qq.com#smtp_port: 25#smtp_auth_file: /etc/elastalert/smtp_auth_file.yaml#email_reply_to: ci@qq.com#from_addr: ci@qq.comrealert: ?minutes: 5exponential_realert: ?hours: 1alert:#- "email"- "elastalert_modules.dingtalk_alert.DingTalkAlerter"dingtalk_webhook: "https://oapi.dingtalk.com/robot/send?access_token=47194e6904c6e3133a9080980984444c8e5d7745e1f76c12cefa99c8c8ac718dd88d4c"dingtalk_msgtype: "text"alert_text_type: alert_text_onlyalert_text: " ??====elastalert message====\n ??EventTime>>: ?{0}\n ??Event_involvedObject_name>>: ?{1}\n ??Event_involvedObject_kind>>: ?{2}\n ??Event_involvedObject_namespace>>: ?{3}\n ??Message>>: ?{4}\n ??Event_reason>>: {5}\n ??verb>>: {6}"alert_text_args:- "@timestamp"- event.involvedObject.name- event.source.component- event.involvedObject.namespace- event.message- event.reason- verb# (required, email specific)# a list of email addresses to send alerts to#email:#- "ci@qq.com"

alert:#- "email"- "elastalert_modules.dingtalk_alert.DingTalkAlerter"dingtalk_webhook: "https://oapi.dingtalk.com/robot/send?access_token=47194e6904c6e3133a9080980984444c8e5d7745e1f76c12cefa99c8c8ac718dd88d4c"dingtalk_msgtype: "text"alert_text_type: alert_text_onlyalert_text: " ??====elastalert message====\n ??EventTime>>: ?{0}\n ??Event_involvedObject_name>>: ?{1}\n ??Event_involvedObject_kind>>: ?{2}\n ??Event_involvedObject_namespace>>: ?{3}\n ??Message>>: ?{4}\n ??Event_reason>>: {5}\n ??verb>>: {6}"alert_text_args:- "@timestamp"- event.involvedObject.name- event.source.component- event.involvedObject.namespace- event.message- event.reason- verb