首先理清思路
首先从s3上用s3cmd命令将logs同步下来 再将日志写到文件 再通过elk展示出来
一、安装s3cmd命令
S3cmd工具的安装与简单使用：
参考文档
https://www.cnblogs.com/xd502djj/p/3604783.html https://github.com/s3tools/s3cmd/releases
先下载S3cmd安装包 从github中获取
mkdir /home/tools/ && cd /home/tools/ wget https://github.com/s3tools/s3cmd/releases/download/v2.0.1/s3cmd-2.0.1.tar.gztar xf s3cmd-2.0.1.tar.gzmv s3cmd-2.0.1 /usr/local/mv /usr/local/ s3cmd-2.0.1 /usr/local/s3cmdln -s /usr/local/s3cmd/s3cmd /usr/bin/s3cmd
安装完成之后 使用 s3cmd –configure 设置key
主要就是access key和 secure key 配置完成之后会生成下边的配置文件
[root@prod-sg s3cmd]# cat /root/.s3cfg[default]access_key = AKIAI4Q3PTOQ5xxxxxxx aws s3的access key 必须access_token =add_encoding_exts =add_headers =bucket_location = USca_certs_file =cache_file =check_ssl_certificate = Truecheck_ssl_hostname = Truecloudfront_host = cloudfront.amazonaws.comdefault_mime_type = binary/octet-streamdelay_updates = Falsedelete_after = Falsedelete_after_fetch = Falsedelete_removed = Falsedry_run = Falseenable_multipart = Trueencrypt = Falseexpiry_date =expiry_days =expiry_prefix =follow_symlinks = Falseforce = Falseget_continue = Falsegpg_command = /usr/bin/gpggpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)sgpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)sgpg_passphrase = aviagamesguess_mime_type = Truehost_base = s3.amazonaws.comhost_bucket = %(bucket)s.s3.amazonaws.comhuman_readable_sizes = Falseinvalidate_default_index_on_cf = Falseinvalidate_default_index_root_on_cf = Trueinvalidate_on_cf = Falsekms_key =limit = -1limitrate = 0list_md5 = Falselog_target_prefix =long_listing = Falsemax_delete = -1mime_type =multipart_chunk_size_mb = 15multipart_max_chunks = 10000preserve_attrs = Trueprogress_meter = Trueproxy_host =proxy_port = 0put_continue = Falserecursive = Falserecv_chunk = 65536reduced_redundancy = Falserequester_pays = Falserestore_days = 1restore_priority = Standardsecret_key = 0uoniJrn9qQhAnxxxxxxCZxxxxxxxxxxxx aws s3的secret_key 必须send_chunk = 65536server_side_encryption = Falsesignature_v2 = Falsesignurl_use_https = Falsesimpledb_host = sdb.amazonaws.comskip_existing = Falsesocket_timeout = 300stats = Falsestop_on_error = Falsestorage_class =urlencoding_mode = normaluse_http_expect = Falseuse_https = Falseuse_mime_magic = Trueverbosity = WARNINGwebsite_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/website_error =website_index = index.html
二、s3cmd命令安装完成之后 编写脚本
#!/bin/bash#进入S3同步目录mkdir /server/s3dir/logs/ -p && cd /server/s3dir/logs/#每隔5分钟将S3的日志列表放到S3.log文件中#while true#do /usr/bin/s3cmd ls s3://bigbearsdk/logs/ >S3.log #执行同步命令确认 S3与服务器 日志一样 /usr/bin/s3cmd sync --skip-existing s3://bigbearsdk/logs/ ./#done#当天的日志排并序追加到一个文件 grep $(date +%F) S3.log |sort -nk1,2 |awk -F [/] '{print $NF}' > date.log sed -i 's#\_#\\_#g' date.log sed -i 's#<#\\\<#g' date.log sed -i 's#\ #\\ #g' date.log sed -i 's#>#\\\>#g' date.log##[ -f ELK.log ] &&#{# cat ELK.log >> ELK_$(date +%F).log# echo > ELK.log# find /home/tools/ -name ELK*.log -mtime +7 |xargs rm -f#}#将每个文件的日志都追加到S3上传日志中while read linedo echo "$line"|sed 's#(#\\\(#g'|sed 's#)#\\\)#g'| sed 's#\_#\\_#g'|sed 's#<#\\\<#g'|sed 's#>#\\\>#g'|sed 's#\ #\\ #g' >while.log head -1 while.log |xargs cat >> /server/s3dir/s3elk.logdone < date.log这样的话 S3日志里边的内容全都到了 s3elk.log 这个文件中 再通过elk监控日志
未完待续。。。
有意向加微信 Dellinger_blue
AWS S3日志文件通过服务器上传到elk
原文地址：http://blog.51cto.com/dellinger/2117121