Docker Compose一键ELK部署的方法实现

version: "3"services: es-master:  container_name: es-master  hostname: es-master  image: elasticsearch:7.5.1  restart: always  ports:   - 9200:9200   - 9300:9300  volumes:   - ./elasticsearch/master/conf/es-master.yml:/usr/share/elasticsearch/config/elasticsearch.yml   - ./elasticsearch/master/data:/usr/share/elasticsearch/data   - ./elasticsearch/master/logs:/usr/share/elasticsearch/logs  environment:   - "ES_JAVA_OPTS=-Xms512m -Xmx512m" es-slave1:  container_name: es-slave1  image: elasticsearch:7.5.1  restart: always  ports:   - 9201:9200   - 9301:9300  volumes:   - ./elasticsearch/slave1/conf/es-slave1.yml:/usr/share/elasticsearch/config/elasticsearch.yml   - ./elasticsearch/slave1/data:/usr/share/elasticsearch/data   - ./elasticsearch/slave1/logs:/usr/share/elasticsearch/logs  environment:   - "ES_JAVA_OPTS=-Xms512m -Xmx512m" es-slave2:  container_name: es-slave2  image: elasticsearch:7.5.1  restart: always  ports:   - 9202:9200   - 9302:9300  volumes:   - ./elasticsearch/slave2/conf/es-slave2.yml:/usr/share/elasticsearch/config/elasticsearch.yml   - ./elasticsearch/slave2/data:/usr/share/elasticsearch/data   - ./elasticsearch/slave2/logs:/usr/share/elasticsearch/logs  environment:   - "ES_JAVA_OPTS=-Xms512m -Xmx512m" kibana:  container_name: kibana  hostname: kibana  image: kibana:7.5.1  restart: always  ports:   - 5601:5601  volumes:   - ./kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml  environment:   - elasticsearch.hosts=http://es-master:9200  depends_on:   - es-master   - es-slave1   - es-slave2 # filebeat: #  # 容器名称 #  container_name: filebeat #  # 主机名称 #  hostname: filebeat #  # 镜像 #  image: docker.elastic.co/beats/filebeat:7.5.1 #  # 重启机制 #  restart: always #  # 持久化挂载 #  volumes: #   - ./filebeat/conf/filebeat.yml:/usr/share/filebeat/filebeat.yml #   # 映射到容器中[作为数据源] #   - ./logs:/home/project/spring-boot-elasticsearch/logs #   - ./filebeat/logs:/usr/share/filebeat/logs #   - ./filebeat/data:/usr/share/filebeat/data #  # 将指定容器连接到当前连接，可以设置别名，避免ip方式导致的容器重启动态改变的无法连接情况 #  links: #   - logstash #  # 依赖服务[可无] #  depends_on: #   - es-master #   - es-slave1 #   - es-slave2 logstash:  container_name: logstash  hostname: logstash  image: logstash:7.5.1  command: logstash -f ./conf/logstash-filebeat.conf  restart: always  volumes:   # 映射到容器中   - ./logstash/conf/logstash-filebeat.conf:/usr/share/logstash/conf/logstash-filebeat.conf   - ./logstash/ssl:/usr/share/logstash/ssl  environment:   - elasticsearch.hosts=http://es-master:9200   # 解决logstash监控连接报错   - xpack.monitoring.elasticsearch.hosts=http://es-master:9200  ports:   - 5044:5044  depends_on:   - es-master   - es-slave1   - es-slave2

# 集群名称cluster.name: es-cluster# 节点名称node.name: es-master# 是否可以成为master节点node.master: true# 是否允许该节点存储数据,默认开启node.data: false# 网络绑定network.host: 0.0.0.0# 设置对外服务的http端口http.port: 9200# 设置节点间交互的tcp端口transport.port: 9300# 集群发现discovery.seed_hosts: - es-master - es-slave1 - es-slave2# 手动指定可以成为 mater 的所有节点的 name 或者 ip，这些配置将会在第一次选举中进行计算cluster.initial_master_nodes: - es-master# 支持跨域访问http.cors.enabled: truehttp.cors.allow-origin: "*"# 安全认证xpack.security.enabled: false#http.cors.allow-headers: "Authorization"

# 集群名称cluster.name: es-cluster# 节点名称node.name: es-slave1# 是否可以成为master节点node.master: true# 是否允许该节点存储数据,默认开启node.data: true# 网络绑定network.host: 0.0.0.0# 设置对外服务的http端口http.port: 9201# 设置节点间交互的tcp端口#transport.port: 9301# 集群发现discovery.seed_hosts: - es-master - es-slave1 - es-slave2# 手动指定可以成为 mater 的所有节点的 name 或者 ip，这些配置将会在第一次选举中进行计算cluster.initial_master_nodes: - es-master# 支持跨域访问http.cors.enabled: truehttp.cors.allow-origin: "*"# 安全认证xpack.security.enabled: false#http.cors.allow-headers: "Authorization"

# 集群名称cluster.name: es-cluster# 节点名称node.name: es-slave2# 是否可以成为master节点node.master: true# 是否允许该节点存储数据,默认开启node.data: true# 网络绑定network.host: 0.0.0.0# 设置对外服务的http端口http.port: 9202# 设置节点间交互的tcp端口#transport.port: 9302# 集群发现discovery.seed_hosts: - es-master - es-slave1 - es-slave2# 手动指定可以成为 mater 的所有节点的 name 或者 ip，这些配置将会在第一次选举中进行计算cluster.initial_master_nodes: - es-master# 支持跨域访问http.cors.enabled: truehttp.cors.allow-origin: "*"# 安全认证xpack.security.enabled: false#http.cors.allow-headers: "Authorization"

input {  # 来源beats  beats {    # 端口    port => "5044"    ssl_certificate_authorities => ["/usr/share/logstash/ssl/ca.crt"]    ssl_certificate => "/usr/share/logstash/ssl/server.crt"    ssl_key => "/usr/share/logstash/ssl/server.key"    ssl_verify_mode => "force_peer"  }}# 分析、过滤插件，可以多个filter {  grok {    match => { "message" => "%{COMBINEDAPACHELOG}"}  }  geoip {    source => "clientip"  }}output {  # 选择elasticsearch  elasticsearch {    hosts => ["http://es-master:9200"]    index => "%{[fields][service]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"  }}

filebeat.inputs: - type: log  enabled: true  paths:   # 当前目录下的所有.log文件   - /root/tmp/logs/*.log  fields:   service: "our31-java"  multiline.pattern: ^/[  multiline.negate: true  multiline.match: after - type: log  enabled: true  paths:   # 当前目录下的所有.log文件   - /root/tmp/log/*.log  fields:   service: "our31-nginx"filebeat.config.modules: path: ${path.config}/modules.d/*.yml reload.enabled: false# setup.template.settings:#  index.number_of_shards: 1# setup.dashboards.enabled: false# setup.kibana:#  host: "http://localhost:5601"# 不直接传输至ES#output.elasticsearch:# hosts: ["http://es-master:9200"]# index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"setup.ilm.enabled: falseoutput.logstash: hosts: ["logstash.server.com:5044"]  # Optional SSL. By default is off. # List of root certificates for HTTPS server verifications ssl.certificate_authorities: "./ssl/ca.crt" # Certificate for SSL client authentication ssl.certificate: "./ssl/client.crt" # Client Certificate Key ssl.key: "./ssl/client.key"# processors:#  - add_host_metadata: ~#  - add_cloud_metadata: ~

#生成ca私钥openssl genrsa 2048 > ca.key #使用ca私钥建立ca证书openssl req -new -x509 -nodes -key ca.key -subj /CN=elkCA/ CA/OU=Development/ group/O=HomeIT/ SIA/DC=elk/DC=com > ca.crt #生成服务器csr证书请求文件openssl req -newkey rsa:2048 -nodes -keyout server.key -subj /CN=logstash.server.com/OU=Development/ group/O=Home/ SIA/DC=elk/DC=com > server.csr #使用ca证书与私钥签发服务器证书openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 > server.crt #生成客户端csr证书请求文件openssl req -newkey rsa:2048 -nodes -keyout client.key -subj /CN=filebeat.client.com/OU=Development/ group/O=Home/ SIA/DC=elk/DC=com > client.csr #使用ca证书与私钥签发客户端证书openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 > client.crt

$ yum -y install httpd-tools

server {  ......    auth_basic "Kibana Auth";  auth_basic_user_file /usr/local/nginx/pwd/kibana/passwd;    ......}

$ nohup ./filebeat 2>&1 &

$ docker-compose up --build -d