Best Practices for Deploying ELK7.3.0 Log Collection Service with Docker

# Modify the configuration sysctl.conf
vi /etc/sysctl.conf
# Add the following configuration:
vm.max_map_count=262144
# Reload:
sysctl -p
# Finally, restart elasticsearch to start successfully.

# Install the dependent tool bash-complete
yum install -y bash-completion
ource /usr/share/bash-completion/completions/docker
source /usr/share/bash-completion/bash_completion

mkdir -p {/mnt/es1/master/data,/mnt/es1/master/logs}
vim /mnt/es1/master/conf/es-master.yml

# Cluster name cluster.name: es-cluster
#Node namenode.name: es-master
# Whether it can become a master nodenode.master: true
# Whether to allow the node to store data, enabled by default node.data: false
# Network binding network.host: 0.0.0.0
# Set the http port for external services http.port: 9200
# Set the TCP port for communication between nodes transport.port: 9300
# Cluster discovery discovery.seed_hosts:
 - 172.17.0.2:9300
 - 172.17.0.3:9301
# Manually specify the name or IP address of all nodes that can become master nodes. These configurations will be calculated in the first election cluster.initial_master_nodes:
 - 172.17.0.2
# Support cross-domain access http.cors.enabled: true
http.cors.allow-origin: "*"
# Security authentication xpack.security.enabled: false
#http.cors.allow-headers: "Authorization"
bootstrap.memory_lock: false
bootstrap.system_call_filter: false

#Solve cross-domain issues#http.cors.enabled: true
#http.cors.allow-origin: "*"
#http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
#http.cors.allow-headers: "X-Requested-With, Content-Type, Content-Length, X-User"

# Pull the image. You can build the container directly and ignore this step. docker pull elasticsearch:7.3.0

# Build the container## Map 5601 to the port reserved for Kibanadocker run -d -e ES_JAVA_OPTS="-Xms256m -Xmx256m" \
-p 9200:9200 -p 9300:9300 -p 5601:5601 \
-v /mnt/es1/master/conf/es-master.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
-v /mnt/es1/master/data:/usr/share/elasticsearch/data \
-v /mnt/es1/master/logs:/usr/share/elasticsearch/logs \
-v /etc/localtime:/etc/localtime \
--name es-master elasticsearch:7.3.0

mkdir -p {/mnt/es1/slave1/data,/mnt/es1/slave1/logs}
vim /mnt/es1/slave1/conf/es-slave1.yml

# Cluster name cluster.name: es-cluster
#Node namenode.name: es-slave1
# Whether it can become a master nodenode.master: true
# Whether to allow the node to store data, enabled by default node.data: true
# Network binding network.host: 0.0.0.0
# Set the http port for external services http.port: 9201
# Set the TCP port for communication between nodes transport.port: 9301
# Cluster discovery discovery.seed_hosts:
 - 172.17.0.2:9300
 - 172.17.0.3:9301
# Manually specify the name or IP address of all nodes that can become master nodes. These configurations will be calculated in the first election cluster.initial_master_nodes:
 - 172.17.0.2
# Support cross-domain access http.cors.enabled: true
http.cors.allow-origin: "*"
# Security authentication xpack.security.enabled: false
#http.cors.allow-headers: "Authorization"
bootstrap.memory_lock: false
bootstrap.system_call_filter: false

# Pull the image. You can build the container directly and ignore this step. docker pull elasticsearch:7.3.0

# Build container docker run -d -e ES_JAVA_OPTS="-Xms256m -Xmx256m" \
-p 9201:9200 -p 9301:9300 \
-v /mnt/es1/slave1/conf/es-slave1.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
-v /mnt/es1/slave1/data:/usr/share/elasticsearch/data \
-v /mnt/es1/slave1/logs:/usr/share/elasticsearch/logs \
-v /etc/localtime:/etc/localtime \
--name es-slave1 elasticsearch:7.3.0

# View the master and slave container IP
docker inspect es-master
docker inspect es-slave1 

docker restart es-master

docker restart es-slave1

# View es logs docker logs -f --tail 100f es-master

vim /mnt/kibana.yml
#
## ** THIS IS AN AUTO-GENERATED FILE **
##
#
## Default Kibana configuration for docker target
server.name: kibana
#Configure Kibana's remote access server.host: "0.0.0.0"
#Configure es access address elasticsearch.hosts: [ "http://127.0.0.1:9200" ]
#Chinese interface i18n.locale: "zh-CN"

#xpack.monitoring.ui.container.elasticsearch.enabled: true

docker ps | grep es-master 

# Pull the image. You can build the container directly and ignore this step. docker pull docker.elastic.co/kibana/kibana:7.3.0

# Build container## --network=container means sharing container network docker run -it -d \
-v /mnt/kibana.yml:/usr/share/kibana/config/kibana.yml \
-v /etc/localtime:/etc/localtime \
-e ELASTICSEARCH_URL=http://172.17.0.2:9200 \
--network=container:40eff5876ffd \
--name kibana docker.elastic.co/kibana/kibana:7.3.0

docker logs -f --tail 100f kibana 

input {
  # Source beats
  beats {
    # Port port => "5044"
  }
}
# Analysis and filtering plugins, multiple filters are possible {
  grok {
	# Where grok expressions are stored patterns_dir => "/grok"
	
	# grok expression rewrite # match => {"message" => "%{SYSLOGBASE} %{DATA:message}"}
	
	# Delete the native message field overwrite => ["message"]

  # Define your own format match => {
		"message" => "%{URIPATH:request} %{IP:clientip} %{NUMBER:response:int} \"%{WORD:sources}\" (?:%{URI:referrer}|-) \[%{GREEDYDATA:agent}\] ​​\{%{GREEDYDATA:params}\}"
	}
  }
 # Query classification plugin geoip {
    source => "message"
  }
}
output {
	# Select elasticsearch
	elasticsearch
		# es cluster hosts => ["http://172.17.0.2:9200"]
      #username => "root"
      #password => "123456"

		# Index format index => "omc-block-server-%{[@metadata][version]}-%{+YYYY.MM.dd}"

		# Set to true to indicate that if you have a custom template called logstash, your custom template will overwrite the default template logstash
		template_overwrite => true
	}
}

# Pull the image, you can build the container directly and ignore this step docker pull logstash:7.3.1 

# Build container# xpack.monitoring.enabled turns on X-Pack's security and monitoring services# xpack.monitoring.elasticsearch.hosts sets the ES address, 172.17.0.2 is the es-master container IP
# Docker allows you to execute some commands when the container starts. logsatsh -f means running logstash by specifying the configuration file. /usr/share/logstash/config/logstash-sample.conf is the directory file in the container. docker run -p 5044:5044 -d \
-v /mnt/logstash-filebeat.conf:/usr/share/logstash/config/logstash-sample.conf \
-v /etc/localtime:/etc/localtime \
-e elasticsearch.hosts=http://172.17.0.2:9200 \
-e xpack.monitoring.enabled=true \
-e xpack.monitoring.elasticsearch.hosts=http://172.17.0.2:9200 \
--name logstash logstash:7.3.1 -f /usr/share/logstash/config/logstash-sample.conf

input {
  # Source beats
  beats {
    # Port port => "5044"
  }
  file {
    type => "server-log"
    path => "/logs/*.log"
    start_position => "beginning"
    codec=>multiline{
        // Regular expression, all logs with prefix "20". If your logs have prefixes like "[2020-06-15", you can replace it with "^["
        pattern => "^20"
        // Whether to negate the regular rule negate => true
        // previous means merge into the previous line, next means merge into the next line what => "previous"
    }

  }
}

## /mnt/omc-dev/logs is the application log directory. The application deployment directory must be mapped out. mkdir -p {/mnt/omc-dev/logs,/mnt/filebeat/logs,/mnt/filebeat/data}
vim /mnt/filebeat/filebeat.yml

filebeat.inputs:
- type: log
 enabled: true
 paths:
  # All .log files in the current directory - /home/project/spring-boot-elasticsearch/logs/*.log
 multiline.pattern: '^20'
 multiline.negate: true
 multiline.match: previous

logging.level: debug

filebeat.config.modules:
 path: ${path.config}/modules.d/*.yml
 reload.enabled: false

setup.template.settings:
 index.number_of_shards: 1

setup.dashboards.enabled: false

setup.kibana:
 host: "http://172.17.0.2:5601"

# Not directly transferred to ES
#output.elasticsearch:
# hosts: ["http://es-master:9200"]
# index: "filebeat-%{[beat.version]}-%{+yyyy.MM.dd}"

output.logstash:
 hosts: ["172.17.0.5:5044"]

#scan_frequency: 1s
close_inactive: 12h
backoff: 1s
max_backoff: 1s
backoff_factor: 1
flush.timeout: 1s

processors:
 - add_host_metadata: ~
 - add_cloud_metadata: ~

# Pull the image, you can build the container directly and ignore this step docker pull docker.elastic.co/beats/filebeat:7.3.0

# Build container## --link logstash Connect the specified container to the current connection. You can set an alias to avoid the situation where the container cannot be connected due to dynamic changes caused by the ip method. logstash is the container name. docker run -d -v /mnt/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml \
-v /mnt/omc-dev/logs:/home/project/spring-boot-elasticsearch/logs \
-v /mnt/filebeat/logs:/usr/share/filebeat/logs \
-v /mnt/filebeat/data:/usr/share/filebeat/data \
-v /etc/localtime:/etc/localtime \
--link logstash --name filebeat docker.elastic.co/beats/filebeat:7.3.0

docker logs -f --tail 100f filebeat