EP15: simple log server - elasticsearch

In this episode
---
install elasticsearch and kibana from a docker image
install filebeat and send log from /var/log/*.log to es
convert nginx access log to json format
send access log to elasticsearch and parse it
simple dashboard for nginx access log

docker
---
sudo apt update; sudo apt install docker.io -y
sudo usermod -aG docker rui
newgrp docker

install es via docker
---
https://www.elastic.co/guide/en/elast...
sudo sysctl -a | grep max_map
echo "vm.max_map_count=262144"| sudo tee /etc/sysctl.d/98-max_map_count.conf
ls -al /etc/sysctl.d
sudo sysctl -p /etc/sysctl.d/98-max_map_count.conf
sudo sysctl -a | grep max_map

curl -s 'https://registry.hub.docker.com/v2/re... | jq '."results"[]["name"]' | sort
#show es tag

docker pull elasticsearch:8.13.0
docker pull kibana:8.13.0
docker tag elasticsearch:8.13.0 elasticsearch:latest
docker tag kibana:8.13.0 kibana:latest

create docker network
---
docker network create elastic

run instance
---
echo '#!/bin/bash' > es.sh
cat >> es.sh << EOF
docker run -d --name es \
--net elastic -p 9200:9200 \
-e ES_JAVA_OPTS="-Xms1g -Xmx1g" \
-e discovery.type="single-node" \
-v es-data:/usr/share/elasticsearch/data \
-v es-config:/usr/share/elasticsearch/config \
-v /etc/localtime:/etc/localtime:ro \
--restart=always \
elasticsearch
EOF


cat >> es.sh << EOF
docker run -d --name kibana \
--net elastic -p 5601:5601 \
-v kibana-data:/usr/share/kibana/data \
-v kibana-config:/usr/share/kibana/config \
-v /etc/localtime:/etc/localtime:ro \
--restart=always \
kibana
EOF

kibana init
---
docker exec -it es /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana
docker exec -it kibana bin/kibana-verification-code
docker exec -it es /usr/share/elasticsearch/bin/elasticsearch-reset-password -u elastic -b


filebeat
---
https://www.elastic.co/guide/en/beats...

curl -L -O https://artifacts.elastic.co/download...

sudo dpkg -i filebeat-8.13.2-amd64.deb
sudo systemctl status filebeat
sudo systemctl enable --now filebeat
sudo systemctl status filebeat

filebeat es
---
output.elasticsearch:
hosts: ["192.168.122.244:9200"]
ssl.verification_mode: none
protocol: "https"
username: "elastic"
password: "{esip}"

filebeat input
---
type: log
enabled: true
paths:
/var/log/*.log
index: "var-log-%{+yyyy.MM.dd}"
tags: ["var-log"]

sudo filebeat test config
sudo systemctl restart filebeat # restart to reload filebeat.yml

disable local metadata
---

```
#processors
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
```

filebeat nginx
---
https://nginx.org/en/docs/http/ngx_ht...
The configuration always includes the predefined “combined” format:

log_format combined '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent"';
```
log_format json_combined escape=json
'{"remote_addr":"$remote_addr",'
'"remote_user":"$remote_user",'
'"time_local":"$time_local",'
'"request":"$request", '
'"status":"$status",'
'"body_bytes_sent":"$body_bytes_sent",'
'"http_referer":"$http_referer",'
'"http_user_agent":"$http_user_agent"}';

access_log /var/log/nginx/access.log json_combined;
```

filebeat input json
---
```
type: log
enabled: true
paths:
/var/log/nginx/access.log
json.keys_under_root: true
index: "nginx-accesslog-%{+yyyy.MM.dd}"
tags: ["nginx-accesslog"]
```

```[ref](https://www.elastic.co/guide/en/beats...)
type: filestream
enabled: true
paths:
/var/log/nginx/access.log
index: "nginx-accesslog-%{+yyyy.MM.dd}"
tags: ["nginx-accesslog"]
processors:
decode_json_fields:
fields: ["message"]
target: ""
overwrite_keys: true
```

filebeat least privilege
---
https://www.elastic.co/guide/en/beats...

kibana | mgt | security | role
create role, filebeat_publish (role ,
Cluster| monitor, read_ilm, read_pipeline
index | myindexname-* , create_doc + create_index

openssl rand -base64 15