I have a simple express app with Nginx and I use Filebeat with ELK stack. Filebeat takes in charge of streaming log file from Nginx to Logstash then processing it and visualize to Kibana. This pipeline works fine. However, I wanted to check how can I incorporate Packetbeat to this pipeline to check the different possibilities that gives to me.
My docker-compose.yml
is the following:
version: '3.2'
services:
elasticsearch:
build:
context: elasticsearch/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./elasticsearch/config/elasticsearch.yml
target: /usr/share/elasticsearch/config/elasticsearch.yml
read_only: true
- type: volume
source: elasticsearch
target: /usr/share/elasticsearch/data
ports:
- "9200:9200"
- "9300:9300"
environment:
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
ELASTIC_PASSWORD: changeme
networks:
- elk
logstash:
build:
context: logstash/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./logstash/config/logstash.yml
target: /usr/share/logstash/config/logstash.yml
read_only: true
- type: bind
source: ./logstash/pipeline
target: /usr/share/logstash/pipeline
read_only: true
ports:
- "5000:5000"
- "9600:9600"
expose:
- "5044"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- elk
depends_on:
- elasticsearch
kibana:
build:
context: kibana/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./kibana/config/kibana.yml
target: /usr/share/kibana/config/kibana.yml
read_only: true
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch
app:
build : ./app
volumes:
- ./app/:/usr/src/app
- /usr/src/app/node_modules/ # make node_module empty in container
command: npm start
ports:
- "3000:3000"
networks:
- elk
nginx:
build: ./nginx
volumes:
- ./nginx/config:/etc/nginx/conf.d
- ./nginx/log:/var/log/nginx
ports:
- "80:80"
- "443:443"
links:
- app:app
depends_on:
- app
networks:
- elk
filebeat:
build: ./filebeat
entrypoint: "filebeat -e -strict.perms=false"
volumes:
- ./filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
- ./nginx/log:/var/log/nginx
networks:
- elk
depends_on:
- app
- nginx
- logstash
- elasticsearch
- kibana
links:
- logstash
packetbeat:
build: ./packetbeat
restart: always
cap_add:
- NET_ADMIN
volumes:
- type: bind
source: ./packetbeat/config/packetbeat.yml
target: /usr/share/packetbeat/config/packetbeat.yml
read_only: true
network_mode: host
networks:
elk:
driver: bridge
volumes:
elasticsearch:
packetbeat:
The packetbeat.yml
for configuration is the following:
packetbeat.interfaces.device: any
packetbeat.flows:
timeout: 30s
period: 5s
packetbeat.protocols:
- type: icmp
enabled: true
- type: dns
ports: [53]
include_authorities: true
include_additionals: true
- type: http
ports: [80, 5601, 9200, 8080, 8081, 5000, 8002]
- type: amqp
ports: [5672]
- type: cassandra
ports: [9042]
- type: memcache
ports: [11211]
- type: mysql
ports: [3306]
- type: mongodb
ports: [27017]
- type: redis
ports: [6379]
- type: pgsql
ports: [5432]
- type: thrift
ports: [9090]
processors:
- add_cloud_metadata:
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
hosts: ["http://elasticsearch:9200"]
username: "elastic"
password: "changeme"
setup.kibana:
host: "http://kibana:5601"
username: "elastic"
password: "changeme"
#============================== Dashboards =====================================
# These settings control loading the sample dashboards to the Kibana index. Loading
# the dashboards is disabled by default and can be enabled either by setting the
# options here, or by using the `-setup` CLI flag.
setup.dashboards.enabled: true
logging.level: warning
logging.to_files: true
logging.to_syslog: false
logging.files:
path: /var/log/packetbeat
name: packetbeat.log
keepfiles: 2
permissions: 0644
The logs from the packetbeat are the following:
2022-05-30T10:46:50.621Z INFO instance/beat.go:606 Home path: [/usr/share/packetbeat] Config path: [/usr/share/packetbeat] Data path: [/usr/share/packetbeat/data] Logs path: [/usr/share/packetbeat/logs]
2022-05-30T10:46:50.627Z INFO instance/beat.go:614 Beat ID: ce831d3c-5dce-44f0-8353-c9bca19bf24d
2022-05-30T10:46:50.645Z INFO [seccomp] seccomp/seccomp.go:124 Syscall filter successfully installed
2022-05-30T10:46:50.645Z INFO [beat] instance/beat.go:902 Beat info {"system_info": {"beat": {"path": {"config": "/usr/share/packetbeat", "data": "/usr/share/packetbeat/data", "home": "/usr/share/packetbeat", "logs": "/usr/share/packetbeat/logs"}, "type": "packetbeat", "uuid": "ce831d3c-5dce-44f0-8353-c9bca19bf24d"}}}
2022-05-30T10:46:50.645Z INFO [beat] instance/beat.go:911 Build info {"system_info": {"build": {"commit": "a4be71b90ce3e3b8213b616adfcd9e455513da45", "libbeat": "7.3.1", "time": "2019-08-19T19:23:55.000Z", "version": "7.3.1"}}}
2022-05-30T10:46:50.645Z INFO [beat] instance/beat.go:914 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":12,"version":"go1.12.4"}}}
2022-05-30T10:46:50.647Z INFO [beat] instance/beat.go:918 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2022-05-30T06:32:16Z","containerized":true,"name":"docker-desktop","ip":["127.0.0.1/8","::1/128","192.168.65.4/32","fe80::540e:a7ff:fe70:721c/64","192.168.65.3/24","fe80::50:ff:fe00:1/64","172.17.0.1/16","172.21.0.1/16","172.20.0.1/16","172.31.0.1/16","172.30.0.1/16","172.24.0.1/16"],"kernel_version":"5.10.16.3-microsoft-standard-WSL2","mac":["56:0e:a7:70:72:1c","02:50:00:00:00:01","02:42:d5:46:bb:83","02:42:c0:78:f6:49","02:42:1d:e7:31:90","02:42:ef:1d:59:0b","02:42:11:ae:79:7b","02:42:b6:15:d2:c9","06:3e:db:fc:bc:40","da:60:24:42:a1:57","b2:4b:44:ef:47:21","7a:11:9f:73:e0:fb"],"os":{"family":"redhat","platform":"centos","name":"CentOS Linux","version":"7 (Core)","major":7,"minor":6,"patch":1810,"codename":"Core"},"timezone":"UTC","timezone_offset_sec":0}}}
2022-05-30T10:46:50.647Z INFO [beat] instance/beat.go:947 Process info {"system_info": {"process": {"capabilities": {"inheritable":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_admin","net_raw","sys_chroot","mknod","audit_write","setfcap"],"permitted":["net_admin","net_raw"],"effective":["net_admin","net_raw"],"bounding":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_admin","net_raw","sys_chroot","mknod","audit_write","setfcap"],"ambient":null}, "cwd": "/usr/share/packetbeat", "exe": "/usr/share/packetbeat/packetbeat", "name": "packetbeat", "pid": 1, "ppid": 0, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2022-05-30T10:46:49.520Z"}}}
2022-05-30T10:46:50.647Z INFO instance/beat.go:292 Setup Beat: packetbeat; Version: 7.3.1
2022-05-30T10:46:50.648Z INFO [index-management] idxmgmt/std.go:178 Set output.elasticsearch.index to 'packetbeat-7.3.1' as ILM is enabled.
2022-05-30T10:46:50.648Z INFO elasticsearch/client.go:170 Elasticsearch url: http://elasticsearch:9200
2022-05-30T10:46:50.648Z INFO [publisher] pipeline/module.go:97 Beat name: docker-desktop
2022-05-30T10:46:50.648Z INFO procs/procs.go:105 Process watcher disabled
2022-05-30T10:46:50.649Z WARN [cfgwarn] protos/protos.go:118 DEPRECATED: dictionary style protocols configuration has been deprecated. Please use list-style protocols configuration. Will be removed in version: 7.0.0
2022-05-30T10:46:50.649Z INFO [monitoring] log/log.go:118 Starting metrics logging every 30s
2022-05-30T10:46:50.650Z INFO instance/beat.go:421 packetbeat start running.
2022-05-30T10:46:53.628Z INFO add_cloud_metadata/add_cloud_metadata.go:347 add_cloud_metadata: hosting provider type not detected.
2022-05-30T10:46:54.629Z INFO pipeline/output.go:95 Connecting to backoff(elasticsearch(http://elasticsearch:9200))
2022-05-30T10:47:04.630Z WARN transport/tcp.go:53 DNS lookup failure "elasticsearch": lookup elasticsearch on 192.168.65.5:53: read udp 192.168.65.4:41347->192.168.65.5:53: i/o timeout
2022-05-30T10:47:05.932Z ERROR pipeline/output.go:100 Failed to connect to backoff(elasticsearch(http://elasticsearch:9200)): Get http://elasticsearch:9200: lookup elasticsearch on 192.168.65.5:53: read udp 192.168.65.4:41347->192.168.65.5:53: i/o timeout
2022-05-30T10:47:05.932Z INFO pipeline/output.go:93 Attempting to reconnect to backoff(elasticsearch(http://elasticsearch:9200)) with 1 reconnect attempt(s)
2022-05-30T10:47:10.266Z WARN transport/tcp.go:53 DNS lookup failure "elasticsearch": lookup elasticsearch on 192.168.65.5:53: no such host
2022-05-30T10:47:13.032Z ERROR pipeline/output.go:100 Failed to connect to backoff(elasticsearch(http://elasticsearch:9200)): Get http://elasticsearch:9200: lookup elasticsearch on 192.168.65.5:53: no such host
2022-05-30T10:47:13.033Z INFO pipeline/output.go:93 Attempting to reconnect to backoff(elasticsearch(http://elasticsearch:9200)) with 2 reconnect attempt(s)
2022-05-30T10:47:17.355Z WARN transport/tcp.go:53 DNS lookup failure "elasticsearch": lookup elasticsearch on 192.168.65.5:53: no such host
2022-05-30T10:47:20.652Z INFO [monitoring] log/log.go:145 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cpu":{"system":{"ticks":140,"time":{"ms":150}},"total":{"ticks":520,"time":{"ms":532},"value":0},"user":{"ticks":380,"time":{"ms":382}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":6},"info":{"ephemeral_id":"878eb582-5eb7-4eca-a8c9-6b6523eb642f","uptime":{"ms":30041}},"memstats":{"gc_next":38388752,"memory_alloc":25277368,"memory_total":44835472,"rss":179154944},"runtime":{"goroutines":30}},"dns":{"unmatched_responses":3},"libbeat":{"config":{"module":{"running":0}},"output":{"type":"elasticsearch"},"pipeline":{"clients":11,"events":{"active":108,"published":108,"retry":6,"total":108}}},"system":{"cpu":{"cores":12},"load":{"1":4.23,"15":1.03,"5":2.08,"norm":{"1":0.3525,"15":0.0858,"5":0.1733}}}}}}
Any idea of where is the mistake?