Suricata filebeat logs not real time

Elastic Stack Beats
1

I have setup new cluster elasticsearch for IDS using suricata , I have set JVM for ingest, master, data01, data02 each 2GB. but when I start file beat I got this error

{"log.level":"info","@timestamp":"2022-12-12T14:50:09.911Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":186},"message":"Non-zero metrics in the last 30s","service.name":"filebeat","monitoring":{"metrics":{"beat":{"cgroup":{"memory":{"mem":{"usage":{"bytes":195510272}}}},"cpu":{"system":{"ticks":6330,"time":{"ms":220}},"total":{"ticks":91910,"time":{"ms":3010},"value":91910},"user":{"ticks":85580,"time":{"ms":2790}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":14},"info":{"ephemeral_id":"090bced0-f897-45a3-bbd5-946ecb1b7f05","uptime":{"ms":630131},"version":"8.5.1"},"memstats":{"gc_next":117449352,"memory_alloc":110172064,"memory_total":2863998144,"rss":218976256},"runtime":{"goroutines":46}},"filebeat":{"events":{"added":3584,"done":3584},"harvester":{"open_files":1,"running":1}},"libbeat":{"config":{"module":{"running":1}},"output":{"events":{"acked":3584,"active":200,"batches":77,"total":3584},"read":{"bytes":797433},"write":{"bytes":7080931}},"pipeline":{"clients":1,"events":{"active":4097,"published":3584,"total":3584},"queue":{"acked":3584}}},"registrar":{"states":{"current":2,"update":3584},"writes":{"success":25,"total":25}},"system":{"load":{"1":6.62,"15":5.27,"5":6,"norm":{"1":1.655,"15":1.3175,"5":1.5}}}},"ecs.version":"1.6.0"}}

this my filebeat config

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

queue.mem:
  events: 4096
  flush.min_events: 512
  flush.timeout: 1s

setup.template.settings:
  index.number_of_shards: 3

xpack.monitoring:
  enabled: true


output.elasticsearch:
  hosts: 'https://10.104.x.xxx:9200'
  username: 'xxx'
  password: 'xxx'
  ssl.verification_mode: none
  worker: 4
setup.kibana:
  host: "https://10.104.x.xxx:5601"
  username: 'xxx'
  password: 'xxx'
  ssl.verification_mode: none

and then this my suricata..yml

# Module: suricata
# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html

- module: suricata
  # All logs
  eve:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/suricata/eve.json"]

And also this docker-compose.yml

version: '3'
services:
  filebeat:
    image: docker.elastic.co/beats/filebeat:8.5.1
    container_name: filebeat
    user: root
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
      - /var/log/suricata:/usr/share/filebeat/logs
      - filebeat:/usr/share/filebeat/data
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - ./suricata.yml:/usr/share/filebeat/modules.d/suricata.yml
volumes:
  filebeat:
    name: filebeat
    driver_opts:
      type: "none"
      o: "bind,device:/mnt/disks/data/elasticstack/filebeat"
      device: "/mnt/disks/data/elasticstack/filebeat"version: '3'

services:
  filebeat:
    image: docker.elastic.co/beats/filebeat:8.5.1
#    build: .
    container_name: filebeat
    user: root
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
      - /var/log/suricata:/usr/share/filebeat/logs
      - filebeat:/usr/share/filebeat/data
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - ./suricata.yml:/usr/share/filebeat/modules.d/suricata.yml
volumes:
  filebeat:
    name: filebeat
    driver_opts:
      type: "none"
      o: "bind,device:/mnt/disks/data/elasticstack/filebeat"
      device: "/mnt/disks/data/elasticstack/filebeat"

This stack for dev purpose, I think log from eve.json not too fast, but why log on filebeat-* not real time (have a delay about 15minutes)
I try using 4 worker for output elastic but same problem.

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.