Metricbeat error about scanning empty io.stat file

I'm deploying metricbeat and using the system and docker modules but keep getting an error message in my logs. My configs are below. The content of /hostfs/sys/fs/cgroup/io.stat on my system is nothing - it is an empty file.

I was redirected to this forum from Github.

The error message:

{
  "log.level": "error",
  "@timestamp": "2022-06-23T23:39:06.781Z",
  "log.origin": { "file.name": "metrics/metrics.go", "file.line": 294 },
  "message": "error getting cgroup stats for V2: error fetching stats for controller io: error fetching IO stats: error getting io.stats for path /hostfs/sys/fs/cgroup: error scanning file: /hostfs/sys/fs/cgroup/io.stat: input does not match format",
  "service.name": "metricbeat",
  "ecs.version": "1.6.0",
}

From docker-compose file (its 2k+ lines long, this is a snippet):

version: "3.9"

networks:
  beats_net:
    internal: false

services:

  metricbeat:
    image: docker.elastic.co/beats/metricbeat:8.2.2

    hostname: 'test.lab'

    user: root

    privileged: true


    volumes:
      - /home/thebench/test/sim1/simtest/beats/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
      - /proc:/hostfs/proc:ro
      - /:/hostfs:ro
      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro

    restart: unless-stopped

    networks:
      - beats_net

My metricbeat.yml:

logging.to_stderr: true                                                                                                                                             [52/4739]
logging.level: error                                                                  
                                           
output.logstash:                     
  hosts: ["logstash.test.lab:5044"]
                                           
monitoring.enabled: false
                                           
metricbeat.autodiscover:     
  providers:
    - type: docker                         
      hints.enabled: true 
                                           
metricbeat.modules:              
                                           
  # metrics that are collected more frequently
  - module: system
    hostfs: /hostfs   
    period: 10s            
    cpu.metrics:  [percentages]                                                       
    core.metrics: [percentages]
    metricsets:
      - cpu      # no priv
      - load     # no priv
      - memory   # no priv, e.g. swap, total, used, free, actual
      - network  # no priv, network metrics for interfaces

  # metrics that are collected less frequently
  - module: system
    hostfs: /hostfs
    period: 1m
    processors:
      - drop_event.when.regexp:
          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
    metricsets:
      - filesystem  # no priv, e.g. size, free, used

  # metrics that are collected rarely
    hostfs: /hostfs
  - module: system
    period: 15m
    metricsets:
      - uptime      # no priv


  - module: docker
    metricsets:
      - cpu
      - diskio
      - event
      - info
      - memory
      - network
    hosts:
      - unix:///var/run/docker.sock
    period: 10s

  - module: docker
    period: 1m
    hosts:
      - unix:///var/run/docker.sock
    metricsets:
      - healthcheck

  - module: docker
    period: 15m
    hosts:
      - unix:///var/run/docker.sock
    metricsets:
      - container
      - image

processors:
  - add_fields:
      target: project
      fields:
        name: simtest

  - drop_fields:
      ignore_missing: true
      fields:
        - docker.container.labels
        - docker.event.actor.attributes

  - add_host_metadata:
      netinfo.enabled: true

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.