Metricbeat error about scanning empty io.stat file

[matutter]

I'm deploying metricbeat and using the system and docker modules but keep getting an error message in my logs. My configs are below. The content of /hostfs/sys/fs/cgroup/io.stat on my system is nothing - it is an empty file.

I was redirected to this forum from Github.

The error message:

{
  "log.level": "error",
  "@timestamp": "2022-06-23T23:39:06.781Z",
  "log.origin": { "file.name": "metrics/metrics.go", "file.line": 294 },
  "message": "error getting cgroup stats for V2: error fetching stats for controller io: error fetching IO stats: error getting io.stats for path /hostfs/sys/fs/cgroup: error scanning file: /hostfs/sys/fs/cgroup/io.stat: input does not match format",
  "service.name": "metricbeat",
  "ecs.version": "1.6.0",
}

From docker-compose file (its 2k+ lines long, this is a snippet):

version: "3.9"

networks:
  beats_net:
    internal: false

services:

  metricbeat:
    image: docker.elastic.co/beats/metricbeat:8.2.2

    hostname: 'test.lab'

    user: root

    privileged: true


    volumes:
      - /home/thebench/test/sim1/simtest/beats/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
      - /proc:/hostfs/proc:ro
      - /:/hostfs:ro
      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro

    restart: unless-stopped

    networks:
      - beats_net

My metricbeat.yml:

logging.to_stderr: true                                                                                                                                             [52/4739]
logging.level: error                                                                  
                                           
output.logstash:                     
  hosts: ["logstash.test.lab:5044"]
                                           
monitoring.enabled: false
                                           
metricbeat.autodiscover:     
  providers:
    - type: docker                         
      hints.enabled: true 
                                           
metricbeat.modules:              
                                           
  # metrics that are collected more frequently
  - module: system
    hostfs: /hostfs   
    period: 10s            
    cpu.metrics:  [percentages]                                                       
    core.metrics: [percentages]
    metricsets:
      - cpu      # no priv
      - load     # no priv
      - memory   # no priv, e.g. swap, total, used, free, actual
      - network  # no priv, network metrics for interfaces

  # metrics that are collected less frequently
  - module: system
    hostfs: /hostfs
    period: 1m
    processors:
      - drop_event.when.regexp:
          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
    metricsets:
      - filesystem  # no priv, e.g. size, free, used

  # metrics that are collected rarely
    hostfs: /hostfs
  - module: system
    period: 15m
    metricsets:
      - uptime      # no priv


  - module: docker
    metricsets:
      - cpu
      - diskio
      - event
      - info
      - memory
      - network
    hosts:
      - unix:///var/run/docker.sock
    period: 10s

  - module: docker
    period: 1m
    hosts:
      - unix:///var/run/docker.sock
    metricsets:
      - healthcheck

  - module: docker
    period: 15m
    hosts:
      - unix:///var/run/docker.sock
    metricsets:
      - container
      - image

processors:
  - add_fields:
      target: project
      fields:
        name: simtest

  - drop_fields:
      ignore_missing: true
      fields:
        - docker.container.labels
        - docker.event.actor.attributes

  - add_host_metadata:
      netinfo.enabled: true