Docker logs in metrics

Elastic Stack Beats
,
1

Hi there.
I set up Metricbeat in my Rancher environment.
I set it up with module docker so far.
In tab metrics in kibana I can see metrics on docker containers but can not see logs, only this:

docker.memory

failed to find message

In this section:

obraz
obraz1910×388 53.6 KB

How do I fix that?

I have metricbeat index, it has plenty of fields, I can see in logs he is using event.dataset. But which field is responsible for the "failed to find message"?

My config:

- module: docker
  metricsets:
- "container"
- "cpu"
- "diskio"
- "event"
- "healthcheck"
- "info"
- "image"
- "memory"
- "network"
  hosts: ["unix:///var/run/docker.sock"]
  period: 30s
  enabled: true

metricbeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
  reload.period: 10s

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

setup.template.name: "metricbeat"
setup.template.pattern: "metricbeat-*"
setup.template.settings:
  index.number_of_shards: 1
  index.number_of_replicas: 1
  index.codec: best_compression

#setup.dashboards.enabled: true

setup.ilm.enabled: false

output.elasticsearch:
  hosts: ["elk-pl:9200"]
  index: "metricbeat-%{+yyyy.MM.dd}"

In rancher there is of course set:

- user: root
- /var/run/docker.sock:/var/run/docker.sock

Help appreciated :slight_smile:

2

Could you please share the debug logs of Metricbeat?

3 
2020-09-24T11:42:51.140Z	DEBUG	[processors]	processing/processors.go:187	Publish event: {
  "@timestamp": "2020-09-24T11:42:48.802Z",
  "@metadata": {
    "beat": "metricbeat",
    "type": "_doc",
    "version": "7.9.1"
  },
  "container": {
    "id": "cdc713b8bc5f107f0d89d5ae2e5c579a5c5fcb71cc9f0ab999a37f4349752e48",
    "name": "r-app-filebeat5-1-ce6e62c5",
    "image": {
      "name": "filbeat-address:7.7.0"
    },
    "runtime": "docker"
  },
  "docker": {
    "container": {
      "labels": {
        "io_rancher_stack_service_name": "app/app/filebeat5",
        "org_label-schema_name": "filebeat",
        "org_opencontainers_image_licenses": "GPL-2.0-only",
        "io_rancher_service_launch_config": "filebeat5",
        "io_rancher_container_name": "app-filebeat5-1",
        "io_rancher_project_name": "app",
        "io_rancher_container_uuid": "ce6e62c5-041e-4af0-8aa4-75676b5bfca1",
        "io_rancher_container_mac_address": "02:1e:b5:51:ab:4f",
        "license": "Elastic License",
        "description": "Filebeat sends log files to Logstash or directly to Elasticsearch.",
        "org_label-schema_version": "7.7.0",
        "org_opencontainers_image_created": "2020-05-04 00:00:00+01:00",
        "org_label-schema_vcs-ref": "5e69e25b920e3d93bec76a09a31da3ab35a55607",
        "org_label-schema_vendor": "Elastic",
        "org_opencontainers_image_vendor": "CentOS",
        "io_rancher_container_hostname_override": "container_name",
        "io_rancher_container_ip": "10.42.248.181/16",
        "org_label-schema_url": "https://www.elastic.co/products/beats/filebeat",
        "io_rancher_project_service_name": "app/app/filebeat5",
        "org_opencontainers_image_title": "CentOS Base Image",
        "io_rancher_service_deployment_unit": "1f685122-24ea-4765-a830-b41babbe5269",
        "org_label-schema_license": "Elastic License",
        "io_rancher_cni_network": "ipsec",
        "io_rancher_cni_wait": "true",
        "io_rancher_environment_uuid": "330ad713-f0d1-4617-8402-e0f8461cbb40",
        "org_label-schema_vcs-url": "github.com/elastic/beats/v7",
        "org_label-schema_build-date": "2020-05-12T00:53:01Z",
        "org_label-schema_schema-version": "1.0",
        "io_rancher_stack_name": "app"
      }
    },
    "diskio": {
      "read": {
        "queued": 0,
        "ops": 93896,
        "bytes": 742446080,
        "rate": 0,
        "service_time": 0,
        "wait_time": 0
      },
      "write": {
        "wait_time": 0,
        "queued": 0,
        "ops": 2313080,
        "bytes": 11304315904,
        "rate": 7.570001874235229,
        "service_time": 0
      },
      "summary": {
        "bytes": 12046761984,
        "rate": 7.570001874235229,
        "service_time": 0,
        "wait_time": 0,
        "queued": 0,
        "ops": 2406976
      },
      "reads": 0,
      "writes": 7.570001874235229,
      "total": 7.570001874235229
    }
  },
  "service": {
    "address": "/var/run/docker.sock",
    "type": "docker"
  },
  "event": {
    "duration": 2337320974,
    "dataset": "docker.diskio",
    "module": "docker"
  },
  "ecs": {
    "version": "1.5.0"
  },
  "host": {
    "name": "Metricbeat-metricbeat-4"
  },
  "agent": {
    "type": "metricbeat",
    "version": "7.9.1",
    "hostname": "Metricbeat-metricbeat-4",
    "ephemeral_id": "6ed8a084-03b0-4c77-88d4-bf4bcd66bfab",
    "id": "f255cb20-1e9e-4e23-b985-88cf2fb96846",
    "name": "Metricbeat-metricbeat-4"
  },
  "metricset": {
    "name": "diskio",
    "period": 30000
  }
}
2020-09-24T11:42:51.141Z	DEBUG	[processors]	processing/processors.go:187	Publish event: {
  "@timestamp": "2020-09-24T11:42:48.819Z",
  "@metadata": {
    "beat": "metricbeat",
    "type": "_doc",
    "version": "7.9.1"
  },
  "event": {
    "duration": 2321375962,
    "dataset": "docker.memory",
    "module": "docker"
  },
  "host": {
    "name": "Metricbeat-metricbeat-4"
  },
  "agent": {
    "ephemeral_id": "6ed8a084-03b0-4c77-88d4-bf4bcd66bfab",
    "id": "f255cb20-1e9e-4e23-b985-88cf2fb96846",
    "name": "Metricbeat-metricbeat-4",
    "type": "metricbeat",
    "version": "7.9.1",
    "hostname": "Metricbeat-metricbeat-4"
  },
  "ecs": {
    "version": "1.5.0"
  },
  "container": {
    "id": "294a5233a9bb8bdba0b5202755d59c885d8f030cf1a3ccaa3a0502465c7f1500",
    "name": "r-app2-filebeat-7-30900e89",
    "image": {
      "name": "/filebeat:7.7.0"
    },
    "runtime": "docker"
  },
  "docker": {
    "container": {
      "labels": {
        "io_rancher_container_uuid": "30900e89-1762-47d9-907b-0fb8a46a546c",
        "io_rancher_container_ip": "10.42.227.246/16",
        "io_rancher_service_requested_host_id": "360",
        "org_label-schema_vcs-ref": "5e69e25b920e3d93bec76a09a31da3ab35a55607",
        "org_opencontainers_image_vendor": "CentOS",
        "io_rancher_stack_name": "app2",
        "io_rancher_container_hostname_override": "container_name",
        "org_label-schema_name": "filebeat",
        "io_rancher_project_service_name": "app2/app2/filebeat",
        "license": "Elastic License",
        "org_label-schema_vcs-url": "github.com/elastic/beats/v7",
        "org_label-schema_build-date": "2020-05-12T00:53:01Z",
        "io_rancher_container_pull_image": "always",
        "org_label-schema_vendor": "Elastic",
        "io_rancher_container_name": "app2-filebeat-7",
        "io_rancher_project_name": "app2",
        "io_rancher_cni_wait": "true",
        "org_label-schema_license": "Elastic License",
        "io_rancher_service_launch_config": "filebeat",
        "org_opencontainers_image_created": "2020-05-04 00:00:00+01:00",
        "io_rancher_container_mac_address": "02:1e:b5:98:4f:83",
        "org_label-schema_url": "https://www.elastic.co/products/beats/filebeat",
        "io_rancher_service_deployment_unit": "e86c067f-9370-4c9e-b408-d1bf1abff972",
        "org_opencontainers_image_title": "CentOS Base Image",
        "org_opencontainers_image_licenses": "GPL-2.0-only",
        "org_label-schema_version": "7.7.0",
        "io_rancher_stack_service_name": "app2/app2/filebeat",
        "description": "Filebeat sends log files to Logstash or directly to Elasticsearch.",
        "org_label-schema_schema-version": "1.0",
        "io_rancher_cni_network": "ipsec",
        "io_rancher_environment_uuid": "330ad713-f0d1-4617-8402-e0f8461cbb40"
      }
    },
    "memory": {
      "fail": {
        "count": 0
      },
      "limit": 536870912,
      "rss": {
        "total": 27250688,
        "pct": 0.05075836181640625
      },
      "usage": {
        "pct": 0.07782745361328125,
        "max": 57761792,
        "total": 41783296
      },
      "stats": {
        "unevictable": 0,
        "active_anon": 2.7250688e+07,
        "total_inactive_anon": 0,
        "total_pgpgout": 25441,
        "pgpgin": 24074,
        "hierarchical_memory_limit": 5.36870912e+08,
        "inactive_anon": 0,
        "total_pgfault": 13582,
        "total_rss": 2.7250688e+07,
        "cache": 2.732032e+06,
        "pgmajfault": 52,
        "pgpgout": 25441,
        "dirty": 0,
        "total_rss_huge": 2.3068672e+07,
        "total_writeback": 0,
        "rss_huge": 2.3068672e+07,
        "total_active_file": 1.687552e+06,
        "total_inactive_file": 1.04448e+06,
        "total_mapped_file": 1.544192e+06,
        "active_file": 1.687552e+06,
        "hierarchical_memsw_limit": 1.073741824e+09,
        "inactive_file": 1.04448e+06,
        "pgfault": 13582,
        "rss": 2.7250688e+07,
        "total_dirty": 0,
        "total_pgmajfault": 52,
        "total_pgpgin": 24074,
        "total_unevictable": 0,
        "writeback": 0,
        "mapped_file": 1.544192e+06,
        "total_active_anon": 2.7250688e+07,
        "total_cache": 2.732032e+06
      }
    }
  },
  "metricset": {
    "name": "memory",
    "period": 30000
  },
  "service": {
    "address": "/var/run/docker.sock",
    "type": "docker"
  }
}
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.