Filebeats 7.4.2/nomad OOM error

[djesrani (darshan jesrani)]

Hello all,

I've been running into Out Of Memory/"task killed" issues deploying Filebeats and docker-gen into our production cluster.

Nomad version is 0.9.4, Filebeats version is 7.4.2. Memory allocation to Filebeats by Nomad is currently set to 256 MB, logs - max_files is set to 10, max_file_size is set to 15.

I've been reading that there were memory leak issues associated with this type of problem, but they seem to have been remediated since 6.x and subsequently closed.

Filebeats 7.4.2 was deployed for about a week before these problems started to re-appear so that seems to speak to something related to accumulation/lack of GC/harvesting?

Am I missing some sort of configuration option, environment variable or flag which might mitigate this kind of thing?

Any insight would be greatly appreciated.

Thanks very much in advance,

Darshan Jesrani
Analytic Partners, Inc.
New York , NY

[exekias (Carlos Pérez Aradros)]

Hi @djesrani,

Could you share some more details about your deployment? Including the config file.

Best regards

[djesrani (darshan jesrani)]

Hi @exekias , thanks for asking -- below is the config:

job "filebeats" {
type = "system"
datacenters = ["[[.datacenter.value]]"]

group "filebeat" {
task "filebeat" {
driver = "docker"
user = "root"

  config {
    image = "docker.elastic.co/beats/filebeat:7.4.2"

    volumes = [
      "/var/run/docker.sock:/var/run/docker.sock",
      "/var/lib/docker:/var/lib/docker",
      "/var/nomad/alloc:/var/nomad/alloc",
      "local/filebeat_yml:/usr/share/filebeat/filebeat.yml",
    ]

    dns_servers = ["${attr.unique.network.ip-address}"]
  } #conf

  env {
    ELASTICSEARCH_HOSTS    = "elk.service.lg1.consul:9200"
    ELASTICSEARCH_USERNAME = "my_login"
    ELASTICSEARCH_PASSWORD = "my_password"
  }

  resources {
    memory = 256
  } #res

  logs {
    max_files     = 10
    max_file_size = 15
  }

  template {
    data = <<EOH

filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false

processors:

• add_cloud_metadata: ~

output.elasticsearch:
hosts: '{ELASTICSEARCH_HOSTS}' username: '{ELASTICSEARCH_USERNAME}'
password: '${ELASTICSEARCH_PASSWORD}'

filebeat.config.inputs:
enabled: true
path: /alloc/data/*.yml
reload.enabled: true
reload.period: 10s
EOH

    destination = "local/filebeat_yml"
  }
}

task "dockergen" {
  leader = true
  driver = "docker"

  config {
    image = "jwilder/docker-gen:latest"

    args = [
      "-watch",
      "-include-stopped",
      "-wait",
      "5s:10s",
      "/etc/docker-gen/templates/filebeat.tmpl",
      "/alloc/data/inputs.yml",
    ]

    volumes = [
      "/var/run/docker.sock:/tmp/docker.sock",
      "local/filebeat_tmpl:/etc/docker-gen/templates/filebeat.tmpl",
    ]

    dns_servers = ["${attr.unique.network.ip-address}"]
  }

  env {
    "NOMAD_ALLOC" = "/var/nomad/alloc"
    "SERVER_HOST" = "${node.unique.name}"
  }

  template {
    left_delimiter  = "{{{"
    right_delimiter = "}}}"

    data = <<EOH

{{/* this is a docker-gen template. See docker-gen for more details /}}
{{ $nomad_alloc := .Env.NOMAD_ALLOC }}
{{ range $key, $value := . }}
{{ $nomad_id := $value.Env.NOMAD_ALLOC_ID }}
- type: log
paths:
{{ if $nomad_id }}
- {{ $nomad_alloc }}/{{ $nomad_id }}/alloc/logs/{{ $value.Env.NOMAD_TASK_NAME }}.stdout.
- {{ $nomad_alloc }}/{{ $nomad_id }}/alloc/logs/{{ $value.Env.NOMAD_TASK_NAME }}.stderr.*
{{ else }}
- /var/lib/docker/containers/{{ $value.ID }}/{{ $value.ID }}-json.log
{{ end }}
document_type: docker
ignore_older: 30m
scan_frequency: 5s
fields_under_root: true
fields:
docker/id: {{ $value.ID }}
docker/image: {{ if $value.Image.Registry }}{{ $value.Image.Registry }}/{{ end }}{{ $value.Image.Repository }}{{ if $value.Image.Tag }}:{{ $value.Image.Tag }}{{ else }}:latest{{ end }}
docker/name: {{ $value.Name }}
{{ if $nomad_id }}docker/task_name: {{ $value.Env.NOMAD_TASK_NAME }}{{end}}
{{ if $nomad_id }}docker/nomad_dc: {{ $value.Env.NOMAD_DC }}{{end}}
{{ if $nomad_id }}docker/nomad_alloc_id: {{ $value.Env.NOMAD_ALLOC_ID }}{{end}}
{{ if $nomad_id }}docker/nomad_alloc_index: {{ $value.Env.NOMAD_ALLOC_INDEX }}{{end}}
docker/hostname: {{ value.Hostname }} docker/server_host: {{ .Env.SERVER_HOST }}
{{ range $k, $v := $value.Labels }}
docker/label/{{ $k }}: {{ $v }}
{{ end }}
{{ end }}
EOH

    destination = "local/filebeat_tmpl"
  }

  resources {
    memory = 128
  }

  logs {
    max_files     = 10
    max_file_size = 15
  }
}

}
}