Filebeat not filtering for separate index

I have problem with my filebeat configuration, it doesnt filter logs to separate index in elasticsearch.
My configuration looks like:

    logging:
 level: error
 metrics:
  enabled: false


filebeat.modules:
  - module: mongodb
  - module: kafka
  - module: traefik

filebeat.autodiscover:
  providers:
    - type: docker
      labels.dedot: true
      templates:
        - condition:
            equals:
              docker.container.labels.jsonLog: 'true'
          config:
            - type: container
              format: docker
              paths:
                - "/var/lib/docker/containers/${data.docker.container.id}/*.log"
              exclude_lines: [".*eureka endpoints.*", "DEBUG"]
              processors:
                - decode_json_fields:
                    when.equals:
                      docker.container.labels.jsonLog: 'true'
                    fields: ["message"]
                    target: "logstash"
                    overwrite_keys: true
                - add_docker_metadata: ~
                - script:
                    lang: javascript
                    id: shorten_message
                    source: >
                      function process(event) {
                        var message = event.Get("logstash.message");
                        if (message != null && message.length > 900000) {
                          event.Put("logstash.message", message.substring(0, 900000));
                        }
                      }
                - drop_fields:
                    fields: ["message"]
        - condition:
            equals:
              docker.container.labels.proxy: 'true'
          config:
            - type: container
              format: docker
              paths:
                - "/var/lib/docker/containers/${data.docker.container.id}/*.log"
              processors:
                - decode_json_fields:
                    when.equals:
                      docker.container.labels.proxy: 'true'
                    fields: ["message"]
                    target: "logstash"
                    overwrite_keys: true
                - add_docker_metadata: ~
                - drop_fields:
                    fields: ["message"]

output.elasticsearch:
  hosts: ["elasticsearch-master:9200", "elasticsearch-slave:9200"]
  pipelines:
    - pipeline: "logstash-json-timestamp"
      when.equals:
        docker.container.labels.jsonLog: 'true'
  indices:
    - index: "filebeat-%{[beat.version]}-%{[docker.container.labels.com.docker.swarm.service.name]}-%{+yyyy.MM.dd}"
      when:
        or:
          - equals:
              docker.container.labels.jsonLog: 'true'
          - equals:
              container.labels.jsonLog: 'true'
          - contains:
              docker.container.labels.jsonLog: 'tru'
          - contains:
              container.labels.jsonLog: 'tru'
          - has_fields: ['docker.container.labels.jsonLog']
    - index: "traefik-%{[beat.version]}-%{[docker.container.labels.com.docker.swarm.service.name]}-%{+yyyy.MM.dd}"
      when:
        or:
          - equals:
              docker.container.labels.proxy: 'true'
          - equals:
              container.labels.proxy: 'true'
          - equals:
              docker.container.labels.proxy: 'tru'
          - equals:
              container.labels.proxy: 'tru'
          - has_fields: ['docker.container.labels.proxy']

And also sample doument (too lengt post):

    {
  "_index": "filebeat-7.3.0-2019.08.03-000001",
  "_type": "_doc",
  "_id": "XuCyYGwBHJ227QPYFYO_",
  "_version": 1,
  "_score": null,
  "_source": {
    "@timestamp": "2019-08-05T07:32:55.857Z",
    "stream": "stdout",
    "logstash": {
      "OriginStatusLine": "200 OK",
      "RequestAddr": "automaton-mt.mitus-test.teta.net",
      "origin_X-Xss-Protection": "1 ; mode=block",
      "BackendName": "backend-gateway-mt",
      "Duration": 10862475,
      "OriginDuration": 10776795,
      "OriginStatus": 200,
      "downstream_Date": "Mon, 05 Aug 2019 07:32:55 GMT",
      "request_Accept-Language": "pl-PL,pl;q=0.9,en-US;q=0.8,en;q=0.7",
      "request_Authorization": "Bearer eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJraWtvbCIsInNjb3BlIjoiYXBpIiwiaXNzIjoia2lrb2wiLCJpYXQiOjE1NjQ5OTAzNTQsImV4cCI6MTU2NTA3Njc1NH0.t4pD5nuedDeZhbUYBtNvYGcUW5VwgyfqhP9iE20d6F9h_s-v3m-SDjyQX2ByEDKPjMx4JHGUdzgmSUvgnQKaqQ",
      "Overhead": 85680,
      "downstream_Referrer-Policy": "no-referrer",
      "level": "info",
      "ClientUsername": "-",
      "StartUTC": "2019-08-05T07:32:55.846776227Z",
      "downstream_Content-Type": "application/json;charset=UTF-8",
      "downstream_X-Content-Type-Options": "nosniff",
      "origin_Pragma": "no-cache",
      "request_User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
      "origin_Expires": "0",
      "ClientAddr": "10.48.92.178:62941",
      "FrontendName": "Host-automaton-mt-mitus-test-teta-net-automaton-review-mitus-test-teta-net-PathPrefixStrip-api-61",
      "RequestHost": "automaton-mt.mitus-test.teta.net",
      "RetryAttempts": 0,
      "downstream_Vary": "Accept-Encoding",
      "downstream_X-Xss-Protection": "1 ; mode=block",
      "origin_Referrer-Policy": "no-referrer",
      "origin_X-Content-Type-Options": "nosniff",
      "request_Accept": "application/json, text/plain, */*",
      "request_X-Forwarded-Proto": "http",
      "DownstreamStatus": 200,
      "RequestPath": "/api/taskmanagement/tasks/statistic/",
      "StartLocal": "2019-08-05T09:32:55.846776227+02:00",
      "time": "2019-08-05T09:32:55+02:00",
      "RequestCount": 6939,
      "DownstreamStatusLine": "200 OK",
      "RequestProtocol": "HTTP/1.1",
      "origin_X-Frame-Options": "DENY",
      "request_X-Forwarded-For": "127.0.0.1",
      "request_X-Forwarded-Prefix": "/api",
      "ClientPort": "62941",
      "origin_Content-Type": "application/json;charset=UTF-8",
      "request_Referer": "http://localhost:8080/tasks/user",
      "request_X-Forwarded-Host": "localhost:8080",
      "request_X-Forwarded-Port": "8080",
      "RequestLine": "GET /api/taskmanagement/tasks/statistic/ HTTP/1.1",
      "downstream_Cache-Control": "no-cache, no-store, max-age=0, must-revalidate",
      "msg": "",
      "request_Connection": "close",
      "BackendURL": {
        "Scheme": "http",
        "Path": "",
        "Fragment": "",
        "ForceQuery": false,
        "RawQuery": "",
        "Opaque": "",
        "User": null,
        "Host": "10.0.26.111:8080",
        "RawPath": ""
      }
    },
    "host": {
      "name": "1516ac37019c"
    },
    "log": {
      "file": {
        "path": "/var/lib/docker/containers/8d932184e0af1dc988db75de02981fc39de388aa940fc2dc4f53ed5c2713e0b8/8d932184e0af1dc988db75de02981fc39de388aa940fc2dc4f53ed5c2713e0b8-json.log"
      },
      "offset": 1388535
    },
    "ecs": {
      "version": "1.0.1"
    },
    "input": {
      "type": "container"
    },
    "docker": {
      "container": {
        "labels": {
          "org_opencontainers_image_url": "https://traefik.io",
          "com_docker_swarm_node_id": "vjpbn32i27e2l3xhw098iug65",
          "com_docker_swarm_task": "",
          "com_docker_swarm_service_id": "pg1nc3phr2wd7wpkexo6pijb7",
          "org_opencontainers_image_vendor": "Containous",
          "org_opencontainers_image_documentation": "https://docs.traefik.io",
          "proxy": "true",
          "com_docker_stack_namespace": "infrastructure",
          "com_docker_swarm_service_name": "infrastructure_traefik",
          "org_opencontainers_image_title": "Traefik",
          "org_opencontainers_image_description": "A modern reverse-proxy",
          "com_docker_swarm_task_name": "infrastructure_traefik.1.cdz042qgr260paryvqkgue8vn",
          "logstash": "false",
          "com_docker_swarm_task_id": "cdz042qgr260paryvqkgue8vn",
          "org_opencontainers_image_version": "v1.7.11",
          "traefik": "false"
        }
      }
    },
    "container": {
      "name": "infrastructure_traefik.1.cdz042qgr260paryvqkgue8vn",
      "image": {
        "name": "traefik:1.7.11@sha256:d590b5ef1278809b8809025eba2bd67afc2fdfe1926d87e67fcada14deb38652"
      },
      "id": "8d932184e0af1dc988db75de02981fc39de388aa940fc2dc4f53ed5c2713e0b8",
      "labels": {
        "org_opencontainers_image_vendor": "Containous",
        "org_opencontainers_image_version": "v1.7.11",
        "traefik": "false",
        "logstash": "false",
        "com_docker_swarm_task_id": "cdz042qgr260paryvqkgue8vn",
        "com_docker_swarm_service_id": "pg1nc3phr2wd7wpkexo6pijb7",
        "com_docker_swarm_service_name": "infrastructure_traefik",
        "org_opencontainers_image_description": "A modern reverse-proxy",
        "org_opencontainers_image_title": "Traefik",
        "org_opencontainers_image_url": "https://traefik.io",
        "com_docker_swarm_task_name": "infrastructure_traefik.1.cdz042qgr260paryvqkgue8vn",
        "org_opencontainers_image_documentation": "https://docs.traefik.io",
        "com_docker_stack_namespace": "infrastructure",
        "proxy": "true",
        "com_docker_swarm_node_id": "vjpbn32i27e2l3xhw098iug65",
        "com_docker_swarm_task": ""
      }
    },
    "agent": {
      "version": "7.3.0",
      "type": "filebeat",
      "ephemeral_id": "bea05f1b-0ea3-48df-84e8-6843ffa92ac4",
      "hostname": "1516ac37019c",
      "id": "0f6a83c3-6d32-4229-91e3-5a706a9244f9"
    }
  },
  "fields": {
    "@timestamp": [
      "2019-08-05T07:32:55.857Z"
    ],
    "suricata.eve.timestamp": [
      "2019-08-05T07:32:55.857Z"
    ]
  },
  "sort": [
    1564990375857
  ]
}

Could you please share the debug logs of Filebeat (./filebeat -e -d "*")? Also, example events read by Filebeat?

2019-08-05T14:30:59.815+0200	DEBUG	[processors]	processing/processors.go:183	Publish event: {
  "@timestamp": "2019-08-05T12:01:32.678Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.3.0"
  },
  "input": {
    "type": "container"
  },
  "container": {
    "image": {
      "name": "traefik:1.7.11@sha256:d590b5ef1278809b8809025eba2bd67afc2fdfe1926d87e67fcada14deb38652"
    },
    "id": "8d932184e0af1dc988db75de02981fc39de388aa940fc2dc4f53ed5c2713e0b8",
    "name": "infrastructure_traefik.1.cdz042qgr260paryvqkgue8vn",
    "labels": {
      "com_docker_swarm_node_id": "vjpbn32i27e2l3xhw098iug65",
      "org_opencontainers_image_vendor": "Containous",
      "org_opencontainers_image_version": "v1.7.11",
      "org_opencontainers_image_title": "Traefik",
      "org_opencontainers_image_documentation": "https://docs.traefik.io",
      "traefik": "false",
      "logstash": "false",
      "com_docker_swarm_service_id": "pg1nc3phr2wd7wpkexo6pijb7",
      "org_opencontainers_image_description": "A modern reverse-proxy",
      "com_docker_swarm_task_name": "infrastructure_traefik.1.cdz042qgr260paryvqkgue8vn",
      "org_opencontainers_image_url": "https://traefik.io",
      "com_docker_stack_namespace": "infrastructure",
      "com_docker_swarm_task_id": "cdz042qgr260paryvqkgue8vn",
      "com_docker_swarm_service_name": "infrastructure_traefik",
      "proxy": "true",
      "com_docker_swarm_task": ""
    }
  },
  "logstash": {
    "downstream_Date": "Mon, 05 Aug 2019 12:01:32 GMT",
    "level": "info",
    "request_Soapaction": "\"http://www.teta.com/SOA/PDS/PropertyStructureService/v_2_3/getStructure\"",
    "origin_Date": "Mon, 05 Aug 2019 12:01:32 GMT",
    "origin_Vary": "Accept-Encoding",
    "request_Te": "trailers, deflate, gzip, compress",
    "DownstreamStatusLine": "200 OK",
    "GzipRatio": 17.14320585842148,
    "OriginStatus": 200,
    "origin_Accept": "text/xml",
    "ClientPort": "31620",
    "OriginDuration": 15003749,
    "Duration": 15224019,
    "RequestHost": "propertystructure-mt3.mitus-test.teta.net",
    "StartUTC": "2019-08-05T12:01:32.661969358Z",
    "downstream_Content-Type": "text/xml;charset=UTF-8",
    "request_Content-Length": "3347",
    "request_User-Agent": "Oracle HTTPClient Version 10h",
    "RequestLine": "POST /services HTTP/1.1",
    "downstream_Vary": "Accept-Encoding",
    "request_Connection": "TE",
    "ClientHost": "10.49.103.66",
    "RequestContentSize": 3347,
    "downstream_Accept": "text/xml",
    "origin_Soapaction": "\"\"",
    "request_Ecid-Context": "1.6eac2d01dd7415fe:54e8aff5:16af7f0155f:-8000-00000000151ac6bb;k_vE0ZJOoOTLkKPOoLRKlSODoITT_G",
    "RetryAttempts": 0,
    "ClientUsername": "-",
    "RequestProtocol": "HTTP/1.1",
    "StartLocal": "2019-08-05T14:01:32.661969358+02:00",
    "origin_Content-Type": "text/xml;charset=UTF-8",
    "BackendAddr": "10.0.21.17:8080",
    "BackendName": "backend-propertystructure-mt3",
    "RequestMethod": "POST",
    "BackendURL": {
      "Host": "10.0.21.17:8080",
      "Path": "",
      "RawPath": "",
      "RawQuery": "",
      "Scheme": "http",
      "User": null,
      "Fragment": "",
      "Opaque": "",
      "ForceQuery": false
    },
    "DownstreamStatus": 200,
    "RequestPort": "-",
    "downstream_Content-Encoding": "gzip",
    "FrontendName": "Host-propertystructure-mt3-mitus-test-teta-net-38",
    "RequestPath": "/services",
    "ClientAddr": "10.49.103.66:31620",
    "OriginContentSize": 21069,
    "origin_Content-Encoding": "gzip",
    "time": "2019-08-05T14:01:32+02:00",
    "OriginStatusLine": "200 OK",
    "RequestAddr": "propertystructure-mt3.mitus-test.teta.net",
    "request_Accept-Encoding": "gzip, x-gzip, compress, x-compress",
    "RequestCount": 8142,
    "msg": "",
    "request_Content-Type": "text/xml; charset=UTF-8",
    "DownstreamContentSize": 1229,
    "Overhead": 220270,
    "downstream_Soapaction": "\"\""
  },
  "host": {
    "name": "5aa362359510"
  },
  "agent": {
    "ephemeral_id": "1faab82b-86ea-408a-bb38-ac9c0b2c038a",
    "hostname": "5aa362359510",
    "id": "7318c47d-bd68-439a-bed0-02623e5ee7e0",
    "version": "7.3.0",
    "type": "filebeat"
  },
  "log": {
    "offset": 4546292,
    "file": {
      "path": "/var/lib/docker/containers/8d932184e0af1dc988db75de02981fc39de388aa940fc2dc4f53ed5c2713e0b8/8d932184e0af1dc988db75de02981fc39de388aa940fc2dc4f53ed5c2713e0b8-json.log"
    }
  },
  "stream": "stdout",
  "ecs": {
    "version": "1.0.1"
  },
  "docker": {
    "container": {
      "labels": {
        "proxy": "true",
        "org_opencontainers_image_version": "v1.7.11",
        "org_opencontainers_image_documentation": "https://docs.traefik.io",
        "org_opencontainers_image_description": "A modern reverse-proxy",
        "traefik": "false",
        "com_docker_stack_namespace": "infrastructure",
        "logstash": "false",
        "org_opencontainers_image_title": "Traefik",
        "com_docker_swarm_task_id": "cdz042qgr260paryvqkgue8vn",
        "com_docker_swarm_task_name": "infrastructure_traefik.1.cdz042qgr260paryvqkgue8vn",
        "org_opencontainers_image_vendor": "Containous",
        "com_docker_swarm_node_id": "vjpbn32i27e2l3xhw098iug65",
        "com_docker_swarm_service_name": "infrastructure_traefik",
        "com_docker_swarm_service_id": "pg1nc3phr2wd7wpkexo6pijb7",
        "com_docker_swarm_task": "",
        "org_opencontainers_image_url": "https://traefik.io"
      }
    }
  }
}

Aslo more logs I have paste on pastebin: 2019-08-05T14:30:59.813+0200 DEBUG [processors] processing/processors.go:183 - Pastebin.com

Is ILM enabled in ES? If yes, Filebeat automatically uses it if it is. If you set setup.ilm.enabled to false, you can turn it off in FB and let indices option do its thing.

I have set ILM to false but still have only one index. My full filebeat.yml :

logging:
 level: error
 metrics:
  enabled: false


filebeat.modules:
  - module: mongodb
  - module: kafka
  - module: traefik

filebeat.autodiscover:
  providers:
    - type: docker
      labels.dedot: true
      templates:
        - condition:
            equals:
              docker.container.labels.jsonLog: 'true'
          config:
            - type: container
              format: docker
              paths:
                - "/var/lib/docker/containers/${data.docker.container.id}/*.log"
              exclude_lines: [".*eureka endpoints.*", "DEBUG"]
              processors:
                - decode_json_fields:
                    when.equals:
                      docker.container.labels.jsonLog: 'true'
                    fields: ["message"]
                    target: "logstash"
                    overwrite_keys: true
                - add_docker_metadata: ~
                - script:
                    lang: javascript
                    id: shorten_message
                    source: >
                      function process(event) {
                        var message = event.Get("logstash.message");
                        if (message != null && message.length > 900000) {
                          event.Put("logstash.message", message.substring(0, 900000));
                        }
                      }
                - drop_fields:
                    fields: ["message"]
        - condition:
            equals:
              docker.container.labels.proxy: 'true'
          config:
            - type: container
              format: docker
              paths:
                - "/var/lib/docker/containers/${data.docker.container.id}/*.log"
              processors:
                - decode_json_fields:
                    when.equals:
                      docker.container.labels.proxy: 'true'
                    fields: ["message"]
                    target: "logstash"
                    overwrite_keys: true
                - add_docker_metadata: ~
                - drop_fields:
                    fields: ["message"]

output.elasticsearch:
  hosts: ["elasticsearch-master:9200", "elasticsearch-slave:9200"]
  pipelines:
    - pipeline: "logstash-json-timestamp"
      when.equals:
        docker.container.labels.jsonLog: 'true'
  indices:
    - index: "filebeat-%{[beat.version]}-%{[docker.container.labels.com.docker.swarm.service.name]}-%{+yyyy.MM.dd}"
      when:
        or:
          - equals:
              docker.container.labels.jsonLog: 'true'
          - equals:
              container.labels.jsonLog: 'true'
          - contains:
              docker.container.labels.jsonLog: 'tru'
          - contains:
              container.labels.jsonLog: 'tru'
          - has_fields: ['docker.container.labels.jsonLog']
    - index: "traefik-%{[beat.version]}-%{[docker.container.labels.com.docker.swarm.service.name]}-%{+yyyy.MM.dd}"
      when:
        or:
          - equals:
              docker.container.labels.proxy: 'true'
          - equals:
              container.labels.proxy: 'true'
          - equals:
              docker.container.labels.proxy: 'tru'
          - equals:
              container.labels.proxy: 'tru'
          - has_fields: ['docker.container.labels.proxy']


setup:
  ilm:
    enabled: false

I am sorry if I was not clear. You need to run setup again, so the changes are applied in ES.

I have created my whole cluster (because I have contenerized elk, kibana and filebeat) from begining with also set:

setup:
  ilm:
    enabled: false

but still have only one index filebeat-*

Any help ?

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.