Configure filebeat to send only required fields

Elastic Stack Beats
1

Hi

My filebeat send the following data to ES. I don't need all the fields Is there a way I can configure filebeat to send only required fields.

Supporse I only need few fileds from the below data to be sent to ES.

{
  "_index": "filebeat-2019.05.15-000001",
  "_type": "_doc",
  "_id": "Ezl_wmoB3maiDp4bI8vP",
  "_version": 1,
  "_score": null,
  "_source": {
    "agent": {
      "hostname": "hostname.com",
      "id": "e0b20292-8514-4a64-8890-418e1e80c7dc",
      "type": "filebeat",
      "ephemeral_id": "ee293ff1-e8fa-423a-8f3e-0f40e7a902fb",
      "version": "7.0.1"
    },
    "log": {
      "file": {
        "path": "/usr/share/tomcat/logs/localhost_access_log.2019-05-16.txt"
      },
      "offset": 11473
    },
    "source": {
      "address": "10.49.102.104",
      "ip": "10.49.102.104"
    },
    "fileset": {
      "name": "access"
    },
    "url": {
      "original": "/portal/home/"
    },
    "input": {
      "type": "log"
    },
    "apache": {
      "access": {}
    },
    "@timestamp": "2019-05-16T21:14:25.000Z",
    "ecs": {
      "version": "1.0.0"
    },
    "service": {
      "type": "apache"
    },
    "host": {
      "hostname": "hostname.com",
      "os": {
        "kernel": "3.10.0-862.14.4.el7.x86_64",
        "codename": "Maipo",
        "name": "Red Hat Enterprise Linux Server",
        "family": "redhat",
        "version": "7.5 (Maipo)",
        "platform": "rhel"
      },
      "containerized": true,
      "name": "hostname.com",
      "id": "0709cd0acc86463da7d518c28f145a66",
      "architecture": "x86_64"
    },
    "http": {
      "request": {
        "method": "HEAD"
      },
      "response": {
        "status_code": 200
      },
      "version": "1.1"
    },
    "fields": {
      "version": "10.6"
    },
    "event": {
      "created": "2019-05-16T21:14:32.293Z",
      "module": "apache",
      "dataset": "apache.access"
    },
    "user": {
      "name": "-"
    }
  },
  "fields": {
    "suricata.eve.timestamp": [
      "2019-05-16T21:14:25.000Z"
    ],
    "@timestamp": [
      "2019-05-16T21:14:25.000Z"
    ],
    "event.created": [
      "2019-05-16T21:14:32.293Z"
    ]
  },
  "sort": [
    1558041265000
  ]
}
2

For this you can use the Include fields processor:

processors:
 - include_fields:
     fields: ["log.file", "source.address", ...]

This will result in documents with only the included fields them.

See the docs for further details like conditions and also have a look at the complementary Drop fields processor.

3

Thanks for the quick reply. I am using IIs module for windows and apache module for linux server logs.

So when using the modules we have iis.yml and apache.yml which have fields defined in it. Can I still used processors to remove unwanted fields.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.