Filebeat -> Logstash match metadata

Elastic Stack Logstash
1

Hi,

I've noticed filebeat sends some additional metadata for each generated event such as: log.file.path and process.name.

I can access to log.file.path ([log][file][path]) but not to process.name ([process][name]). ??

 {
  "_index": "filebeat-7.4.0-2019.10.29-000006",
  "_type": "_doc",
  "_id": "LUdQN24BFAd3xpeubZvm",
  "_version": 1,
  "_score": null,
  "_source": {
    "agent": {
      "hostname": "http2",
      "id": "9ceb38e6-9835-4fd8-9eb0-d741449fbd6b",
      "type": "filebeat",
      "ephemeral_id": "6edd52a2-a706-4a89-87b1-8a0fb6ce6c12",
      "version": "7.4.0"
    },
    "process": {
      "name": "(squid-1)"
    },
    "log": {
      "file": {
        "path": "/var/log/messages"
      },
      "offset": 1333266097
    },
    "fileset": {
      "name": "syslog"
    },
    "message": "11.12.28.183 - - [04/Nov/2019:17:47:02 +0100] \"CONNECT autodi.fr:443 HTTP/1.1\" 407 3728 101 \"-\" \"-\" TCP_DENIED:HIER_NONE",
    "input": {
      "type": "log"
    },
    "@timestamp": "2019-11-04T17:47:02.000+01:00",
    "system": {
      "syslog": {}
    },
    "ecs": {
      "version": "1.1.0"
    },
    "service": {
      "type": "system"
    },
    "host": {
      "hostname": "http1323",
      "os": {
        "kernel": "3",
        "codename": "Core",
        "name": "CentOS Linux",
        "family": "redhat",
        "version": "7 (Core)",
        "platform": "centos"
      },
      "containerized": false,
      "name": "http2",
      "id": "a8a5cf55a7ad46e8a2e6ab26a32e8571",
      "architecture": "x86_64"
    },
    "event": {
      "timezone": "+01:00",
      "module": "system",
      "dataset": "system.syslog"
    }
  },
  "fields": {
    "suricata.eve.timestamp": [
      "2019-11-04T16:47:02.000Z"
    ],
    "@timestamp": [
      "2019-11-04T16:47:02.000Z"
    ]
  },
  "highlight": {
    "process.name": [
      "@kibana-highlighted-field@(squid-1)@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1572886022000
  ]
}

Thank you,

2

How are you trying to use [process][name] in your logstash configuration?

3

Hello @Badger,

I want to filter the event under certain conditions :

if [process][name] =~ /\([a-z]+-1\)/{
dissect {
      mapping => {
        "message" => '%{a} %{b} %{c} [%{mydate}] "%{d} %{e}
%{f}" %{g} %{h} %{i} "%{j}" "%{k}" %{l}:%{m}'
      }
    }
date {
        match => ["mydate", "dd/MMM/YYYY:HH:mm:ss Z"]
        target => "date"
      }
        mutate {                    remove_field => [ "mydate" ]
        }
    }

Thank you

4

When I run

input { generator { count => 1 lines => [ '11.12.28.183 - - [04/Nov/2019:17:47:02 +0100] "CONNECT autodi.fr:443 HTTP/1.1" 407 3728 101 "-" "-" TCP_DENIED:HIER_NONE' ] } }
filter {
    mutate { add_field => { "[process][name]" => "(squid-1)" } }
    if [process][name] =~ /\([a-z]+-1\)/ {
        dissect { mapping => { "message" => '%{a} %{b} %{c} [%{mydate}] "%{d} %{e} %{f}" %{g} %{h} %{i} "%{j}" "%{k}" %{l}:%{m}' } }
        date { match => ["mydate", "dd/MMM/YYYY:HH:mm:ss Z"] target => "date" }
        mutate { remove_field => [ "mydate" ] }
    }
}
output { stdout { codec => rubydebug { metadata => false } } }

I get

   "message" => "11.12.28.183 - - [04/Nov/2019:17:47:02 +0100] \"CONNECT autodi.fr:443 HTTP/1.1\" 407 3728 101 \"-\" \"-\" TCP_DENIED:HIER_NONE",
         "e" => "autodi.fr:443",
         "i" => "101",
      "date" => 2019-11-04T16:47:02.000Z,
         "f" => "HTTP/1.1",
         "k" => "-",
   "process" => {
    "name" => "(squid-1)"
},

etc. Are you sure [process][name] exists at the point where you are testing it?

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.