Sample apache filter doesnt works 7.1.x

Elastic Stack Logstash
1

Hello Guyz!

I need some help.. :frowning:

I try to use ELK stack(lastest 7.1.x) for log centralization... but something wrong with the apache log filter conf example.

In the dashboard, all data has appear but doesnt parsed.
Anyone can help me please ?

Screenshot_2019-06-24%20%5BFilebeat%20Apache%5D%20Access%20and%20error%20logs%20ECS%20-%20Kibana
Screenshot_2019-06-24 [Filebeat Apache] Access and error logs ECS - Kibana.png1912×944 95.7 KB 

input {
  beats {
port => 5044
host => "0.0.0.0"
  }
}
filter {
  if [fileset][module] == "apache2" {
if [fileset][name] == "access" {
  grok {
    match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
      "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
    remove_field => "message"
  }
  mutate {
    add_field => { "read_timestamp" => "%{@timestamp}" }
  }
  date {
    match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
    remove_field => "[apache2][access][time]"
  }
  useragent {
    source => "[apache2][access][agent]"
    target => "[apache2][access][user_agent]"
    remove_field => "[apache2][access][agent]"
  }
  geoip {
    source => "[apache2][access][remote_ip]"
    target => "[apache2][access][geoip]"
  }
}
else if [fileset][name] == "error" {
  grok {
    match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
      "\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
    pattern_definitions => {
      "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
    }
    remove_field => "message"
  }
  mutate {
    rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
  }
  date {
    match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
    remove_field => "[apache2][error][timestamp]"
  }
}
  }
}
output {
  elasticsearch {
hosts => ["//localhost:9200"]
	manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
  }
}
2

If you select one of those events in Discover and go to the JSON tab, what does the event look like? Please post text, not a picture of text.

3

You're using [fileset][module] == "apache2" in the filter, but in version 7 it is [event][module] == "apache2". This works in my configuration.

4

Thank you! I'll try right now, and I'll tell you what's the result.

5 
{
  "_index": "filebeat-7.1.1-2019.06.25",
  "_type": "_doc",
  "_id": "YEfIjWsB5j_G9LEo83b0",
  "_version": 1,
  "_score": null,
  "_source": {
    "input": {
      "type": "log"
    },
    "ecs": {
      "version": "1.0.0"
    },
    "log": {
      "offset": 1285883092,
      "file": {
        "path": "/var/log/httpd/access_log"
      }
    },
    "fileset": {
      "name": "access"
    },
    "@version": "1",
    "service": {
      "type": "apache"
    },
    "message": "89.132.119.1 - kecske [25/Jun/2019:10:35:43 +0200] \"POST /api/index_management/indices/refresh HTTP/1.1\" 200 - \"https://kibana.xyz.com/app/kibana\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0\"",
    "read_timestamp": "2019-06-25T08:37:58.236Z",
    "cloud": {
      "instance": {
        "id": "147324943"
      },
      "provider": "digitalocean",
      "region": "fra1"
    },
    "host": {
      "containerized": false,
      "hostname": "hk-elks",
      "architecture": "x86_64",
      "os": {
        "platform": "centos",
        "kernel": "3.10.0-957.21.3.el7.x86_64",
        "version": "7 (Core)",
        "codename": "Core",
        "family": "redhat",
        "name": "CentOS Linux"
      },
      "name": "hk-elks",
      "id": "387b06bbed1dc5ea41c228855d0225dd"
    },
    "event": {
      "dataset": "apache.access",
      "module": "apache"
    },
    "tags": [
      "beats_input_codec_plain_applied",
      "_grokparsefailure",
      "_geoip_lookup_failure"
    ],
    "agent": {
      "ephemeral_id": "55f38ce1-d5a9-4baf-9eaa-91c334c8da9b",
      "hostname": "hk-elks",
      "version": "7.1.1",
      "type": "filebeat",
      "id": "e4ec413a-0f36-4b48-a6b0-83ab274d7c3b"
    },
    "@timestamp": "2019-06-25T08:37:58.236Z"
  },
  "fields": {
    "suricata.eve.timestamp": [
      "2019-06-25T08:37:58.236Z"
    ],
    "@timestamp": [
      "2019-06-25T08:37:58.236Z"
    ]
  },
  "sort": [
    1561451878236
  ]
}
6

I tried it but after the restart, nothing changed.

{
  "_index": "filebeat-7.1.1-2019.06.25",
  "_type": "_doc",
  "_id": "OkjgjWsB5j_G9LEoXW6H",
  "_version": 1,
  "_score": null,
  "_source": {
    "cloud": {
      "region": "fra1",
      "provider": "digitalocean",
      "instance": {
        "id": "147324943"
      }
    },
    "agent": {
      "type": "filebeat",
      "ephemeral_id": "55f38ce1-d5a9-4baf-9eaa-91c334c8da9b",
      "id": "e4ec413a-0f36-4b48-a6b0-83ab274d7c3b",
      "version": "7.1.1",
      "hostname": "hk-elks"
    },
    "@version": "1",
    "event": {
      "dataset": "apache.error",
      "module": "apache"
    },
    "@timestamp": "2019-06-25T09:03:31.739Z",
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "log": {
      "file": {
        "path": "/var/log/httpd/error_log"
      },
      "offset": 1216672641
    },
    "host": {
      "id": "387b06bbed1dc5ea41c228855d0225dd",
      "name": "hk-elks",
      "architecture": "x86_64",
      "os": {
        "name": "CentOS Linux",
        "codename": "Core",
        "kernel": "3.10.0-957.21.3.el7.x86_64",
        "family": "redhat",
        "version": "7 (Core)",
        "platform": "centos"
      },
      "containerized": false,
      "hostname": "hk-elks"
    },
    "input": {
      "type": "log"
    },
    "fileset": {
      "name": "error"
    },
    "service": {
      "type": "apache"
    },
    "message": "[Tue Jun 25 11:02:40.555500 2019] [authz_core:error] [pid 2555:tid 140461301806848] [client 185.62.190.78:60122] AH01630: client denied by server configuration: /etc/httpd/htdocs",
    "ecs": {
      "version": "1.0.0"
    }
  },
  "fields": {
    "suricata.eve.timestamp": [
      "2019-06-25T09:03:31.739Z"
    ],
    "@timestamp": [
      "2019-06-25T09:03:31.739Z"
    ]
  },
  "sort": [
    1561453411739
  ]
}
7

But now it's executing the if-clause, but as your grok isn't correct, it will not create the fields. I've put the message in the Grok debugger in Kibana and with some changes to the grok it worked.

This is the message:
89.132.119.1 - kecske [25/Jun/2019:10:35:43 +0200] "POST /api/index_management/indices/refresh HTTP/1.1" 200 - "https://kibana.xyz.com/app/kibana" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0"

This is the grok I've usedin the debugger:
%{IPORHOST:[apache2][access][remote_ip]} - %{WORD:[apache2][access][user_name]} [%{HTTPDATE:[apache2][access][time]}] \"%{WORD:[apache2][access][method]} %{NOTSPACE:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} (?:%{NUMBER:[apache2][access][body_sent][bytes]}|-) \"%{DATA:[apache2][access][referrer]}\" \"%{DATA:[apache2][access][agent]}\"

This is the result:
{
"[apache2][access][referrer]": "https://kibana.xyz.com/app/kibana",
"[apache2][access][user_name]": "kecske",
"[apache2][access][url]": "/api/index_management/indices/refresh",
"[apache2][access][time]": "25/Jun/2019:10:35:43 +0200",
"[apache2][access][response_code]": "200",
"[apache2][access][remote_ip]": "89.132.119.1",
"[apache2][access][agent]": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
"[apache2][access][http_version]": "1.1",
"[apache2][access][method]": "POST"
}

I think you have to verify the grok.

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.