Sample apache filter doesnt works 7.1.x

gyurielf · June 24, 2019, 2:29pm

Hello Guyz!

I need some help..

I try to use ELK stack(lastest 7.1.x) for log centralization... but something wrong with the apache log filter conf example.

In the dashboard, all data has appear but doesnt parsed.
Anyone can help me please ?

input {
  beats {
port => 5044
host => "0.0.0.0"
  }
}
filter {
  if [fileset][module] == "apache2" {
if [fileset][name] == "access" {
  grok {
    match => { "message" => ["%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} %{NUMBER:[apache2][access][body_sent][bytes]}( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?",
      "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \\[%{HTTPDATE:[apache2][access][time]}\\] \"-\" %{NUMBER:[apache2][access][response_code]} -" ] }
    remove_field => "message"
  }
  mutate {
    add_field => { "read_timestamp" => "%{@timestamp}" }
  }
  date {
    match => [ "[apache2][access][time]", "dd/MMM/YYYY:H:m:s Z" ]
    remove_field => "[apache2][access][time]"
  }
  useragent {
    source => "[apache2][access][agent]"
    target => "[apache2][access][user_agent]"
    remove_field => "[apache2][access][agent]"
  }
  geoip {
    source => "[apache2][access][remote_ip]"
    target => "[apache2][access][geoip]"
  }
}
else if [fileset][name] == "error" {
  grok {
    match => { "message" => ["\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{LOGLEVEL:[apache2][error][level]}\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message]}",
      "\[%{APACHE_TIME:[apache2][error][timestamp]}\] \[%{DATA:[apache2][error][module]}:%{LOGLEVEL:[apache2][error][level]}\] \[pid %{NUMBER:[apache2][error][pid]}(:tid %{NUMBER:[apache2][error][tid]})?\]( \[client %{IPORHOST:[apache2][error][client]}\])? %{GREEDYDATA:[apache2][error][message1]}" ] }
    pattern_definitions => {
      "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
    }
    remove_field => "message"
  }
  mutate {
    rename => { "[apache2][error][message1]" => "[apache2][error][message]" }
  }
  date {
    match => [ "[apache2][error][timestamp]", "EEE MMM dd H:m:s YYYY", "EEE MMM dd H:m:s.SSSSSS YYYY" ]
    remove_field => "[apache2][error][timestamp]"
  }
}
  }
}
output {
  elasticsearch {
hosts => ["//localhost:9200"]
	manage_template => false
index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
  }
}

Badger · June 24, 2019, 2:37pm

If you select one of those events in Discover and go to the JSON tab, what does the event look like? Please post text, not a picture of text.

norgro2601 · June 24, 2019, 3:05pm

You're using [fileset][module] == "apache2" in the filter, but in version 7 it is [event][module] == "apache2". This works in my configuration.

gyurielf · June 25, 2019, 8:27am

Thank you! I'll try right now, and I'll tell you what's the result.

gyurielf · June 25, 2019, 8:40am

{
  "_index": "filebeat-7.1.1-2019.06.25",
  "_type": "_doc",
  "_id": "YEfIjWsB5j_G9LEo83b0",
  "_version": 1,
  "_score": null,
  "_source": {
    "input": {
      "type": "log"
    },
    "ecs": {
      "version": "1.0.0"
    },
    "log": {
      "offset": 1285883092,
      "file": {
        "path": "/var/log/httpd/access_log"
      }
    },
    "fileset": {
      "name": "access"
    },
    "@version": "1",
    "service": {
      "type": "apache"
    },
    "message": "89.132.119.1 - kecske [25/Jun/2019:10:35:43 +0200] \"POST /api/index_management/indices/refresh HTTP/1.1\" 200 - \"https://kibana.xyz.com/app/kibana\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0\"",
    "read_timestamp": "2019-06-25T08:37:58.236Z",
    "cloud": {
      "instance": {
        "id": "147324943"
      },
      "provider": "digitalocean",
      "region": "fra1"
    },
    "host": {
      "containerized": false,
      "hostname": "hk-elks",
      "architecture": "x86_64",
      "os": {
        "platform": "centos",
        "kernel": "3.10.0-957.21.3.el7.x86_64",
        "version": "7 (Core)",
        "codename": "Core",
        "family": "redhat",
        "name": "CentOS Linux"
      },
      "name": "hk-elks",
      "id": "387b06bbed1dc5ea41c228855d0225dd"
    },
    "event": {
      "dataset": "apache.access",
      "module": "apache"
    },
    "tags": [
      "beats_input_codec_plain_applied",
      "_grokparsefailure",
      "_geoip_lookup_failure"
    ],
    "agent": {
      "ephemeral_id": "55f38ce1-d5a9-4baf-9eaa-91c334c8da9b",
      "hostname": "hk-elks",
      "version": "7.1.1",
      "type": "filebeat",
      "id": "e4ec413a-0f36-4b48-a6b0-83ab274d7c3b"
    },
    "@timestamp": "2019-06-25T08:37:58.236Z"
  },
  "fields": {
    "suricata.eve.timestamp": [
      "2019-06-25T08:37:58.236Z"
    ],
    "@timestamp": [
      "2019-06-25T08:37:58.236Z"
    ]
  },
  "sort": [
    1561451878236
  ]
}

gyurielf · June 25, 2019, 8:47am

I tried it but after the restart, nothing changed.

{
  "_index": "filebeat-7.1.1-2019.06.25",
  "_type": "_doc",
  "_id": "OkjgjWsB5j_G9LEoXW6H",
  "_version": 1,
  "_score": null,
  "_source": {
    "cloud": {
      "region": "fra1",
      "provider": "digitalocean",
      "instance": {
        "id": "147324943"
      }
    },
    "agent": {
      "type": "filebeat",
      "ephemeral_id": "55f38ce1-d5a9-4baf-9eaa-91c334c8da9b",
      "id": "e4ec413a-0f36-4b48-a6b0-83ab274d7c3b",
      "version": "7.1.1",
      "hostname": "hk-elks"
    },
    "@version": "1",
    "event": {
      "dataset": "apache.error",
      "module": "apache"
    },
    "@timestamp": "2019-06-25T09:03:31.739Z",
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "log": {
      "file": {
        "path": "/var/log/httpd/error_log"
      },
      "offset": 1216672641
    },
    "host": {
      "id": "387b06bbed1dc5ea41c228855d0225dd",
      "name": "hk-elks",
      "architecture": "x86_64",
      "os": {
        "name": "CentOS Linux",
        "codename": "Core",
        "kernel": "3.10.0-957.21.3.el7.x86_64",
        "family": "redhat",
        "version": "7 (Core)",
        "platform": "centos"
      },
      "containerized": false,
      "hostname": "hk-elks"
    },
    "input": {
      "type": "log"
    },
    "fileset": {
      "name": "error"
    },
    "service": {
      "type": "apache"
    },
    "message": "[Tue Jun 25 11:02:40.555500 2019] [authz_core:error] [pid 2555:tid 140461301806848] [client 185.62.190.78:60122] AH01630: client denied by server configuration: /etc/httpd/htdocs",
    "ecs": {
      "version": "1.0.0"
    }
  },
  "fields": {
    "suricata.eve.timestamp": [
      "2019-06-25T09:03:31.739Z"
    ],
    "@timestamp": [
      "2019-06-25T09:03:31.739Z"
    ]
  },
  "sort": [
    1561453411739
  ]
}

norgro2601 · June 25, 2019, 10:12am

But now it's executing the if-clause, but as your grok isn't correct, it will not create the fields. I've put the message in the Grok debugger in Kibana and with some changes to the grok it worked.

This is the message:
89.132.119.1 - kecske [25/Jun/2019:10:35:43 +0200] "POST /api/index_management/indices/refresh HTTP/1.1" 200 - "https://kibana.xyz.com/app/kibana" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0"

This is the grok I've usedin the debugger:
%{IPORHOST:[apache2][access][remote_ip]} - %{WORD:[apache2][access][user_name]} [%{HTTPDATE:[apache2][access][time]}] \"%{WORD:[apache2][access][method]} %{NOTSPACE:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} (?:%{NUMBER:[apache2][access][body_sent][bytes]}|-) \"%{DATA:[apache2][access][referrer]}\" \"%{DATA:[apache2][access][agent]}\"

This is the result:
{
"[apache2][access][referrer]": "https://kibana.xyz.com/app/kibana",
"[apache2][access][user_name]": "kecske",
"[apache2][access][url]": "/api/index_management/indices/refresh",
"[apache2][access][time]": "25/Jun/2019:10:35:43 +0200",
"[apache2][access][response_code]": "200",
"[apache2][access][remote_ip]": "89.132.119.1",
"[apache2][access][agent]": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
"[apache2][access][http_version]": "1.1",
"[apache2][access][method]": "POST"
}

I think you have to verify the grok.

system · July 23, 2019, 10:13am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.