Help needed with OpenLDAP Logstash filter


(pdna356) #1

I have the following logstash.conf file that I have taken from github

input {
  file {
      path => "/var/log/openldap.log"
      start_position => beginning
      type => "openldap_log_file"
  }
}
filter {
  grok {
    match => [ "message", "%{SYSLOGBASE} (?:(?:<= (?:b|m)db_%{DATA:index_error_filter_type}_candidates: \(%{WORD:index_error_attribute_name}\) not indexed)|(?:ppolicy_%{DATA:ppolicy_op}: %{DATA:ppolicy_data})|(?:connection_input: conn=%{INT:connection} deferring operation: %{DATA:deferring_op})|(?:connection_read\(%{INT:fd_number}\): no connection!)|(?:conn=%{INT:connection} (?:(?:fd=%{INT:fd_number} (?:(?:closed(?: \(connection lost\)|))|(?:ACCEPT from IP=%{IP:src_ip}\:%{INT:src_port} \(IP=%{IP:dst_ip}\:%{INT:dst_port}\))|(?:TLS established tls_ssf=%{INT:tls_ssf} ssf=%{INT:ssf})))|(?:op=%{INT:operation_number} (?:(?:(?:(?:SEARCH )|(?:))RESULT (?:tag=%{INT:tag}|oid=(?:%{DATA:oid}(?:))) err=%{INT:error_code}(?:(?: nentries=%{INT:nentries})|(?:)) text=(?:(?:%{DATA:error_text})|(?:)))|(?:%{WORD:operation_name}(?:(?: %{DATA:data})|(?:))))))))%{SPACE}$" ]
  }
  date {
    match => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
    target => "@timestamp"
  }
  if [operation_name] == "BIND" {
    grok {
      match => [ "data", "(?:(?:(?<bind_dn>anonymous))|(?:dn=\"%{DATA:bind_dn}\")) (?:(?:method=%{WORD:bind_method})|(?:mech=%{WORD:bind_mech} ssf=%{INT:bind_ssf}))%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "SRCH" {
    grok {
      match => [ "data", "(?:(?:base=\"%{DATA:search_base}\" scope=%{INT:search_scope} deref=%{INT:search_deref} filter=\"%{DATA:search_filter}\")|(?:attr=%{DATA:search_attr}))%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "MOD" {
    grok {
      match => [ "data", "(?:(?:dn=\"%{DATA:mod_dn}\")|(?:attr=%{DATA:mod_attr}))%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "MODRDN" {
    grok {
      match => [ "data", "dn=\"%{DATA:modrdn_dn}\"%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "ADD" {
    grok {
      match => [ "data", "dn=\"%{DATA:add_dn}\"%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "DEL" {
    grok {
      match => [ "data", "dn=\"%{DATA:del_dn}\"%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "CMP" {
    grok {
      match => [ "data", "dn=\"%{DATA:cmp_dn}\" attr=\"%{DATA:cmp_attr}\"%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [operation_name] == "EXT" {
    grok {
      match => [ "data", "oid=%{DATA:ext_oid}%{SPACE}$" ]
      remove_field => [ "data" ]
    }
  }
  if [ppolicy_op] == "bind" {
    grok {
      match => [ "ppolicy_data", "(?:(?:Entry %{DATA:ppolicy_bind_dn} has an expired password: %{INT:ppolicy_grace} grace logins)|(?:Setting warning for password expiry for %{DATA:ppolicy_bind_dn} = %{INT:ppolicy_expiration} seconds))%{SPACE}$" ]
      remove_field => [ "ppolicy_data" ]
    }
  }
}
output {
       stdout { codec => rubydebug }
}

I have logs like below

2017-10-19T13:36:02.354614-04:00 hostname slapd[7160]: conn=613017 op=1 BIND dn="cn=Groups,ou=Apps,dc=example,dc=com" method=128
2017-10-19T13:36:02.354268-04:00 hostname slapd[7160]: conn=613016 op=3 SRCH attr=dn
2017-10-09T10:54:12.467741-04:00 hostname slapd[1144]: conn=208653 op=15 MOD attr=memberOf
2017-10-12T09:30:50.252555-04:00 hostname slapd[11963]: conn=23724 op=1 ADD dn="uid=test,ou=etc,ou=example,dc=example,dc=com"
2017-10-09T17:55:13.194629-04:00 hostname slapd[11963]: conn=1000 op=46 DEL dn="cn=test,ou=es,ou=Groups,dc=example,dc=com"
2017-10-15T06:25:02.123083-04:00 hostname slapd[11963]: conn=47630 op=0 EXT oid=1.2.3.4.5.6.7434.48821

I get a grokparsefailure for all of them. I am not sure where the issue is.


(Magnus B├Ąck) #2

The first problem is SYSLOGBASE. It only matches a timestamp like "Oct 19 21:07:49".

I suggest you scrap the expression you have and build one from scratch. The one you have doesn't appear to match your logs and all and it's quite inefficient with the frequent occurrences of the DATA pattern.


(system) #3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.