Logstash not parsing logs into Elasticsearch

[calanon (Chris)]

I have setup my paths to the logs files as stated in the documentation, the problem is when, for example; I turn on the nginx module the logs seems to be parsed and sent to elasticsearch. But when I turn off the nginx module it stops parsing the normal logs in my /var/log/*.log path.

I am on the latest version of filebeats / Logstash / Elasticsearch and Kibana.

Here is a snippet of my config:

logstash.yml

    node.name: erp-mon-1
    path.data: /var/lib/logstash
    pipeline.workers: 2
    pipeline.batch.size: 125
    pipeline.batch.delay: 5
    pipeline.unsafe_shutdown: false
    path.config: /etc/logstash/conf.d/*.conf
    config.reload.automatic: true
    path.logs: /var/log/logstash

01-beats-input.conf
    input {
       beats {
           port => 2561
           ssl => true
           ssl_certificate => "/etc/ssl/certs/erp-mon-1.crt"
           ssl_key => "/etc/ssl/private/erp-mon-1.p8"
       }
    }

15-beats-filter.conf

filter {
  grok {
    patterns_dir => ["/etc/logstash/patterns"]
    match => { "message" => "\[%{DATESTAMP_ADMIN_LOG:log_timestamp}\] %{LOGLEVEL:loglevel} %{WORD:thread_name} %{NUMBER:application_runtime_ms}ms \(*%{WORD:user_name}\)* - %{GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_ADMIN_LOG:log_timestamp}\] %{LOGLEVEL:loglevel} %{NUMBER:pid} %{NUMBER:application_runtime_ms}ms \- %{NOTSPACE:command_type} %{NUMBER:command_id} for %{NOTSPACE:subject_type} %{NUMBER:subject_id} - %{GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_ADMIN_LOG:log_timestamp}\] %{NUMBER:pid} %{LOGLEVEL:loglevel} %{NUMBER:application_runtime_ms}ms \- %{GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_ADMIN_LOG:log_timestamp}\] %{LOGLEVEL:loglevel} %{NUMBER:pid} %{NUMBER:application_runtime_ms}ms \- %{GREEDYDATA:message}" }
    #match => { "message" => "%{COMBINEDAPACHELOG} %{NOTSPACE:Host_header} %{NOTSPACE} %{NOTSPACE:port} %{NOTSPACE:ssl_protcol} %{NOTSPACE:ssl_cipher} %{NOTSPACE:ssl_tls_sni}" }
    match => { "message" => "%{COMBINEDAPACHELOG} %{NOTSPACE:Host_header} %{NOTSPACE} %{NOTSPACE:port} %{NOTSPACE:request_time:int} %{NOTSPACE:ssl_protcol} %{NOTSPACE:ssl_cipher} %{NOTSPACE:ssl_tls_sni}" }
    #match => { "message" => "%{COMBINEDAPACHELOG} %{NOTSPACE:Host_header} %{NOTSPACE} %{NOTSPACE:port}" }
    match => { "message" => "%{COMBINEDAPACHELOG} %{NOTSPACE:Host_header} %{NOTSPACE} %{NOTSPACE:port} %{NOTSPACE:request_time:int}" }
    match => { "message" => "%{HTTPD_ERRORLOG}" }
    match => { "message" => "\[%{DATESTAMP_PHPERROR:timestamp}\] %{DATA:level}[\.\:][\ ]+%{GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_PHP_FPMERROR:timestamp}\] %{LOGLEVEL:loglevel}[\:] \[pool %{DATA:pool}\][\ ]+%{GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_PHP_FPMERROR:timestamp}\] %{LOGLEVEL:loglevel}[\:\.]?[\ ]+%{GREEDYDATA:message}" }
    match => { "message" => "%{NOTSPACE:remote_ip} %{NOTSPACE:remote_user}[\ ]+%{HTTPDATE:timestamp} \"%{WORD:verb} %{NOTSPACE:request}\" %{NUMBER:response} %{NOTSPACE:script_filename} %{NOTSPACE:request_time:int} %{NUMBER:memory_kb:int} %{NUMBER:cpu_usage_percent:float}\%" }
#    match => { "message" => "%{NOTSPACE:remote_ip} %{NOTSPACE:remote_user} %{HTTPDATE:timestamp} {GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_ADMIN_LOG:log_timestamp}\] %{LOGLEVEL:loglevel}[\:]? %{GREEDYDATA:message}" }
    match => { "message" => "\[%{DATESTAMP_ADMIN_LOG:log_timestamp}\]\[%{IP:remote_ip}\] %{LOGLEVEL:loglevel}[\:]? %{GREEDYDATA:message}" }

    # Common log format
    match => { "message" => "\[%{TIMESTAMP_ISO8601:log_timestamp}\] %{IP:remote_ip} %{LOGLEVEL:loglevel} %{GREEDYDATA:message} %{CONTEXT_PATTERN:context} %{EXTRA_PATTERN:extra}" }
    match => { "message" => "\[%{TIMESTAMP_ISO8601:log_timestamp}\] %{IP:remote_ip} %{LOGLEVEL:loglevel} %{GREEDYDATA:message}" }
  
    match => { "message" => "\[%{TIMESTAMP_ISO8601:log_timestamp}\] %{NUMBER:pid} %{LOGLEVEL:loglevel} %{GREEDYDATA:message} %{CONTEXT_PATTERN:context} %{EXTRA_PATTERN:extra}" }
    match => { "message" => "\[%{TIMESTAMP_ISO8601:log_timestamp}\] %{NUMBER:pid} %{LOGLEVEL:loglevel} %{GREEDYDATA:message}" }

    overwrite => [ "message" ]
}

  mutate {
    gsub => ["timestamp","\.\d\d\d\d\d\d "," "]
  }

  mutate {
    convert => {"bytes" => "integer" }
  }

  date {
    match => [ "log_timestamp", "yyyyMMdd HHmmss", "yyyy-MM-dd HH:mm:ss" ]
    target => "@timestamp"
    timezone => "Europe/Berlin"
    add_field => { "debug" => "timestampMatched" }
  }

  date {
    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "EEE MMM dd HH:mm:ss yyyy", "dd-MMM-yyyy HH:mm:ss ZZZ", "dd-MMM-yyyy HH:mm:ss" ]
    target => "@timestamp"
    timezone => "Europe/Berlin"
    add_field => { "debug" => "timestampMatched" }
  }

}

35-elastic-output.conf

output {
  if [source] == "/var/log/apache2/ssl_access.log" or [source] == "/var/log/apache2/access.log" or [source] == "/var/log/apache2/ssl_error.log" or [source] == "/var/log/apache2/error.log" {
    elasticsearch {
      hosts => "localhost:9200"
      index => "var_log_apache2-%{+YYYY.MM.dd}"
    }
  } 
  else if [source] == "/var/log/nginx/access.log" or [source] == "/var/log/nginx/error.log" {
    elasticsearch {
      hosts => "localhost:9200"
      index => "var_log_nginx-%{+YYYY.MM.dd}"
    }
  } 
  else {
    elasticsearch {
      hosts => "localhost:9200"
      index => "var_log_common-%{+YYYY.MM.dd}"
    }
  }
}

Any help would be great

[Badger]

@calanon

when I turn off the nginx module it stops parsing the normal logs

I have no idea what you mean by that, so I am unable to offer any assistance. I only mention that because it might apply to others too.