Wrong source name


#1

HI there,

I have some troubles with the source field on logstash that is incorrect for som logs. Here is an example (taken from Kibana):

message: 2017-05-05 14:12:48,395 [ INFO] [ID=XXXXXX] [User=] [org.my.class] Successfully loaded custom xml mappings from URL: [jar:file:/opt/[...]-0.0.4-SNAPSHOT.jar!/BOOT-INF/lib/[...]-0.0.4-SNAPSHOT.jar!/[...]]
source: config.log

after a check, this log comes from interfaces.log

All the stack (filebeat, logstash, elasticsearch and kibana) are up to date.

Here is the (anonymized) config of filebeat

  prospectors:
    -
      paths:
        - /var/log/mail.log
      document_type: postfix
    -
      paths:
        - /var/log/wildfly/server.log
      document_type: wildfly
      fields_under_root: true
      force_close_files: true
      multiline:
        pattern: '^\<'
        negate: true
        match: after
    -
      paths:
        - /opt/app1/logs/backend-app.log
        - /opt/app1/logs/output.log
        - /opt/app1/logs/interface.log
        - /opt/app1/logs/config.log
      document_type: app1
      fields_under_root: true
      force_close_files: true
      multiline:
        pattern: '^\<'
        negate: true
        match: after
    -
      paths:
        - /opt/app2/log/batch.log
        - /opt/app2/log/error.log
        - /opt/app2/log/batch/**/*.log
      document_type: app2
      fields_under_root: true
      force_close_files: true
      multiline:
        pattern: '^\<'
        negate: true
        match: after

output:
  ### logstash as output
  logstash:
    enabled: true
    hosts: [ "elasticsearch:5000" ]
    timeout: 15
    ssl:
      # List of root certificates for HTTPS server verifications
      certificate_authorities: ["/etc/ssl/certs/logstash-.crt"]
      certificate: /etc/ssl/certs/logstash.crt
      key: /etc/pki/tls/private/logstash.key
  file:
    path: /var/log/filebeat
    filename: event
    rotate_every_kb: 10000
    number_of_files: 7

logging:
  to_files: true
  files:
    path: /var/log/filebeat
    name: filebeat
    rotateeverybytes: 104857600
    keepfiles: 3
  level: info

and here is logstash input config file:

  input {
  beats {
    port => 5000
    ssl => true
    ssl_certificate => "/etc/ssl/certs/logstash.crt"
    ssl_key => "/etc/pki/tls/private/logstash.key"
    codec => multiline {
      pattern => "^%{TIMESTAMP_ISO8601}"
      negate => true
      what => "previous"
    }

  }
}
filter {
  if [source] =~ "\/opt\/app1\/logs\/(config|backend-app|outputmgt|interface)\.log" {
    grok {
      match => ["message", "%{TIMESTAMP_ISO8601:log.timestamp} \[[ ]?%{LOGLEVEL:log.level}\] \[CorrelationID=(%{NOTSPACE:log.correlationId})?\] \[User=(%{NOTSPACE:log.username})?\] \[%{JAVACLASS:log.logger}\] %{GREEDYDATA:log.message}"]
    }
    date {
      match => ["log.date", "YYYY-MM-dd HH:mm:ss,SSS"]
      target => "log.date"
    }
  }
}

and, finally, logstash output:

filter {
  uuid {
    target => "@uuid"
    overwrite => true
  }
  fingerprint {
    source => ["message"]
    target => "fingerprint"
    key => "XXXXXX"
    method => "SHA1"
    concatenate_sources => true
  }

}

output {
        elasticsearch {
                hosts => ["localhost:9200"]
                sniffing => true
                manage_template => false
        #       document_id => "%{fingerprint}"
        }
}

I don't really know what is the problem here. Thanks in advance for your help!


(system) #2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.