Kibana shows incorrect syslog_timestamp version 6.5.4

Elastic Stack Logstash
1

I have Below logstash.conf file, i see the data is processing correctly but today i see very weird issues where indices for apic_logs are not showing the correct syslog_timestamp .

input {
  file {
    path => [ "/scratch/rsyslog/*/messages.log" ]
    start_position => beginning
    sincedb_path => "/dev/null"
    max_open_files => 64000
    type => "noi-syslog"
  }
  file {
    path => [ "/scratch/rsyslog_CISCO/*/network.log" ]
    start_position => beginning
    sincedb_path => "/dev/null"
    max_open_files => 64000
    type => "apic_logs"
  }
}

filter {
  if [type] == "noi-syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp } %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      #remove_field => [ "host", "path" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
 }
}
  if [type] == "apic_logs" {
    grok {
      match => { "message" => "%{CISCOTIMESTAMP:syslog_timestamp} %{CISCOTIMESTAMP} %{SYSLOGHOST:syslog_hostname} (?<prog>[\w._/%-]+) %{SYSLOG5424SD:fault_code}%{SYSLOG5424SD:fault_state}%{SYSLOG5424SD:crit_info}%{SYSLOG5424SD:log_severity}%{SYSLOG5424SD:log_info} %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      #remove_field => [ "host", "path" ]
    }
    syslog_pri { }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
  }
 }
}
output {
        if [type] == "noi-syslog" {
        elasticsearch {
                hosts => "noida-elk:9200"
                manage_template => false
                index => "noi-syslog-%{+YYYY.MM.dd}"
                document_type => "messages"
  }
 }
}

output {
        if [type] == "apic_logs" {
        elasticsearch {
                hosts => "noida-elk:9200"
                manage_template => false
                index => "apic_logs-%{+YYYY.MM.dd}"
                document_type => "messages"
  }
 }
}

Indices For the apic_logs:

$ curl -s -XGET http://127.0.0.1:9200/_cat/indices?v |  grep apic_logs
green  open   apic_logs-2019.02.14            no1Pc0fsQZuTqlbXiBPgVQ   5   1    4322391            0      1.9gb        992.7mb
green  open   apic_logs-2019.02.21            VRaK8BPyT26BzMPrkW4kkw   5   1    7425060            0      3.2gb          1.6gb
green  open   apic_logs-2019.01.13            a9R22M4hTei_hP5vDFofJg   5   1      15929            0      5.5mb          2.7mb
green  open   apic_logs-2019.03.07            ek3KGV1YQBi1Z3aTLB-SJA   5   1    4875726            0      2.2gb          1.1gb
green  open   apic_logs-2019.02.15            IXfTfEVKQxK5SFFbSkhoqA   5   1    3932260            0      1.7gb        902.9mb
green  open   apic_logs-2019.01.29            yp2NpE2-SQq0E0EeBSZUUQ   5   1      18427            0      6.5mb          3.2mb
green  open   apic_logs-2019.03.09            r8-MiHgoQN-LKs_Hw4Y6dg   5   1    2737714            0      1.2gb          624mb
green  open   apic_logs-2019.01.27            BDz-yYirT9-LhuW7UlkHhw   5   1      18646            0      6.6mb          3.3mb
green  open   apic_logs-2019.02.13            tvJ5FhR7S520yuVGJyw1GQ   5   1   10091998            0      4.5gb          2.2gb
green  open   apic_logs-2019.01.17            k_UieSYPTiGg-DZfvKMFZA   5   1      18414            0      6.6mb          3.3mb
green  open   apic_logs-2019.02.28            2WhQslGqRcWk6Y_otwfY7Q   5   1    5226033            0      2.3gb          1.1gb
green  open   apic_logs-2019.03.06            20LuKcQhTZa6QlowBEJWpA   5   1    5056087            0      2.3gb          1.1gb
green  open   apic_logs-2019.02.11            Q546DnhiTgCgIZ4rAdXbgA   5   1    5562247            0      1.8gb        967.2mb
green  open   apic_logs-2019.01.11            K43puyKrQVOPrfKmL6aAUA   5   1      17062            0      6.2mb          3.1mb
green  open   apic_logs-2019.03.04            FHB6TfXKTAy6tbi1F5IRJw   5   1    4281349            0      1.9gb        992.8mb
green  open   apic_logs-2019.01.26            pezb8a9ARQ-V5SOeZNwQhg   5   1      18471            0      6.4mb          3.2mb
green  open   apic_logs-2019.03.12            iWUX2qALSbev8KRHvX4Frw   5   1   13257835            0        7gb          3.5gb
green  open   apic_logs-2019.01.16            _H_YylVIRkqSi3lCFWR5wA   5   1      18294            0      6.7mb          3.3mb
green  open   apic_logs-2019.02.17            2cgxLIOcTZy9ZaqHqGg4VQ   5   1    4304920            0      1.8gb        966.5mb
green  open   apic_logs-2019.03.01            eUmq-H9kSnCKm9HSFqlwOw   5   1    4772930            0      2.1gb            1gb
green  open   apic_logs-2019.02.22            R1y-aos4SFapJmRMoC4kPA   5   1   10721569            0      4.6gb          2.3gb
green  open   apic_logs-2019.03.08            4m4bA6YmTmKG1ry04ekMGw   5   1    3643720            0      1.6gb        849.5mb
green  open   apic_logs-2019.03.11            xCsqLp0KRyymUFzWwqOkdw   5   1   13596920            0      5.9gb          2.9gb
green  open   apic_logs-2019.01.15            zs4Y0jxnR0eCyvfHEwRNiQ   5   1      18309            0      6.6mb          3.3mb
green  open   apic_logs-2019.01.23            R87m0dAoQmWm-cT2UyXWAA   5   1      19035            0      7.1mb          3.5mb
green  open   apic_logs-2019.01.18            sAM_fH_MTRWY7iPazmpHZw   5   1      18371            0      6.7mb          3.3mb
green  open   apic_logs-2019.02.25            5KMLpMijTvilD3_UqiwReA   5   1   10537317            0      4.5gb          2.2gb
green  open   apic_logs-2019.02.19            rZ0LgjA5Tjegh5YCjOyZvg   5   1    3585512            0      1.6gb        820.9mb
3

You do not have a date filter for apic_logs. The syslog_timestamp field matches the start of the message field as expected. What's the problem?

4

my bad, let me check on that.

5

@Badger, i have updated the config file which was missed during post, but fact is still i'm getting same issues both for noi-syslog and apic_logs.

6

Hard to reconcile this. If you have a date filter and the field you are matching exists, then if the field is parsed the target (@timestamp) is updated, otherwise you get a _dateparsefailure tag. The only time I know of when @timestamp is unchanged and there is no tag is when the source field (syslog_timestamp) does not exist.

1 Like
7

hmm.. Is there anything bad in the filter section which i'm missing?

8

I cannot see anything wrong with the filter.

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.