Get timestamp from your field

The service writes logs in a special way. Part of the time is specified in the file name 23032807.log (yyMMddHHH). Minutes, seconds and microseconds are written inside the log 03:42.370003 (mm:ss.SSS).

I created such a filter.conf for logstash:

filter {
    if "onec" in [tags] {
        grok {
            match => { "message" => "%{NUMBER:num_min}:%{BASE10NUM:num_sec}-%{WORD:duration},%{WORD:event1c},%{WORD:level_event},process=%{WORD:process}" }
            match => { "[log][file][path]" => "%{NUMBER:logfilename}.log"}
            break_on_match => false
        }
        mutate {
            add_field => { "fieldTime" => "%{logfilename}%{num_min}%{num_sec}" }
        }
        date {
            match => ["fieldTime", "yyMMddHHmmss.SSS"]
            timezone => "Asia/Almaty"
            locale => "kz"
            target => "@timestamp"
        }
    }
}

It turns out such a document in kibana:

{
  "@timestamp": [
    "2023-03-28T04:03:43.871Z"
  ],
  "@version": [
    "1"
  ],
  "@version.keyword": [
    "1"
  ],
  "agent.ephemeral_id": [
    "d601fabe-d38a-440c-9bf5-fdbcc5aab055"
  ],
  "agent.ephemeral_id.keyword": [
    "d601fabe-d38a-440c-9bf5-fdbcc5aab055"
  ],
  "agent.id": [
    "246a153a-35c7-4919-81e0-0e1260a371ac"
  ],
  "agent.id.keyword": [
    "246a153a-35c7-4919-81e0-0e1260a371ac"
  ],
  "agent.name": [
    "BEdev1c2"
  ],
  "agent.name.keyword": [
    "BEdev1c2"
  ],
  "agent.type": [
    "filebeat"
  ],
  "agent.type.keyword": [
    "filebeat"
  ],
  "agent.version": [
    "8.6.2"
  ],
  "agent.version.keyword": [
    "8.6.2"
  ],
  "duration": [
    "0"
  ],
  "duration.keyword": [
    "0"
  ],
  "ecs.version": [
    "8.0.0"
  ],
  "ecs.version.keyword": [
    "8.0.0"
  ],
  "event.original": [
    "03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
  ],
  "event.original.keyword": [
    "03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
  ],
  "event1c": [
    "EXCP"
  ],
  "event1c.keyword": [
    "EXCP"
  ],
  "fieldTime": [
    "230328070342.370003"
  ],
  "fieldTime.keyword": [
    "230328070342.370003"
  ],
  "host.name": [
    "BEdev1c2"
  ],
  "host.name.keyword": [
    "BEdev1c2"
  ],
  "input.type": [
    "filestream"
  ],
  "input.type.keyword": [
    "filestream"
  ],
  "level_event": [
    "0"
  ],
  "level_event.keyword": [
    "0"
  ],
  "log.file.path": [
    "D:\\1C\\logs\\rphost_8612\\23032807.log"
  ],
  "log.file.path.keyword": [
    "D:\\1C\\logs\\rphost_8612\\23032807.log"
  ],
  "log.flags": [
    "multiline"
  ],
  "log.flags.keyword": [
    "multiline"
  ],
  "log.offset": [
    9319
  ],
  "logfilename": [
    "23032807"
  ],
  "logfilename.keyword": [
    "23032807"
  ],
  "message": [
    "03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
  ],
  "message.keyword": [
    "03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
  ],
  "num_min": [
    "03"
  ],
  "num_min.keyword": [
    "03"
  ],
  "num_sec": [
    "42.370003"
  ],
  "num_sec.keyword": [
    "42.370003"
  ],
  "process": [
    "rphost"
  ],
  "process.keyword": [
    "rphost"
  ],
  "tags": [
    "onec",
    "beats_input_codec_plain_applied",
    "_dateparsefailure"
  ],
  "tags.keyword": [
    "onec",
    "beats_input_codec_plain_applied",
    "_dateparsefailure"
  ],
  "_id": "n8JjJocBsYhuK8iyIjpK",
  "_index": "onec-2023.03",
  "_score": null
}

The date does not match the timestamp and an error is displayed

Can you tell me how to fix the filter, so that the time of logging replaces the timestamp?

You have to match the entire field in a date filter. SSS only matches 3 digits of sub-second accuracy, you need to match six. Try match => ["fieldTime", "yyMMddHHmmss.SSSSSS"]

That should result in "@timestamp" => 2023-03-28T01:03:42.370Z,. You can search the forums for ways to get micro- or nano-second accuracy parsing.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.