The service writes logs in a special way. Part of the time is specified in the file name 23032807.log (yyMMddHHH). Minutes, seconds and microseconds are written inside the log 03:42.370003 (mm:ss.SSS).
I created such a filter.conf for logstash:
filter {
if "onec" in [tags] {
grok {
match => { "message" => "%{NUMBER:num_min}:%{BASE10NUM:num_sec}-%{WORD:duration},%{WORD:event1c},%{WORD:level_event},process=%{WORD:process}" }
match => { "[log][file][path]" => "%{NUMBER:logfilename}.log"}
break_on_match => false
}
mutate {
add_field => { "fieldTime" => "%{logfilename}%{num_min}%{num_sec}" }
}
date {
match => ["fieldTime", "yyMMddHHmmss.SSS"]
timezone => "Asia/Almaty"
locale => "kz"
target => "@timestamp"
}
}
}
It turns out such a document in kibana:
{
"@timestamp": [
"2023-03-28T04:03:43.871Z"
],
"@version": [
"1"
],
"@version.keyword": [
"1"
],
"agent.ephemeral_id": [
"d601fabe-d38a-440c-9bf5-fdbcc5aab055"
],
"agent.ephemeral_id.keyword": [
"d601fabe-d38a-440c-9bf5-fdbcc5aab055"
],
"agent.id": [
"246a153a-35c7-4919-81e0-0e1260a371ac"
],
"agent.id.keyword": [
"246a153a-35c7-4919-81e0-0e1260a371ac"
],
"agent.name": [
"BEdev1c2"
],
"agent.name.keyword": [
"BEdev1c2"
],
"agent.type": [
"filebeat"
],
"agent.type.keyword": [
"filebeat"
],
"agent.version": [
"8.6.2"
],
"agent.version.keyword": [
"8.6.2"
],
"duration": [
"0"
],
"duration.keyword": [
"0"
],
"ecs.version": [
"8.0.0"
],
"ecs.version.keyword": [
"8.0.0"
],
"event.original": [
"03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
],
"event.original.keyword": [
"03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
],
"event1c": [
"EXCP"
],
"event1c.keyword": [
"EXCP"
],
"fieldTime": [
"230328070342.370003"
],
"fieldTime.keyword": [
"230328070342.370003"
],
"host.name": [
"BEdev1c2"
],
"host.name.keyword": [
"BEdev1c2"
],
"input.type": [
"filestream"
],
"input.type.keyword": [
"filestream"
],
"level_event": [
"0"
],
"level_event.keyword": [
"0"
],
"log.file.path": [
"D:\\1C\\logs\\rphost_8612\\23032807.log"
],
"log.file.path.keyword": [
"D:\\1C\\logs\\rphost_8612\\23032807.log"
],
"log.flags": [
"multiline"
],
"log.flags.keyword": [
"multiline"
],
"log.offset": [
9319
],
"logfilename": [
"23032807"
],
"logfilename.keyword": [
"23032807"
],
"message": [
"03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
],
"message.keyword": [
"03:42.370003-0,EXCP,0,process=rphost,OSThread=9948,Exception=d294e384-7ea6-49c6-be96-f3a6e3de1242,Descr='LoadComponent(liccspr):\nd294e384-7ea6-49c6-be96-f3a6e3de1242: Ошибка загрузки компоненты liccspr: 126(0x0000007E): The specified module could not be found. '"
],
"num_min": [
"03"
],
"num_min.keyword": [
"03"
],
"num_sec": [
"42.370003"
],
"num_sec.keyword": [
"42.370003"
],
"process": [
"rphost"
],
"process.keyword": [
"rphost"
],
"tags": [
"onec",
"beats_input_codec_plain_applied",
"_dateparsefailure"
],
"tags.keyword": [
"onec",
"beats_input_codec_plain_applied",
"_dateparsefailure"
],
"_id": "n8JjJocBsYhuK8iyIjpK",
"_index": "onec-2023.03",
"_score": null
}
The date does not match the timestamp and an error is displayed
Can you tell me how to fix the filter, so that the time of logging replaces the timestamp?