We have log file in which we have to capture the first line matching "TIMESTAMP_ISO8601" against build_StartTime filed and last line matching "TIMESTAMP_ISO8601" against build_EndTime filed. After this we have to calculate the difference and should store the difference against duration filed. We tried with date filter but that's not working out we tried other many options which didn't worked. Can you help us here
Note : For duration calculation we are planning scripted filed or ruby fliter here the build_StartTime & build_EndTime fileds are coming as string rather than date
Log File:
[2019-01-31 21:28:22Z INFO Program] Version: 2.122.1
some logs in the middle
[2019-01-31 21:28:45Z INFO Worker] Job completed.
Below is our current Logstash config File:
input {
beats {
client_inactivity_timeout => 1200
port => 5002
}
}
filter
{
if [message] =~ "Version: 2.122.1"
{
grok {
add_tag => [ "start" ]
match => { "message" => "%{TIMESTAMP_ISO8601:build_StartTime}" }
}
date {
add_tag => [ "start" ]
match => ["build_StartTime","ISO8601"]
target => "build_StartTime"
}
}
grok {
add_tag => [ "start" ]
break_on_match => false
match => {
"message" => [
'"..definitionName": "(?<build_DefinitionName>.?)"',
'"..requestedFor": "(?<build_RequesterName>.?)"'
]
}
}
if [message] =~ "Job result after all post-job steps finish:"
{
grok {
add_tag => [ "start" ]
match => { "message" => "%{TIMESTAMP_ISO8601:build_EndTime}" }
}
date {
add_tag => [ "start" ]
match => ["build_EndTime","ISO8601"]
target => "build_EndTime"
}
grok {
add_tag => ["end"]
match => { "message" => "Job result after all post-job steps finish:(?<build_Status>.([A-Za-z]*))" }
}
if ![build_Status] or [build_Status] == " "
{
mutate {
add_tag => [ "start" ]
update => { "build_Status" => "Succeeded" }
}
}
}
if "start" in [tags] {
aggregate {
task_id => "%{source}"
code => "
map['build_DefinitionName'] = event.get('build_DefinitionName') unless event.get('build_DefinitionName').nil?
map['build_RequesterName'] = event.get('build_RequesterName') unless event.get('build_RequesterName').nil?
map['build_StartTime'] = event.get('build_StartTime') unless event.get('build_StartTime').nil?
map['build_EndTime'] = event.get('build_EndTime') unless event.get('build_EndTime').nil?
map['build_Status'] = event.get('build_Status') unless event.get('build_Status').nil?
"
}
}
if "end" in [tags] {
aggregate {
task_id => "%{source}"
code => "
event.set('build_DefinitionName', map['build_DefinitionName'])
event.set('build_RequesterName', map['build_RequesterName'])
event.set('build_StartTime', map['build_StartTime'])
event.set('build_EndTime', map['build_EndTime'])
event.set('build_Status', map['build_Status'])
"
end_of_task => true
}
}
if "end" not in [tags] or [build_DefinitionName] == 'nil' or ![build_DefinitionName] {
drop { }
}
mutate {
remove_field => [ "message" ]
}
mutate {
remove_tag => [ "start" ]
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "tfslog-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}
Output:
{
"_index": "tfslog-2019.02.21",
"_type": "doc",
"_id": "xdPOD2kBAWdIEry4hEZ0",
"_version": 1,
"_score": null,
"_source": {
"tags": [
"beats_input_codec_plain_applied",
"_dateparsefailure",
"end"
],
"build_DefinitionName": "ResultCalculation_NB",
"@timestamp": "2019-02-21T11:25:24.719Z",
"source": "C:\TFS2017_Agent\_diag\Worker_20190134-00015-utc.log",
"build_EndTime": "2019-01-31 21:28:22Z",
"host": {
"name": "HOST103029"
},
"build_StartTime": "2019-01-31 21:28:22Z",
"build_Status": " Failed",
"@version": "1",
"build_RequesterName": "Microsoft.TeamFoundation.System",
"offset": 19709,
"beat": {
"version": "6.3.0",
"name": "RMD-VRA103029",
"hostname": "RMD-VRA103029"
}
},
"fields": {
"@timestamp": [
"2019-02-21T11:25:24.719Z"
]
},
"sort": [
1550748324719
]
}