grok patterns are not getting applied. Please apply.
grok debugger shows no issue.
please find the filebeat.yml
filebeat.inputs:
- type: log
paths:
- C:/Program Files (x86)/dox/onion.log
fields:
tags: application
environment: test
product: po
datacenter: aws
partition: test0
server_role: test-po-test0-application_server
fields_under_root: true
filebeat.registry.path: 'C:\Program Files\Filebeat\registry'
output.logstash:
hosts: ["logsrv.vision-test.local:5044"]
key: "test-epo"
db: 0
db_topology: 1
timeout: 5
reconnect_interval: 1
shipper:
logging:
to_syslog: false
to_files: true
files:
path: C:\filebeatlogs\
name: filebeat.log
rotateeverybytes: 10485760 # = 10MB
keepfiles: 7
level: debug
input {
beats {
port => 5044
}
}
# po filter
filter
{
if "po" in [product] and "application" in [tags] and ["filebeat" or "log"] in [type]
{
grok
{
patterns_dir => ["/etc/logstash/conf.d/patterns"]
match =>
{
"message" =>
[
# Orion.log
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}\-%{SPACE}BPS\:%{SPACE}New%{SPACE}customer%{SPACE}to%{SPACE}be%{SPACE}added\:%{SPACE}%{GREEDYDATA:CustomerName}$",
"^%{TIMESTAMP_ISO8601:logtimeStamp}%{SPACE}{1,2}%{WORD:LogLevel}%{SPACE}\[%{USERNAME:httpcall}]%{SPACE}%{USERNAME:dbName}%{SPACE}%{USERNAME:tenantGuid}%{SPACE}%{INT:TenantId}%{SPACE}%{INT:userId}%{SPACE}%{USERNAME:sessionID}%{SPACE}%{GREEDYDATA:message}$",
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}%{DATA:ActualMsg}\n%{ExcMessage:ExceptionMessage}$",
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}%{DATA:ActualMsg}\n%{DATA:ExceptionMessage}\n%{CUSTOM_TRACE_Caused:StackTrace}$",
# New filters with startTime, endTime
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}TenantId\:%{WORD:TenantId}\,QueryName\:%{DATA:QueryName}\,QueryGUID\:%{DATA:QueryGUID}\,StartTime\:%{TIMESTAMP_ISO8601:StartTime}\,EndTime\:%{TIMESTAMP_ISO8601:EndTime}\,RowCount\:%{NUMBER:RowCount:int}\,Data\:%{GREEDYDATA:Data}$",
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}TenantId\:%{WORD:TenantId}\,QueryName\:%{DATA:QueryName}\,QueryGUID\:%{DATA:QueryGUID}\,StartTime\:%{TIMESTAMP_ISO8601:StartTime}\,EndTime\:%{TIMESTAMP_ISO8601:EndTime}\,Data\:%{GREEDYDATA:Data}$",
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}TenantId\:%{WORD:TenantId}\,QueryName\:%{DATA:QueryName}\,QueryGUID\:%{DATA:QueryGUID}\,StartTime\:%{TIMESTAMP_ISO8601:StartTime}\,Data\:%{GREEDYDATA:Data}$",
# generic
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}%{GREEDYDATA:ActualMsg}$"
]
}
}
if [StartTime] and [EndTime]
{
if [LogModule] == "squid.QueryRportingImpl"
{
ruby
{
init => "require 'time'"
#5.1 logstash
code => "event.set('ExecutionTime', Time.iso8601(event.get('EndTime').to_s).to_i - Time.iso8601(event.get('StartTime').to_s).to_i) "
# 2.3 logstash:->
#code => "event['ExecutionTime'] = Time.iso8601(event['EndTime'].to_s).to_i - Time.iso8601(event['StartTime'].to_s).to_i ;"
add_tag => [ "calculated_rporting_time" ]
}
}
else
{
ruby
{
init => "require 'time'"
#5.1 logstash
code => "event.set('ExecutionTime', Time.iso8601(event.get('EndTime').to_s).to_i - Time.iso8601(event.get('StartTime').to_s).to_i) "
#2.3 logstash:->
#code => "event['ExecutionTime'] = Time.iso8601(event['EndTime'].to_s).to_i - Time.iso8601(event['StartTime'].to_s).to_i ;"
add_tag => [ "calculated_execution_time" ]
}
}
}
# generic
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}%{GREEDYDATA:ActualMsg}$"
]
}
}
if [StartTime] and [EndTime]
{
if [LogModule] == "squid.QueryRportingImpl"
{
ruby
{
init => "require 'time'"
#5.1 logstash
code => "event.set('ExecutionTime', Time.iso8601(event.get('EndTime').to_s).to_i - Time.iso8601(event.get('StartTime').to_s).to_i) "
# 2.3 logstash:->
#code => "event['ExecutionTime'] = Time.iso8601(event['EndTime'].to_s).to_i - Time.iso8601(event['StartTime'].to_s).to_i ;"
add_tag => [ "calculated_rporting_time" ]
}
}
else
{
ruby
{
init => "require 'time'"
#5.1 logstash
code => "event.set('ExecutionTime', Time.iso8601(event.get('EndTime').to_s).to_i - Time.iso8601(event.get('StartTime').to_s).to_i) "
#2.3 logstash:->
#code => "event['ExecutionTime'] = Time.iso8601(event['EndTime'].to_s).to_i - Time.iso8601(event['StartTime'].to_s).to_i ;"
add_tag => [ "calculated_execution_time" ]
}
}
}
# generic
"^%{TIMESTAMP_ISO8601:LogTimestamp}%{SPACE}{1,2}%{LOGLEVEL:LogLevel}%{SPACE}\[%{DATA:ProcessId}\]%{SPACE}%{DATA:LogModule}%{SPACE}-%{SPACE}%{GREEDYDATA:ActualMsg}$"
]
}
}
if [StartTime] and [EndTime]
{
if [LogModule] == "squid.QueryRportingImpl"
{
ruby
{
init => "require 'time'"
#5.1 logstash
code => "event.set('ExecutionTime', Time.iso8601(event.get('EndTime').to_s).to_i - Time.iso8601(event.get('StartTime').to_s).to_i) "
# 2.3 logstash:->
#code => "event['ExecutionTime'] = Time.iso8601(event['EndTime'].to_s).to_i - Time.iso8601(event['StartTime'].to_s).to_i ;"
add_tag => [ "calculated_rporting_time" ]
}
}
else
{
ruby
{
init => "require 'time'"
#5.1 logstash
code => "event.set('ExecutionTime', Time.iso8601(event.get('EndTime').to_s).to_i - Time.iso8601(event.get('StartTime').to_s).to_i) "
#2.3 logstash:->
#code => "event['ExecutionTime'] = Time.iso8601(event['EndTime'].to_s).to_i - Time.iso8601(event['StartTime'].to_s).to_i ;"
add_tag => [ "calculated_execution_time" ]
}
}
}
add_field => {"onion logs working" => "good"}
}
}
}
output
{
if "po" in [product]
{
amazon_es
{
hosts => ["******************************************************"]
region => "us-east-1"
index => "%{[server_role]}-test-success-working-%{+YYYY.MM.dd}"
}
}
if [product] == "po"
{
stdout { codec => rubydebug }
}
}
logstash output:
{
"input" => {
"type" => "log"
},
"log" => {
"file" => {
"path" => "C:/Program Files (x86)/dox/onion.log"
},
"offset" => 2549706
},
"product" => "po",
"server_role" => "uflek-po-uflek0-app_server",
"partition" => "uflek0",
"@timestamp" => 2021-02-01T14:55:20.355Z,
"ecs" => {
"version" => "1.5.0"
},
"@version" => "1",
"datacenter" => "aws",
"message" => "2021-02-01T14:55:18,799 INFO [http-nio-8080-exec-25] servlet.ReadyStatusServlet 11111111-0000-0000-0000-111111111111 20 9AEDC043453E876C5668A961A650CF0A.route1 - doGet call for ReadyStatusServlet. Calling doPost.",
"tags" => [
[0] "epoapp",
[1] "beats_input_codec_plain_applied"
],
"host" => {
"name" => "*************"
},
"agent" => {
"hostname" => "***************"
"type" => "filebeat",
"version" => "7.9.3",
"id" => "a0b30d9e-580b-4496-846b-506bdb22fab3",
"ephemeral_id" => "5016cdeb-988e-48cf-8329-d94622a85c11"
},
"environment" => "test"
}