I'm trying to parse SOLR logs using logstash. Below i have pasted results from grok debugger but i dont see same results in elastic/kibana.
Here is my logstash filter:
if [fields][log_type] == "SOLR_0343"{
grok {
match => {
"message" => ["%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:LOGLEVEL} %{JAVACLASS:javaclass}: [%{WORD:collection;string}] webapp=/%{WORD:webapp} path=/%{WORD:path} params={%{GREEDYDATA:params}} hits=%{NUMBER:hits} status=%{NUMBER:status} QTime=%{NUMBER:qtime}"]
}
}
date {
match => [ "timestamp" , "yyyy-MM-dd HH:mm:ss,SSS" ]
}
}
}
data i'm parsing :
2017-08-14 23:01:50,340 INFO org.apache.solr.core.SolrCore.Request: [tra_1_shard1_replica1] webapp=/solr path=/old params={q=NUM_s:3692-|BNUM_s:444500-|CC_NM_s:xxxx-|CD_s:72-|FUNC_VAL_s:AL+DRA-|C_CD_s:48-|L_CD_s:75080-|CD_s:444313-|Y_CD_s:&qt=/old&sort=priority+asc&rows=1&wt=javabin&version=2} hits=2 status=0 QTime=8
I have verified this in grok debugger and i get expected results:
{
"timestamp": [
[
"2017-08-09 18:51:13,316"
]
],
"YEAR": [
[
"2017"
]
],
"MONTHNUM": [
[
"08"
]
],
"MONTHDAY": [
[
"09"
]
],
"HOUR": [
[
"18",
null
]
],
"MINUTE": [
[
"51",
null
]
],
"SECOND": [
[
"13,316"
]
],
"ISO8601_TIMEZONE": [
[
null
]
],
"LOGLEVEL": [
[
"INFO"
]
],
"javaclass": [
[
"org.apache.solr.core.SolrCore.Request"
]
],
"collection": [
[
"tra_1_shard1_replica1"
]
],
"webapp": [
[
"solr"
]
],
"path": [
[
"old"
]
],
"params": [
[
"q=NUM_s:3692-|BNUM_s:444500-|CC_NM_s:xxxx-|CD_s:72-|FUNC_VAL_s:AL+DRA-|C_CD_s:48-|L_CD_s:75080-|CD_s:444313-|Y_CD_s:&qt=/old&sort=priority+asc&rows=1&wt=javabin&version=2"
]
],
"hits": [
[
"1"
]
],
"BASE10NUM": [
[
"1",
"0",
"8"
]
],
"status": [
[
"0"
]
],
"qtime": [
[
"8"
]
]
}
Here is the JSON i see in elastic:
{
"_index": "solr-2017.08.14",
"_type": "apache",
"_id": "AV3i9cHbhCPiQUeDbXWa",
"_version": 1,
"_score": null,
"_source": {
"offset": 134975933,
"input_type": "log",
"source": "/var/log/solr/solr-cmf-solr2-SOLR_SERVER-sl55caehdp0343.visa.com.log.out",
"message": "org.apache.solr.core.SolrCore.Request: [tran_stamping_1_shard1_replica1] webapp=/solr path=/old params={q=NUM_s:3692-|BNUM_s:444500-|CC_NM_s:xxxx-|CD_s:72-|FUNC_VAL_s:AL+DRA-|C_CD_s:48-|L_CD_s:75080-|CD_s:444313-|Y_CD_s:&qt=/old&sort=priority+asc&rows=1&wt=javabin&version=2} hits=1 status=0 QTime=10 ",
"type": "apache",
"tags": [
"beats_input_codec_plain_applied",
"version",
"CDH5.5.1",
"_grokparsefailure"
],
"@timestamp": "2017-08-14T22:55:41.918Z",
"@version": "1",
"beat": {
"hostname": "xxxxxxx",
"name": "xxxxxx",
"version": "5.4.1"
},
"host": "xxxxxxxxx",
"fields": {
"log_type": "SOLR_0343"
},
"LOGLEVEL": "INFO",
"timestamp": "2017-08-14 22:55:41,918"
},
"fields": {
"@timestamp": [
1502751341918
]
},
"sort": [
1502751341918
]
}
Any help is much appreciated!!