I can copy and paste a the log from kibana discover into the debugger, then open up the .conf file copy and past the grok from there, and press simulate. Ever single time without fail it works, but Its only partly working in what log stash is sending.
What I see is debugger pulls all data, logstash only sends the first response. I am less than a noob as I am sure you will see looking at the filter. I was so happy when I finally got it to pass in the debugger, I tried two other online debuggers as well they all work. I just am at a loss now as to what to try.
example live log:
<134>Mar 08 13:59:22 bigiphw02 tmm[24490]: 2021-03-08 13:59:22 bigiphw02.mcfeetershq.local qid 65168 to 192.168.100.100#49779: [NOERROR qr,rd,ra] response: TABlEtcAPTIVEpORTAl.cOM. 35 IN A 54.160.88.141; TABlEtcAPTIVEpORTAl.cOM. 35 IN A 54.210.231.5; TABlEtcAPTIVEpORTAl.cOM. 35 IN A 3.220.89.219; TABlEtcAPTIVEpORTAl.cOM. 35 IN A 34.197.75.254; TABlEtcAPTIVEpORTAl.cOM. 35 IN A 52.1.124.91; TABlEtcAPTIVEpORTAl.cOM. 35 IN A 52.204.181.151;
example filter:
%{SYSLOG5424PRI}%{SYSLOGBASE2} %{URIHOST:date_stamp} %{TIME:time_stamp} %{IPORHOST:bigip_host} qid %{NUMBER:qid} to %{IPORHOST:client}#%{NUMBER}: [NOERROR qr,rd,ra] response: %{IPORHOST:query}[.;]* %{NUMBER:ttl} IN %{WORD:response_type} %{IPORHOST:response};( %{IPORHOST:query2}[.;]* %{NUMBER:ttl2} IN %{WORD:response_type2} %{IPORHOST:response2}[.;])?( %{IPORHOST:query3}[.;] %{NUMBER:ttl3} IN %{WORD:response_type3} %{IPORHOST:response3}[.;])?( %{IPORHOST:query4}[.;] %{NUMBER:ttl4} IN %{WORD:response_type4} %{IPORHOST:response4}[.;])?( %{IPORHOST:query5}[.;] %{NUMBER:ttl5} IN %{WORD:response_type5} %{IPORHOST:response5}[.;])?( %{IPORHOST:query6}[.;] %{NUMBER:ttl6} IN %{WORD:response_type6} %{IPORHOST:response6}[.;])?( %{IPORHOST:query7}[.;] %{NUMBER:ttl7} IN %{WORD:response_type7} %{IPORHOST:response7}[.;])?( %{IPORHOST:query8}[.;] %{NUMBER:ttl} IN %{WORD:response_type8} %{IPORHOST:response8}[.;])?( %{IPORHOST:query9}[.;] %{NUMBER:ttl9} IN %{WORD:response_type9} %{IPORHOST:response9}[.;])?( %{IPORHOST:query10}[.;] %{NUMBER:ttl10} IN %{WORD:response_type10} %{IPORHOST:response10}[.;]*)?
example output
{
"bigip_host": "bigiphw02.mcfeetershq.local",
"response_type": "A",
"pid": "24490",
"program": "tmm",
"qid": "65168",
"date_stamp": "2021-03-08",
"syslog5424_pri": "134",
"query4": "TABlEtcAPTIVEpORTAl.cOM.",
"client": "192.168.100.100",
"query5": "TABlEtcAPTIVEpORTAl.cOM.",
"query6": "TABlEtcAPTIVEpORTAl.cOM.",
"query2": "TABlEtcAPTIVEpORTAl.cOM.",
"timestamp": "Mar 08 13:59:22",
"query3": "TABlEtcAPTIVEpORTAl.cOM.",
"time_stamp": "13:59:22",
"query": "TABlEtcAPTIVEpORTAl.cOM.",
"logsource": "bigiphw02",
"ttl": "35",
"ttl2": "35",
"ttl3": "35",
"response": "54.160.88.141",
"response_type4": "A",
"response4": "34.197.75.254",
"response3": "3.220.89.219",
"response_type5": "A",
"response2": "54.210.231.5",
"response_type6": "A",
"ttl6": "35",
"response_type2": "A",
"ttl4": "35",
"response6": "52.204.181.151",
"response_type3": "A",
"ttl5": "35",
"response5": "52.1.124.91"
}
Example live out put from elastic
{
"_index": "dns-2021.03.08",
"_type": "_doc",
"_id": "JyNtE3gBARIOtW6Kf-m0",
"_version": 1,
"_score": null,
"_source": {
"bigip_host": "bigiphw02.mcfeetershq.local",
"query": "tableTcAptIVePorTAl.CoM.",
"response_type": "A",
"date_stamp": "2021-03-08",
"type": "dns",
"syslog5424_pri": "134",
"qid": "57629",
"pid": "24490",
"client": "192.168.100.100",
"response": "52.204.181.151",
"program": "tmm",
"timestamp": "Mar 08 14:00:46",
"host": "192.168.100.251",
"message": "<134>Mar 08 14:00:46 bigiphw02 tmm[24490]: 2021-03-08 14:00:45 bigiphw02.mcfeetershq.local qid 57629 to 192.168.100.100#65284: [NOERROR qr,rd,ra] response: tableTcAptIVePorTAl.CoM. 46 IN A 52.204.181.151; tableTcAptIVePorTAl.CoM. 46 IN A 54.160.88.141; tableTcAptIVePorTAl.CoM. 46 IN A 54.210.231.5; tableTcAptIVePorTAl.CoM. 46 IN A 3.220.89.219; tableTcAptIVePorTAl.CoM. 46 IN A 34.197.75.254; tableTcAptIVePorTAl.CoM. 46 IN A 52.1.124.91;",
"tags": [
"dns",
"5515"
],
"logsource": "bigiphw02",
"ttl": "46",
"@version": "1",
"time_stamp": "14:00:45",
"port": 27435,
"@timestamp": "2021-03-08T20:00:46.327Z"
},
"fields": {
"date_stamp": [
"2021-03-08T00:00:00.000Z"
],
"@timestamp": [
"2021-03-08T20:00:46.327Z"
]
},
"highlight": {
"query": [
"@kibana-highlighted-field@tableTcAptIVePorTAl.CoM@/kibana-highlighted-field@."
]
},
"sort": [
1615233646327
]
}`