In my Logstash configuration I have a block saying:
else if [fields][source] == "Scan4SoftwareAndLicenses" {
grok {
match => { "message" => "\"MachineData\",\"%{WORD:host_short}\",\"%{DATA:fqdn}\",\"%{WORD:os}\",\"%{DATA:os_version}\",\"%{NUMBER:siblings}\",\"%{NUMBER:cpus}\",\"%{NUMBER:gbram}\",\"%{NUMBER:gbtier1}\",\"%{NUMBER:gbtier2}\"" }
match => { "message" => "\"ApacheRPM\",\"%{DATA:apachebin}\",\"%{DATA:apacheversion}\",\"%{DATA:rpmname}\"" }
match => { "message" => "\"JavaRPM\",\"%{DATA:javabin}\",\"%{DATA:javaversion}\",\"%{DATA:rpmname}\"" }
match => { "message" => "\"WeblogicServerVersion\",\"%{DATA:wlsversion}\"" }
match => { "message" => "\"OPatch\",\"%{DATA:oraclepatch}\",\"%{DATA:oraclepatchtype}\"" }
match => { "message" => "\"MQVER\",\"Name\",\"%{DATA:mqname}\",\"Version\",\"%{DATA:mqversion}\",\"Level\",\"%{DATA:mqlevel}\",\"BuildType\",\"%{DATA:mqbuildtype}\",\"Platform\",\"%{DATA:mqplatform}\",\"Mode\",\"%{DATA:mqmode}\",\"O/S\",\"%{DATA:mqos}\",\"O/S Details\",\"%{DATA:mqosdetails}\",\"InstName\",\"%{DATA:mqinstname}\",\"InstDesc\",\"%{DATA:mqinstdesc}\",\"Primary\",\"%{DATA:mqprimary}\",\"InstPath\",\"%{DATA:mqinstpath}\",\"DataPath\",\"%{DATA:mqdatapath}\",\"MaxCmdLevel\",\"%{DATA:mqmaxcmdlevel}\",\"LicenseType\",\"%{DATA:mqlicensetype}\"" }
match => { "message" => "\"ELKLicense\",\"status\",\"%{DATA:elkstatus}\",\"uid\",\"%{DATA:elkuid}\",\"type\",\"%{DATA:elktype}\",\"issue_date\",\"%{TIMESTAMP_ISO8601:elkissue}\",\"issue_date_in_millis\",\"%{POSINT:elkissuems}\",\"expiry_date\",\"%{TIMESTAMP_ISO8601:elkexpiry}\",\"expiry_date_in_millis\",\"%{POSINT:elkexpiryms}\",\"max_nodes\",\"%{POSINT:elkmaxnodes}\",\"max_resource_units\",\"%{DATA:maxresourceunits}\",\"issued_to\",\"%{DATA:elkissuedto}\",\"issuer\",\"%{DATA:elkissuer}\",\"start_date_in_millis\",\"%{POSINT:elkstartdatems}\",\"name\",\"%{DATA:elkname}\",\"cluster_name\",\"%{DATA:elkclustername}\",\"cluster_uuid\",\"%{DATA:elkclusteruuid}\",\"number\",\"%{DATA:elknumber}\",\"lucene_version\",\"%{DATA:elkluceneversion}\",\"minimum_wire_compatibility_version\",\"%{DATA:elkminwirecompatversion}\",\"minimum_index_compatibility_version\",\"%{DATA:elkminindexcompatversion}\",\"logstash\",\"%{DATA:logstashversion}\",\"kibana\",\"%{DATA:kibanaversion}\"" }
# match => { "message" => "\"ELKLicense\",\"status\",\"%{DATA:elkstatus}\",\"uid\",\"%{DATA:elkuid}\",\"type\",\"%{DATA:elktype}\",\"issue_date\",\"%{TIMESTAMP_ISO8601:elkissue}\",\"issue_date_in_millis\",\"%{POSINT:elkissuems}\",\"expiry_date\",\"%{TIMESTAMP_ISO8601:elkexpiry}\",\"expiry_date_in_millis\",\"%{POSINT:elkexpiryms}\",\"max_nodes\",\"%{POSINT:elkmaxnodes}\",\"issued_to\",\"%{DATA:elkissuedto}\",\"issuer\",\"%{DATA:elkissuer}\",\"start_date_in_millis\",\"%{POSINT:elkstartdatems}\",\"name\",\"%{DATA:elkname}\",\"cluster_name\",\"%{DATA:elkclustername}\",\"cluster_uuid\",\"%{DATA:elkclusteruuid}\",\"number\",\"%{DATA:elknumber}\",\"lucene_version\",\"%{DATA:elkluceneversion}\",\"minimum_wire_compatibility_version\",\"%{DATA:elkminwirecompatversion}\",\"minimum_index_compatibility_version\",\"%{DATA:elkminindexcompatversion}\",\"logstash\",\"%{DATA:logstashversion}\",\"kibana\",\"%{DATA:kibanaversion}\"" }
}
mutate {
add_field => { "index_prefix" => "iostatvxvm" }
}
}
In my log is a line
"ELKLicense","status","active","uid","xyz","type","gold","issue_date","2022-04-28T00:00:00.000Z","issue_date_in_millis","1651104000000","expiry_date","2024-07-31T23:59:59.999Z","expiry_date_in_millis","1722470399999","max_nodes","4","max_resource_units","null","issued_to","xyz","issuer","API","start_date_in_millis","1533081600000","name","xyz","cluster_name","xyz","cluster_uuid","xyz","number","8.4.2","lucene_version","9.3.0","minimum_wire_compatibility_version","7.17.0","minimum_index_compatibility_version","7.0.0","logstash","8.4.2","kibana","8.4.2"
When running the Grok pattern in the Dev Tools Debugger, I get
{
"elkminwirecompatversion": "7.17.0",
"elktype": "gold",
"elkmaxnodes": "4",
"logstashversion": "8.4.2",
"kibanaversion": "8.4.2",
"elkclustername": "xyz",
"elkminindexcompatversion": "7.0.0",
"elkissuer": "API",
"elkstatus": "active",
"elkuid": "xyz",
"maxresourceunits": "null",
"elkstartdatems": "1533081600000",
"elkexpiryms": "1722470399999",
"elkluceneversion": "9.3.0",
"elkexpiry": "2024-07-31T23:59:59.999Z",
"elkissuems": "1651104000000",
"elkname": "xyz",
"elkclusteruuid": "xyz",
"elkissuedto": "xyz",
"elkissue": "2022-04-28T00:00:00.000Z",
"elknumber": "8.4.2"
}
But when Logstash tries to read it, it throws a grokparsefailure. There is no entry in logstash-plain.log referring to this particular parse error. Other log entries like
"JavaRPM","/appl/elk/logstash-7.17.0/jdk/bin/java","11.0.13","not packaged"
are handled well by the related pattern. I must make some very basic mistake, but I don't see which. Any ideas?