Logstash elasticsearch ruby filter issues

[raged]

Good morning all,

I am attempting to join some data together from a different index in logstash.

I have the following configuration file:

input {
	beats {
		port => 5044
	}
}

filter {
	elasticsearch {
		hosts => ["https://SERVER:9200"]
		user => USER
		password => PASSWORD
		index => 'wmibeat-*'
		query => 'wmi.Win32_Computersystem.Model :* AND beat.hostname : %{[beat][hostname]}'
		fields => { "wmi.Win32_Computersystem.Model" => Model }
			ruby {
				code => 'event.set("model", event.get("Model"))'
			}
	}
}

output {
	elasticsearch {
		hosts => ["https://SERVER:9200","https://SERVER:9200"]
		truststore => "c:\cacerts"
		truststore_password => "PASSWORD"
		index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
		user => USER
		password => PASSWORD
	}
}

If I remove the ruby code the logstash server starts up, but right now I am getting the following error:

[2019-05-06T12:00:41,933][ERROR][logstash.agent ] Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, => at line 15, column 9 (byte 353) after filter {\n\telasticsearch {\n\t\thosts => ["https://SERVER:9200"]\n\t\tuser => USER\n\t\tpassword => PASSWORD\n\t\tindex => 'wmibeat-'\n\t\tquery => 'wmi.Win32_Computersystem.Model : AND beat.hostname : %{[beat][hostname]}'\n\t\tfields => { "wmi.Win32_Computersystem.Model" => Model }\n\t\t\truby ", :backtrace=>["C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/compiler.rb:41:in compile_imperative'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/compiler.rb:49:in compile_graph'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/compiler.rb:11:in block in compile_sources'", "org/jruby/RubyArray.java:2486:in map'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/compiler.rb:10:in compile_sources'", "org/logstash/execution/AbstractPipelineExt.java:149:in initialize'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/pipeline.rb:22:in initialize'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/pipeline.rb:90:in initialize'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/pipeline_action/create.rb:38:in execute'", "C:/DLs/logstash-6.4.3/logstash-core/lib/logstash/agent.rb:309:in block in converge_state'"]}

JSON of a WMIBEAT event:

{
  "_index": "wmibeat-%{[@metadata][version]}-2019.04.26",
  "_type": "doc",
  "_id": "lbSXWW8B5HXUhqNmq8vW",
  "_version": 1,
  "_score": null,
  "_source": {
    "beat": {
      "name": "PCNAME",
      "hostname": "PCNAME"
    },
    "wmi": {
      "Win32_quickfixengineering": [
        {
          "InstalledOn": "4/12/2019",
          "Description": "Update",
          "HotFixID": "KB4489192",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/22/2019",
          "Description": "Update",
          "HotFixID": "KB4480056",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/12/2019",
          "Description": "Security Update",
          "HotFixID": "KB4493478",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/12/2019",
          "Description": "Security Update",
          "HotFixID": "KB4493510",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/12/2019",
          "Description": "Security Update",
          "HotFixID": "KB4493509",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        }
      ],
      "Win32_Computersystem": [
        {
          "SystemType": "x64-based PC",
          "Model": "HP EliteBook 840 G5",
          "NumberOfLogicalProcessors": 8,
          "SystemSKUNumber": "LOL",
          "TotalPhysicalMemory": "17019641856",
          "Manufacturer": "HP",
          "NumberOfProcessors": 1,
          "Domain": "domain.com"
        }
      ],
      "Win32_OperatingSystem": [
        {
          "BuildNumber": "17763",
          "LastBootUpTime": "20190426071117.500000-300",
          "LocalDateTime": "20190426072111.597000-300",
          "Caption": "Microsoft Windows 10 Enterprise",
          "InstallDate": "20190411113107.000000-300"
        }
      ]
    },
    "@timestamp": "2019-04-26T12:21:11.983Z",
    "@version": "1",
    "type": "wmibeat",
    "host": "PCNAME",
    "tags": [
      "beats_input_raw_event"
    ]
  },
  "fields": {
    "@timestamp": [
      "2019-04-26T12:21:11.983Z"
    ]
  },
  "highlight": {
    "beat.hostname": [
      "@kibana-highlighted-field@PCNAME@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1556281271983
  ]
}

Here is the JSON event I want to append the model from the previous event:

{
  "_index": "winlogbeat-6.4.3-2019.05.06",
  "_type": "doc",
  "_id": "6p6VjWoB5HXUhqNmJ2T9",
  "_version": 1,
  "_score": null,
  "_source": {
    "host": {
      "name": "PCNAME"
    },
    "event_id": 62441,
    "process_id": 9980,
    "source_name": "Microsoft-Windows-Shell-Core",
    "computer_name": "PCNAME.FQDN.COM",
    "@version": "1",
    "provider_guid": "{30336ed4-e327-447c-9de0-51b652c86108}",
    "thread_id": 16588,
    "record_number": "1843",
    "model": null,
    "level": "Information",
    "event_data": {
      "ProgId": "AppXd4nrz8ff68srnhf9t5a8sbjyar1cr723",
      "ExtOrUriScheme": ".pdf"
    },
    "user": {
      "type": "User",
      "domain": "DOMAIN",
      "identifier": "SID",
      "name": "ID"
    },
    "opcode": "Info",
    "beat": {
      "hostname": "PCNAME",
      "version": "6.4.3",
      "name": "PCNAME"
    },
    "message": "User choice has been reset to prog id AppXd4nrz8ff68srnhf9t5a8sbjyar1cr723 for .pdf",
    "log_name": "Microsoft-Windows-Shell-Core/AppDefaults",
    "type": "wineventlog",
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "@timestamp": "2019-05-06T14:38:40.183Z"
  },
  "fields": {
    "@timestamp": [
      "2019-05-06T14:38:40.183Z"
    ]
  },
  "highlight": {
    "beat.hostname": [
      "@kibana-highlighted-field@PCNAME@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1557153520183
  ]
}

I feel like at this point, I have to be close.. can someone point me in the right direction please?

Thanks.

[Badger]

You cannot have a filter inside another filter.

elasticsearch {
	hosts => ["https://SERVER:9200"]
            [...]
	fields => { "wmi.Win32_Computersystem.Model" => Model }
		ruby {
			code => 'event.set("model", event.get("Model"))'
		}
}

should be

elasticsearch {
	hosts => ["https://SERVER:9200"]
            [...]
	fields => { "wmi.Win32_Computersystem.Model" => Model }
}
ruby {
	code => 'event.set("model", event.get("Model"))'
}

Personally I would use a mutate filter rather than ruby for that. Either mutate+copy or mutate+rename

[raged]

Thanks for the reply, do you have an example of using elasticsearch to query a previous event and copy that data to the event that is about to be sent via logstash using mutate?

[Christian_Dahlqvist]

How much is expected between related events? It can take a few seconds for events to become searchable. This is driven by the refresh interval.

[raged]

I have an event that I am trying to correlate that was generated April 26th 2019, 07:21:11.983 in our wmibeat* index.

This is my configuration file. Moving the ruby code outside of the elasticsearch filter seemed to have allowed logstash to startup and send events successfully, however both Model and model are blank with '-'.

Is there an easier way to troubleshoot why it can't find the data rather than trying to manipulate the elasticsearch query inside the config file?

input {
	beats {
		port => 5044
	}
}

filter {
	elasticsearch {
		hosts => ["https://SERVER:9200"]
		user => USER
		password => PASSWORD
		index => 'wmibeat-*'
		query => 'wmi.Win32_Computersystem.Model :* AND beat.hostname : %{[beat][hostname]}'
		fields => { "wmi.Win32_Computersystem.Model" => Model }
	}
		ruby {
			code => 'event.set("model", event.get("Model"))'
		}
}

output {
	elasticsearch {
		hosts => ["https://SERVER:9200","https://SERVER:9200"]
		truststore => "c:\dls\cacerts"
		truststore_password => "PASSWORD"
		index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
		user => USER
		password => PASSWORD
	}
}

Here is a JSON of our winlogbeat index event:

{
  "_index": "winlogbeat-6.4.3-2019.05.07",
  "_type": "doc",
  "_id": "p3GokmoB5HXUhmNmT9Ib",
  "_version": 1,
  "_score": null,
  "_source": {
    "@version": "1",
    "@timestamp": "2019-05-07T14:17:41.579Z",
    "thread_id": 27152,
    "host": {
      "name": "PCNAME"
    },
    "log_name": "System",
    "event_data": {
      "NumberOfGroupPolicyObjects": "13",
      "SupportInfo2": "4213",
      "ProcessingTimeInMilliseconds": "15891",
      "ProcessingMode": "0",
      "DCName": "\\\\DOMAIN.DOMAIN",
      "SupportInfo1": "1"
    },
    "source_name": "Microsoft-Windows-GroupPolicy",
    "user": {
      "type": "User",
      "domain": "SHOESD01",
      "name": "072863",
      "identifier": "S-1-5-21-2019431095-1834360568-1243820751-73444"
    },
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "process_id": 26588,
    "record_number": "8033",
    "provider_guid": "{aea1b4fa-97d1-45f2-a64c-4d69fffd92c9}",
    "computer_name": "PCNAME.DOMAIN",
    "type": "wineventlog",
    "activity_id": "{97b44030-3e62-4e5a-b9c3-1da33f5d0987}",
    "message": "The Group Policy settings for the user were processed successfully. New settings from 13 Group Policy objects were detected and applied.",
    "opcode": "Start",
    "model": null,
    "Model": null,
    "event_id": 1503,
    "level": "Information",
    "beat": {
      "hostname": "PCNAME",
      "name": "PCNAME",
      "version": "6.4.3"
    }
  },
  "fields": {
    "@timestamp": [
      "2019-05-07T14:17:41.579Z"
    ]
  },
  "highlight": {
    "beat.hostname": [
      "@kibana-highlighted-field@PCNAME@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1557238661579
  ]
}

Here is the JSON event from our wmibeat index I am trying to correlate:

{
  "_index": "wmibeat-%{[@metadata][version]}-2019.04.26",
  "_type": "doc",
  "_id": "lbSXWWoB5HXUhmNmq8vW",
  "_version": 1,
  "_score": null,
  "_source": {
    "beat": {
      "name": "PCNAME",
      "hostname": "PCNAME"
    },
    "wmi": {
      "Win32_quickfixengineering": [
        {
          "InstalledOn": "4/12/2019",
          "Description": "Update",
          "HotFixID": "KB4489192",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/22/2019",
          "Description": "Update",
          "HotFixID": "KB4480056",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/12/2019",
          "Description": "Security Update",
          "HotFixID": "KB4493478",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/12/2019",
          "Description": "Security Update",
          "HotFixID": "KB4493510",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        },
        {
          "InstalledOn": "4/12/2019",
          "Description": "Security Update",
          "HotFixID": "KB4493509",
          "InstalledBy": "NT AUTHORITY\\SYSTEM"
        }
      ],
      "Win32_Computersystem": [
        {
          "SystemType": "x64-based PC",
          "Model": "HP EliteBook 840 G5",
          "NumberOfLogicalProcessors": 8,
          "SystemSKUNumber": "LOL",
          "TotalPhysicalMemory": "17019641856",
          "Manufacturer": "HP",
          "NumberOfProcessors": 1,
          "Domain": "DOMAIN"
        }
      ],
      "Win32_OperatingSystem": [
        {
          "BuildNumber": "17763",
          "LastBootUpTime": "20190426071117.500000-300",
          "LocalDateTime": "20190426072111.597000-300",
          "Caption": "Microsoft Windows 10 Enterprise",
          "InstallDate": "20190411113107.000000-300"
        }
      ]
    },
    "@timestamp": "2019-04-26T12:21:11.983Z",
    "@version": "1",
    "type": "wmibeat",
    "host": "PCNAME",
    "tags": [
      "beats_input_raw_event"
    ]
  },
  "fields": {
    "@timestamp": [
      "2019-04-26T12:21:11.983Z"
    ]
  },
  "highlight": {
    "beat.hostname": [
      "@kibana-highlighted-field@PCNAME@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1556281271983
  ]
}

[raged]

OK, I am getting closer, I am able to reference an event from a different filter, but I am having a hard time referencing the object "Model" that is nested inside of the wmi.Win32_Computersystem field.

So far I have tried:

fields => { "[wmi.Win32_Computersystem][Model]" => zz }

fields => { "wmi.Win32_Computersystem.Model" => zz }

fields => { "wmi.Win32_Computersystem" => zz }

In all attempts, I just get a blank property for zz and model in my winlogbeat index.

I know it's correctly referencing the event, because if I change the fields property in the logstash configuration file to this:

fields => { "type" => zz }

I see 'wmibeat' in the event that is written to elasticsearch in the winlogbeat index.

Does anyone have any idea how to reference this Model property correctly?

  "Win32_Computersystem": [
    {
      "SystemType": "x64-based PC",
      "Model": "HP EliteBook 840 G5",
      "NumberOfLogicalProcessors": 8,
      "SystemSKUNumber": "LOL",
      "TotalPhysicalMemory": "17019641856",
      "Manufacturer": "HP",
      "NumberOfProcessors": 1,
      "Domain": "DOMAIN.com"
    }
  ],

[Badger]

Win32_Computersystem is an array! Try "[wmi][Win32_Computersystem][0][Model]"

[raged]

I tried this as well:

fields => { "[wmi][Win32_Computersystem][0][Model]" => zz }

But I end up with this event:

"model": null,

I have been digging into this and I found this post (I am not sure how to mutate like he mentioned, I did message him ) :

Getting Nested Fields Elasticsearch Filter

I using the logstash-elasticsearch-filter to correlate events from Elasticsearch. Some of the fields that I need to return from the Elasticsearch hit are nested. I have been unable to retrieve these docs and the result is always null. I am using Logstash 5.6.5 and ES 5.6.5. Here is my configuration: "fields" => { "field.otherField.anotherField" => "p_time_gmt" } I have also tried: [field][otherField][anotherField] [field][otherField][0][anotherField] …

Is there a limitation for logstash using a nested array property in a elasticsearch field property?

We are currently using version 6.4.3 of logstash.

[Badger]

OK, having looked at the code, I don't think an array reference is going to work. Can you try

fields => { "[wmi][Win32_Computersystem]" => "Computersystem" }

and then extract the Model from that?

[raged]

@Badger

fields => { "[wmi][Win32_Computersystem]" => "Computersystem" }

Progress:

I see a computersystem property as the nested array on the event in my winlogbeat index:

"Computersystem": [
  {
    "SystemSKUNumber": "LOL",
    "NumberOfProcessors": 1,
    "TotalPhysicalMemory": "17019641856",
    "NumberOfLogicalProcessors": 8,
    "Manufacturer": "HP",
    "Domain": "DOMAIN.com",
    "SystemType": "x64-based PC",
    "Model": "HP EliteBook 840 G5"
  }

Model property:

"model": null,

Here is my current elasticsearch filter:

filter {
	elasticsearch {
		hosts => ["https://SERVER:9200","https://SERVER:9200"]
		user => 
		password => 	
		index => 'wmibeat-*'
		query => 'beat.hostname : %{[beat][hostname]}'
		fields => { "[wmi][Win32_Computersystem]" => "Computersystem" }
	}
		ruby {
			code => "event.set('model', event.get('[Computersystem][Model]'))"
		}
}

What is the best way to parse the event to only stash the model property in the computersystem property?

EDIT:

I got it:

		ruby {
			code => "event.set('model', event.get('[Computersystem][0][Model]'))"
		}

Thank you all for your help.

[Badger]

Alternatively, you could use a mutate instead of ruby

mutate { copy => { "[Computersystem][0][Model]" => "model" } }