Check event format against template

Elastic Stack Logstash
1

Hi,

is it possible to check events format against certain template in Logstash?

  1. For example I want to make sure that event contains certain mandatory fields, i.e. "environment", "application" etc.
  2. For example I want to make sure that event contains certain mandatory fields with certain values, i.e. "environment": "production", "application": "application_name"
2

You can use IFs with or without regex. Also is possible to drop field or all event. It depends what do you want.

input {
      generator {
	message => ['Test' ]
        count => 1
      }
}
output {
 stdout {codec => rubydebug{ metadata => true}}
}
filter {
	mutate {
        add_field => {
          "environment" => "test"
          "application" => "SAP"
        }
	}
	# does field exist or not null
	if [environment] {
		mutate { add_field => { "info-exist" => "Check does the field environment exist" } }
	}
	# if environment is not OK, drop all event
    if [environment] == "prod" {
        drop { }
    }
	# strictly exact value
	if [environment]== "test" {
		mutate { add_field => { "info-test" => "This is %{environment} env" } }
	}
	# multiple case-insensitive values check
	if [environment] =~ /(?i)(prod|test|staging|dev)/  {
		mutate { add_field => { "info-env" => "The environment: %{environment}" } }	
	}
    # drop field if is not in approved list of values
	prune {
		whitelist_values => [ "application","(SAP|ORACLE)" , 
		"environment", "(?i)TEST"]
	}
}

Output:

{
    "environment" => "test",
     "@timestamp" => 2025-03-27T13:08:56.311959Z,
    "application" => "SAP",
       "info-env" => "The environment: test",
      "info-test" => "This is test env",
     "info-exist" => "Check does the field environment exist"
}
1 Like
3

@Rios thank you for your response.

Yes, if is the first obvious solution that comes to mind.
I would be interested in if it is possible to save this template in a separate file and somehow compare them.

4

Is it a long list? If is 10-20, I wouldn't complicate it.

1 Like
5

Another idea. You can use @metadata field to store your temporary lists in the runtime. The list will not end up in ES.

	mutate {
        add_field => {
          "[@metadata][app]" => ["Oracle", "SAP", "MySQL", "SQL Server"]
          "[@metadata][env]" => ["test", "prod", "staging"]
        }
	}
	# check does value exist - case sensitive
	if "SQL Server" in [@metadata][app] {
	mutate { add_field => { "infoapp" => "App exist" } }	
	}
	# check does field value exist in list
	if [environment] in [@metadata][env] {
	mutate { add_field => { "infoenv" => "Env exist" } }	
	}

Add metadata => true in the debug mode to see current values.

output {
 stdout {codec => rubydebug{ metadata => true}}
}

Output

{
    "environment" => "test"
      "@metadata" => {
        "app" => [
            [0] "Oracle",
            [1] "SAP",
            [2] "MySQL",
            [3] "SQL Server"
        ],
        "env" => [
            [0] "test",
            [1] "prod",
            [2] "staging"
        ]
    },
        "infoapp" => "App exist",
        "infoenv" => "Env exist",
}
1 Like