Logstash--Exception caught in json filter

Elastic Stack Logstash
1

Logstash--Exception caught in json filter

Below is my json log string:

{
	"traceId": "o0uyveRU/8BkjL+lbpDlnQ==",
	"spanId": "73rmqIRmo34=",
	"operationName": "ProcessWorkItem",
	"startTime": "2024-07-22T06:07:12.650881409Z",
	"duration": "0.256620153s",
	"tags": [
		{
			"key": "otel.library.name",
			"vStr": "com.mdi.core.workmgmt.TypedConsumer"
		},
		{
			"key": "ta.work.type",
			"vStr": "INBOUND_ENTITY_INTEGRATION"
		},
		{
			"key": "ta.companyCode",
			"vStr": "AMBER_ROAD"
		},
		{
			"key": "ta.work.status",
			"vStr": "COMPLETED"
		},
		{
			"key": "ta.work.loadDuration",
			"vType": "INT64"
		},
		{
			"key": "service.version",
			"vStr": "24.2"
		},
		{
			"key": "work.processorclass",
			"vStr": "com.mdi.core.entity.integration.EntityStdIntegrationWorkProcessor"
		},
		{
			"key": "ta.work.receivedTime",
			"vType": "INT64",
			"vInt64": "1721628429423"
		},
		{
			"key": "ta.work.delegateDuration",
			"vType": "INT64",
			"vInt64": "257"
		},
		{
			"key": "ta.work.busKeySearch",
			"vStr": "AMBER_ROAD;SO_TEST_93;SALES_ORDER"
		},
		{
			"key": "ta.orgCode",
			"vStr": "AMBER_ROAD"
		},
		{
			"key": "ta.work.id",
			"vStr": "33267221"
		},
		{
			"key": "length",
			"vType": "INT64",
			"vInt64": "3680"
		},
		{
			"key": "ta.work.reportStatusDuration",
			"vType": "INT64",
			"vInt64": "4"
		},
		{
			"key": "ta.work.errorCount",
			"vType": "INT64"
		},
		{
			"key": "ta.work.delegateConsumerTime",
			"vType": "INT64",
			"vInt64": "253"
		},
		{
			"key": "span.kind",
			"vStr": "internal"
		},
		{
			"key": "otel.status_code",
			"vStr": "OK"
		},
		{
			"key": "internal.span.format",
			"vStr": "otlp"
		}
	],
	"process": {
		"serviceName": "TRADE_SINGLETON",
		"tags": [
			{
				"key": "deployment.environment",
				"vStr": "PSR"
			},
			{
				"key": "e2.environment.name",
				"vStr": "OTEL_PST"
			},
			{
				"key": "e2.instance.name",
				"vStr": "TRADE_SINGLETON_1"
			},
			{
				"key": "e2.product.family",
				"vStr": "GTM"
			},
			{
				"key": "e2.product.tenant.id",
				"vStr": "WMMERCURYTA"
			},
			{
				"key": "service.instance.id",
				"vStr": "URN:PSR:GTM:TA:OTEL_PST:TRADE_SINGLETON:TRADE_SINGLETON_1"
			},
			{
				"key": "service.namespace",
				"vStr": "TA"
			},
			{
				"key": "telemetry.sdk.language",
				"vStr": "java"
			},
			{
				"key": "telemetry.sdk.name",
				"vStr": "opentelemetry"
			},
			{
				"key": "telemetry.sdk.version",
				"vStr": "1.34.0"
			}
		]
	},
	"tag": "jaeger_spans"
}

below is the filter i am using:


filter 
{ 
	json 
	{
		source => "message"
	}
		
	json 
	{
		source => "message"
		target => "tmessage"
	}
	
	mutate {
			copy => 
			{
				"startTime" => "startTimeMillis"
			}
		}
	
	mutate
	{
		add_field => {"process.serviceName" => "%{[tmessage][process][serviceName]}"}
	}
	
	ruby {
		code => '
			event.get("[tmessage][tags]").each { |a|
				name = a["key"]
				name=name.gsub(/\./,"@")
				if a["vStr"]
					value = a["vStr"]
				elsif a["vInt64"] 
					value = a["vInt64"]
				else
					value = ""
				end
				event.set("tag." + name, value)
				}
			'
		}
				
	ruby {
		code => '
			event.get("[tmessage][process][tags]").each { |a|
				name = a["key"]
				name=name.gsub(/\./,"@")
				if a["vStr"]
					value = a["vStr"]
				elsif a["vInt64"]
					value = a["vInt64"]
				else
					value = ""
				end
				event.set("process.tag." + name, value)
			}
		'
		}
		
	mutate {
		remove_field => [ "message", "tmessage","tags"]
	}
}

Below is the error log:




|[2024-07-31T20:54:51,638][WARN ][logstash.filters.json    ][main][589f2d72302e5b707740cbeea78dd46b6dfb74d1739471d9b1e2882a38773fdd] Exception caught in json filter {:source=>"message", :raw=>"{\"traceId\":\"o0uyveRU/8BkjL+lbpDlnQ==\",\"spanId\":\"73rmqIRmo34=\",\"operationName\":\"ProcessWorkItem\",\"startTime\":\"2024-07-22T06:07:12.650881409Z\",\"duration\":\"0.256620153s\",\"tags\":[{\"key\":\"otel.library.name\",\"vStr\":\"com.mdi.core.workmgmt.TypedConsumer\"},{\"key\":\"ta.work.type\",\"vStr\":\"INBOUND_ENTITY_INTEGRATION\"},{\"key\":\"ta.companyCode\",\"vStr\":\"AMBER_ROAD\"},{\"key\":\"ta.work.status\",\"vStr\":\"COMPLETED\"},{\"key\":\"ta.work.loadDuration\",\"vType\":\"INT64\"},{\"key\":\"service.version\",\"vStr\":\"24.2\"},{\"key\":\"work.processorclass\",\"vStr\":\"com.mdi.core.entity.integration.EntityStdIntegrationWorkProcessor\"},{\"key\":\"ta.work.receivedTime\",\"vType\":\"INT64\",\"vInt64\":\"1721628429423\"},{\"key\":\"ta.work.delegateDuration\",\"vType\":\"INT64\",\"vInt64\":\"257\"},{\"key\":\"ta.work.busKeySearch\",\"vStr\":\"AMBER_ROAD;SO_TEST_93;SALES_ORDER\"},{\"key\":\"ta.orgCode\",\"vStr\":\"AMBER_ROAD\"},{\"key\":\"ta.work.id\",\"vStr\":\"33267221\"},{\"key\":\"length\",\"vType\":\"INT64\",\"vInt64\":\"3680\"},{\"key\":\"ta.work.reportStatusDuration\",\"vType\":\"INT64\",\"vInt64\":\"4\"},{\"key\":\"ta.work.errorCount\",\"vType\":\"INT64\"},{\"key\":\"ta.work.delegateConsumerTime\",\"vType\":\"INT64\",\"vInt64\":\"253\"},{\"key\":\"span.kind\",\"vStr\":\"internal\"},{\"key\":\"otel.status_code\",\"vStr\":\"OK\"},{\"key\":\"internal.span.format\",\"vStr\":\"otlp\"}],\"process\":{\"serviceName\":\"TRADE_SINGLETON\",\"tags\":[{\"key\":\"deployment.environment\",\"vStr\":\"PSR\"},{\"key\":\"e2.environment.name\",\"vStr\":\"OTEL_PST\"},{\"key\":\"e2.instance.name\",\"vStr\":\"TRADE_SINGLETON_1\"},{\"key\":\"e2.product.family\",\"vStr\":\"GTM\"},{\"key\":\"e2.product.tenant.id\",\"vStr\":\"WMMERCURYTA\"},{\"key\":\"service.instance.id\",\"vStr\":\"URN:PSR:GTM:TA:OTEL_PST:TRADE_SINGLETON:TRADE_SINGLETON_1\"},{\"key\":\"service.namespace\",\"vStr\":\"TA\"},{\"key\":\"telemetry.sdk.language\",\"vStr\":\"java\"},{\"key\":\"telemetry.sdk.name\",\"vStr\":\"opentelemetry\"},{\"key\":\"telemetry.sdk.version\",\"vStr\":\"1.34.0\"}]},\"tag\":\"jaeger_spans\"}", :exception=>#<Java::OrgLogstash::Event::InvalidTagsTypeException: Could not set the reserved tags field '[tags]' to value 'ConvertedList{delegate=[{key=otel.library.name, vStr=com.mdi.core.workmgmt.TypedConsumer}, {key=ta.work.type, vStr=INBOUND_ENTITY_INTEGRATION}, {key=ta.companyCode, vStr=AMBER_ROAD}, {key=ta.work.status, vStr=COMPLETED}, {key=ta.work.loadDuration, vType=INT64}, {key=service.version, vStr=24.2}, {key=work.processorclass, vStr=com.mdi.core.entity.integration.EntityStdIntegrationWorkProcessor}, {vType=INT64, key=ta.work.receivedTime, vInt64=1721628429423}, {vType=INT64, key=ta.work.delegateDuration, vInt64=257}, {key=ta.work.busKeySearch, vStr=AMBER_ROAD;SO_TEST_93;SALES_ORDER}, {key=ta.orgCode, vStr=AMBER_ROAD}, {key=ta.work.id, vStr=33267221}, {vType=INT64, key=length, vInt64=3680}, {vType=INT64, key=ta.work.reportStatusDuration, vInt64=4}, {key=ta.work.errorCount, vType=INT64}, {vType=INT64, key=ta.work.delegateConsumerTime, vInt64=253}, {key=span.kind, vStr=internal}, {key=otel.status_code, vStr=OK}, {key=internal.span.format, vStr=otlp}]}'. The tags field only accepts string or array of string.>}

Pls let me know where is the issue?

2

The exception you are getting is

:exception=>#<Java::OrgLogstash::Event::InvalidTagsTypeException: Could not set the reserved tags field '[tags]' to value ''. The tags field only accepts string or array of string.>}

The [tags] field, like [@timestamp] is special. logstash knows [tags] is an array of strings, so you can only set it to a string (which will become an array) or array of strings (which will be merged).

So where are you setting the [tags] field? That's happening in the json filter (which you know from the error message). Your JSON has a top level field called "tags" which is an array of hashes. logstash cannot handle that.

If you run

input { generator { count => 1 lines => [
    '{ "tags": [ { "a": "b" } ] }',
    '{ "notTags": [ { "a": "b" } ] }'
] } }
output { stdout { codec => rubydebug { metadata => false } } }
filter { json { source => "message" remove_field => [ "message" ] } }

then the first event will get that exception but the second one will not. In this the error message will include

Could not set the reserved tags field '[tags]' to value 'ConvertedList{delegate=[{a=b}]}'.

That's trying to show you the JSON that is causing the exception.

You have two json filters that parse [message]. The first has no target option, so it gets the exception. The second one runs with no issues. I suggest you remove the first one.

1 Like
3

Hello Badger,

Once i removed first json filter that parse[message], field [tags] error is not coming now. But 1st level json fields are also not coming in like traceId,spanId,duration etc.. Pls let me know

4

What is your input? It is not clear why you have 2 json filters and both using message as the source.

5

Jaegertracing collector send to kafka and we are pulling from kafka

input { 
 kafka
	{
		bootstrap_servers => "xxxxxx:9092"
		topics => "jaeger_spans"
	}
}

As mentioned in before msg, that is the json log received from kafka.
[tmessage]- i am using to get nested fields using ruby as mentioned in above filter.
[message]-there are few 1st level json fields which need to be sent to ES, so formatting that.

6

You could use mutate+add_field to copy fields from [tmessage] to the top-level, but [tags] cannot be one of them.

7

mutate_Add_Field like below can be done. But those 1st level fields are dynamic, how do i get all those fields without knowing the field names?

mutate {
        add_field => { "traceid" => "%{[tmessage][traceId]}" }
      }
8

You could try something like

ruby {
    code => '
        event.get("tmessage").each { |k, v|
            unless k == "tags"
                event.set(k,v)
            end
        }
    '
}
1 Like
9

Yep, working Badger..Thanks

10

I just found out that logstash.yml has a related setting (event_api.tags.illegal) which allows you to change what logstash does when being asked to set [tags] to a disallowed value.