Hi everyone,
I recently upgraded ELK from 7 to 8 and it was working fine for v7, but since v8, it continuously gave me the below error.
Jun 07 16:48:00 gitlab-logger logstash[115245]: [2023-06-07T16:48:00,346][WARN ][logstash.outputs.elasticsearch][main][94c07859ebdacdc56f7fbbe6f7962112cfbd4fffcc1fcba215dfa2b3829e1b84] Could not index event to Elasticsearch. status: 400, action: ["index", {:_id=>nil, :_index=>"logstash", :routing=>nil}, {"params"=>[{"key"=>"info", "value"=>{"name"=>"gitlab-runner", "executor"=>"docker", "config"=>{"gpus"=>""}, "architecture"=>"amd64", "features"=>{"trace_size"=>"[FILTERED]", "raw_variables"=>"[FILTERED]", "service_variables"=>"[FILTERED]", "trace_checksum"=>"[FILTERED]", "vault_secrets"=>"[FILTERED]", "variables"=>"[FILTERED]", "trace_reset"=>"[FILTERED]"}, "version"=>"15.9.0", "platform"=>"linux", "revision"=>"c20f0bec", "shell"=>"bash"}}, {"key"=>"token", "value"=>"[FILTERED]"}, {"key"=>"system_id", "value"=>"s_812a7cd4a197"}], "redis_cache_calls"=>6, "ecs"=>{"version"=>"1.6.0"}, "db_replica_wal_cached_count"=>0, "db_main_wal_count"=>0, "db_main_replica_wal_count"=>0, "@timestamp"=>2023-06-07T06:47:50.134Z, "redis_calls"=>9, "meta.client_id"=>"ip/111.111.111.111", "db_write_count"=>0, "db_primary_wal_count"=>0, "db_main_replica_count"=>0, "input"=>{"type"=>"log"}, "remote_ip"=>"111.111.111.111, 111.111.111.111", "redis_shared_state_duration_s"=>0.002101, "db_main_duration_s"=>0.006, "db_primary_duration_s"=>0.006, "severity"=>"INFO", "rate_limiting_gates"=>[], "redis_shared_state_calls"=>3, "redis_read_bytes"=>1018, "method"=>"POST", "db_main_count"=>4, "db_main_wal_cached_count"=>0, "correlation_id"=>"01H2A9S5JHBN7271MQYDNVDVM5", "mem_total_bytes"=>1826514, "meta.remote_ip"=>"111.111.111.111", "cpu_s"=>0.027277, "duration_s"=>0.0079, "log_id"=>"api_json", "db_replica_wal_count"=>0, "route"=>"/api/:version/jobs/request", "agent"=>{"name"=>"gitlab1", "version"=>"7.10.2", "id"=>"024bdd0d-1ecd-4476-921a-9ed6b00ef763", "hostname"=>"gitlab1", "type"=>"filebeat", "ephemeral_id"=>"084d15db-4775-4fe0-8b8f-7e496f9cafb9"}, "log_group"=>"gitlab_teach", "db_main_replica_duration_s"=>0, "db_primary_count"=>4, "redis_shared_state_read_bytes"=>2, "redis_shared_state_write_bytes"=>258, "pid"=>2032, "target_duration_s"=>5, "tags"=>["beats_input_raw_event"], "redis_duration_s"=>0.005738, "redis_cache_read_bytes"=>1016, "worker_id"=>"puma_3", "content_length"=>"723", "db_count"=>4, "host"=>{"name"=>"gitlab1"}, "ua"=>"gitlab-runner 15.9.0 (15-9-stable; go1.18.10; linux/amd64)", "db_main_replica_cached_count"=>0, "redis_cache_write_bytes"=>419, "mem_bytes"=>1450114, "view_duration_s"=>0.00594, "db_main_cached_count"=>0, "db_duration_s"=>0.00196, "meta.feature_category"=>"continuous_integration", "db_primary_cached_count"=>0, "db_primary_wal_cached_count"=>0, "status"=>403, "path"=>"/api/v4/jobs/request", "request_urgency"=>"low", "redis_cache_duration_s"=>0.003637, "db_cached_count"=>0, "db_replica_cached_count"=>0, "log"=>{"offset"=>10121701, "file"=>{"path"=>"/var/log/gitlab/gitlab-rails/api_json.log"}}, "db_main_replica_wal_cached_count"=>0, "redis_write_bytes"=>677, "mem_mallocs"=>2956, "mem_objects"=>9410, "time"=>"2023-06-07T06:47:50.134Z", "@version"=>"1", "db_replica_duration_s"=>0, "meta.caller_id"=>"POST /api/:version/jobs/request", "db_replica_count"=>0}], response: {"index"=>{"_index"=>"logstash", "_id"=>"CpSclIgBPfuJNBJMvvNY", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [params.value] of type [text] in document with id 'CpSclIgBPfuJNBJMvvNY'. Preview of field's value: '{features={service_variables=[FILTERED], vault_secrets=[FILTERED], variables=[FILTERED], trace_reset=[FILTERED], trace_size=[FILTERED], raw_variables=[FILTERED], trace_checksum=[FILTERED]}, shell=bash, executor=docker, name=gitlab-runner, config={gpus=}, version=15.9.0, platform=linux, architecture=amd64, revision=c20f0bec}'", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:34"}}}}
It seems to me that some of the param.value in the log is text and some are json and logstash can't treat that json object into text?
How can I resolve this error? I think it should fine to treat json object to be text or string. Is there an easy setting in logstash that will allow json and text under the same key at the same time? If anyone can help me look at this issue I would be very appreciate it. 