JSON Parsing issue with elasticsearch ingest pipeline

Elastic Stack Elastic Agent
1

Hello,

I am building an Elasticsearch cluster to aggregate and monitor application logs. I am using ECK for deploying and managing the cluster in k8s and fleet-managed elastic agent deployed across multiple clusters to push logs to Elasticsearch.

ES version is 8.9

Applications write logs as JSON and I am trying to parse the JSON log using elasticsearch ingest pipeline. But some logs are still dropped with the below error. I need some help to troubleshoot and fix this issue.

{
  "version": 3,
  "_meta": {
    "description": "automatic parsing of JSON log messages",
    "managed": true
  },
  "processors": [
    {
      "rename": {
        "if": "ctx.message instanceof String && ctx.message.startsWith('{') && ctx.message.endsWith('}')",
        "field": "message",
        "target_field": "_tmp_json_message",
        "ignore_missing": true
      }
    },
    {
      "json": {
        "if": "ctx._tmp_json_message != null",
        "field": "_tmp_json_message",
        "add_to_root": true,
        "add_to_root_conflict_strategy": "merge",
        "allow_duplicate_keys": true,
        "on_failure": [
          {
            "rename": {
              "field": "_tmp_json_message",
              "target_field": "message",
              "ignore_missing": true
            }
          }
        ]
      }
    },
    {
      "dot_expander": {
        "if": "ctx._tmp_json_message != null",
        "field": "*",
        "override": true
      }
    },
    {
      "remove": {
        "field": "_tmp_json_message",
        "ignore_missing": true
      }
    }
  ]
}

The agent is throwing the below error.

{\"type\":\"document_parsing_exception\",\"reason\":\"[1:5564] object mapping for [event] tried to parse field [event] as object, but found a concrete value\"}
2

This is a mapping error, this means that in some of your documents you have a field named event which is not an object. Since you are using elastic agent, it will use a pre-defined mapping (which you will not be able to change) where the event field is an object.

For example, your documents probably have something like this:

{ "event": "value" }

But this will not be accepted as the event field needs to be an object, what will be accepted is something like this:

{ "event": { "nestedfield": "value" }

You will need to change your json processor to use the target_field option and not add the fields to the root of the document to avoid conflicting your event field with the agent event field.

Which integration are you using? Can you share a sample of the document that give you this error?

3

Thank you for the message. I will try with target_field.

Kubernetes integration (v1.43.1).

sample document without JSON parsing

  {
    "_index": ".ds-logs-kubernetes.container_logs-teleport-auth-2023.09.07-000001",
    "_id": "QtLEb4oB0PDML7suQ-TV",
    "_version": 1,
    "_score": 0,
    "_source": {
      "container": {
        "image": {
          "name": "public.ecr.aws/gravitational/teleport-ent-distroless:13.2.1"
        },
        "runtime": "docker",
        "id": "74a61f8737aa44b7970f89b33520bb0182a23458406b0cea1d16b5e380b1b339"
      },
      "kubernetes": {
        "container": {
          "name": "teleport"
        },
        "node": {
          "uid": "23a9d407-bce0-43f9-bc96-57b1044fbf3d",
          "hostname": "ip-xx-xx-40-107.xxxx.compute.internal",
          "name": "ip-xx-xx-40-107.xxxx.compute.internal",
          "labels": {
            "kubernetes_io/hostname": "ip-xx-xx-xx-107.xxxx.compute.internal",
            "topology_kubernetes_io/zone": "xxxxa",
            "topology_kubernetes_io/region": "xxxx",
            "kubernetes_io/arch": "amd64",
            "topology_ebs_csi_aws_com/zone": "xxxxa",
            "failure-domain_beta_kubernetes_io/region": "xxxx",
            "k8s_io/cloud-provider-aws": "7c12cab0f30156fcbaf3c1e778ea0270",
            "beta_kubernetes_io/instance-type": "c5.2xlarge",
            "eks_amazonaws_com/nodegroup-image": "ami-010fb54ff2a0980d3",
            "eks_amazonaws_com/capacityType": "ON_DEMAND",
            "eks_amazonaws_com/nodegroup": "120220228060507396400000001",
            "failure-domain_beta_kubernetes_io/zone": "xxxxa",
            "node_kubernetes_io/instance-type": "c5.2xlarge",
            "beta_kubernetes_io/os": "linux",
            "Environment": "dev",
            "kubernetes_io/os": "linux",
            "beta_kubernetes_io/arch": "amd64"
          }
        },
        "pod": {
          "uid": "e5e97a2b-0a7f-46dd-977c-cb4efa900b3f",
          "ip": "xx.xx.40.xxx",
          "name": "teleport-auth-6dddf758bc-8hdlk"
        },
        "namespace": "teleport",
        "replicaset": {
          "name": "teleport-auth-6dddf758bc"
        },
        "namespace_uid": "9cc606e9-0755-4cf4-a70e-f7ee8aaceea2",
        "namespace_labels": {
          "kubernetes_io/metadata_name": "teleport",
          "name": "teleport"
        },
        "deployment": {
          "name": "teleport-auth"
        },
        "labels": {
          "app": "teleport",
          "app_kubernetes_io/managed-by": "Helm",
          "helm_sh/chart": "teleport-cluster-13.2.1",
          "pod-template-hash": "6dddf758bc",
          "app_kubernetes_io/version": "13.2.1",
          "app_kubernetes_io/name": "teleport-cluster",
          "app_kubernetes_io/component": "auth",
          "teleport_dev/majorVersion": "13",
          "app_kubernetes_io/instance": "teleport"
        }
      },
      "agent": {
        "name": "elastic-agent-thlzv",
        "id": "5085b08f-d42a-414c-ba5d-458649065e57",
        "type": "filebeat",
        "ephemeral_id": "2778ce61-a7d0-4590-86f3-6de50d5775bb",
        "version": "8.9.0"
      },
      "log": {
        "file": {
          "path": "/var/log/containers/teleport-auth-6dddf758bc-8hdlk_teleport_teleport-74a61f8737aa44b7970f89b33520bb0182a23458406b0cea1d16b5e380b1b339.log"
        },
        "offset": 9905203
      },
      "elastic_agent": {
        "id": "5085b08f-d42a-414c-ba5d-458649065e57",
        "version": "8.9.0",
        "snapshot": false
      },
      "message": "{\"addr.remote\":\"192.xxx.xxx.211:60712\",\"caller\":\"events/emitter.go:265\",\"cluster_name\":\"teleport.com\",\"code\":\"T3009I\",\"component\":\"audit\",\"ei\":0,\"event\":\"kube.request\",\"kubernetes_cluster\":\"dev\",\"level\":\"info\",\"login\":\"xxxx.xxx@xxx.com\",\"message\":\"kube.request\",\"namespace\":\"default\",\"proto\":\"kube\",\"request_path\":\"/apis/batch/v1/namespaces/xxx/jobs\",\"resource_api_group\":\"batch/v1\",\"resource_kind\":\"jobs\",\"resource_namespace\":\"xxx-xxx\",\"response_code\":200,\"server_id\":\"93598ded-262a-45ea-9027-07eb8b0dcbcf\",\"time\":\"2023-09-07T13:10:45.899Z\",\"timestamp\":\"2023-09-07T13:10:45Z\",\"uid\":\"c6f7d765-7adb-4c97-979a-9fb6c64ee08a\",\"user\":\"xxx.xxx@xxx.com\",\"verb\":\"GET\"}\n",
      "cloud": {
        "availability_zone": "xxxxa",
        "instance": {
          "name": "ip-xx-xx-40-107.xxxx.compute.internal",
          "id": "i-xxxxxx"
        },
        "provider": "openstack",
        "machine": {
          "type": "c5.2xlarge"
        },
        "service": {
          "name": "Nova"
        }
      },
      "input": {
        "type": "filestream"
      },
      "@timestamp": "2023-09-07T13:10:45.898Z",
      "ecs": {
        "version": "8.0.0"
      },
      "stream": "stderr",
      "data_stream": {
        "namespace": "infra",
        "type": "logs",
        "dataset": "kubernetes.container_logs"
      },
      "host": {
        "hostname": "elastic-agent-thlzv",
        "os": {
          "kernel": "5.4.247-162.350.amzn2.x86_64",
          "codename": "focal",
          "name": "Ubuntu",
          "family": "debian",
          "type": "linux",
          "version": "20.04.6 LTS (Focal Fossa)",
          "platform": "ubuntu"
        },
        "ip": [
          "xxx.xxx.xxx.189"
        ],
        "containerized": true,
        "name": "elastic-agent-thlzv",
        "id": "ec22fc100e08a79a137d1a3c279a741f",
        "mac": [
          "6A-73-32-E1-50-6C"
        ],
        "architecture": "x86_64"
      },
      "event": {
        "dataset": "kubernetes.container_logs"
      }
    },
    "fields": {
      "kubernetes.node.uid": [
        "23a9d407-bce0-43f9-bc96-57b1044fbf3d"
      ],
      "kubernetes.node.labels.Environment": [
        "infra"
      ],
      "elastic_agent.version": [
        "8.9.0"
      ],
      "kubernetes.namespace_uid": [
        "9cc606e9-0755-4cf4-a70e-f7ee8aaceea2"
      ],
      "kubernetes.deployment.name": [
        "teleport-auth"
      ],
      "host.os.name.text": [
        "Ubuntu"
      ],
      "kubernetes.node.labels.topology_kubernetes_io/zone": [
        "xxxxa"
      ],
      "elastic_agent.id.keyword": [
        "5085b08f-d42a-414c-ba5d-458649065e57"
      ],
      "event.dataset.keyword": [
        "kubernetes.container_logs"
      ],
      "host.hostname": [
        "elastic-agent-thlzv"
      ],
      "kubernetes.node.labels.kubernetes_io/os": [
        "linux"
      ],
      "host.mac": [
        "6A-xx-xx-E1-50-xx"
      ],
      "cloud.availability_zone": [
        "xxxxa"
      ],
      "container.id": [
        "74a61f8737aa44b7970f89b33520bb0182a23458406b0cea1d16b5e380b1b339"
      ],
      "kubernetes.labels.pod-template-hash": [
        "6dddf758bc"
      ],
      "kubernetes.labels.app_kubernetes_io/component": [
        "auth"
      ],
      "container.image.name": [
        "public.ecr.aws/gravitational/teleport-ent-distroless:13.2.1"
      ],
      "host.os.version": [
        "20.04.6 LTS (Focal Fossa)"
      ],
      "kubernetes.labels.app": [
        "teleport"
      ],
      "kubernetes.node.labels.beta_kubernetes_io/os": [
        "linux"
      ],
      "kubernetes.namespace": [
        "teleport"
      ],
      "host.os.name": [
        "Ubuntu"
      ],
      "kubernetes.node.labels.eks_amazonaws_com/nodegroup-image": [
        "ami-010fb54ff2a0980d3"
      ],
      "agent.name": [
        "elastic-agent-thlzv"
      ],
      "host.name": [
        "elastic-agent-thlzv"
      ],
      "kubernetes.labels.app_kubernetes_io/name": [
        "teleport-cluster"
      ],
      "kubernetes.node.labels.topology_kubernetes_io/region": [
        "xxxx"
      ],
      "kubernetes.labels.app_kubernetes_io/instance": [
        "teleport"
      ],
      "host.os.type": [
        "linux"
      ],
      "kubernetes.node.labels.failure-domain_beta_kubernetes_io/zone": [
        "xxxxa"
      ],
      "input.type": [
        "filestream"
      ],
      "cloud.service.name.keyword": [
        "Nova"
      ],
      "log.offset": [
        9905203
      ],
      "data_stream.type": [
        "logs"
      ],
      "host.architecture": [
        "x86_64"
      ],
      "cloud.machine.type": [
        "c5.2xlarge"
      ],
      "container.runtime": [
        "docker"
      ],
      "cloud.provider": [
        "openstack"
      ],
      "kubernetes.node.labels.eks_amazonaws_com/nodegroup": [
        "120220228060507396400000001"
      ],
      "agent.id": [
        "5085b08f-d42a-414c-ba5d-458649065e57"
      ],
      "cloud.service.name": [
        "Nova"
      ],
      "ecs.version": [
        "8.0.0"
      ],
      "host.containerized": [
        true
      ],
      "kubernetes.node.labels.beta_kubernetes_io/instance-type": [
        "c5.2xlarge"
      ],
      "kubernetes.labels.helm_sh/chart": [
        "teleport-cluster-13.2.1"
      ],
      "agent.version": [
        "8.9.0"
      ],
      "host.os.family": [
        "debian"
      ],
      "kubernetes.node.name": [
        "ip-xx-xx-40-107.xxxx.compute.internal"
      ],
      "stream.keyword": [
        "stderr"
      ],
      "kubernetes.node.labels.topology_ebs_csi_aws_com/zone": [
        "xxxxa"
      ],
      "kubernetes.node.labels.failure-domain_beta_kubernetes_io/region": [
        "xxxx"
      ],
      "kubernetes.node.hostname": [
        "ip-xx-xx-40-107.xxxx.compute.internal"
      ],
      "kubernetes.node.labels.node_kubernetes_io/instance-type": [
        "c5.2xlarge"
      ],
      "kubernetes.pod.uid": [
        "e5e97a2b-0a7f-46dd-977c-cb4efa900b3f"
      ],
      "elastic_agent.version.keyword": [
        "8.9.0"
      ],
      "cloud.instance.id": [
        "i-xxxxxxxxx"
      ],
      "host.ip": [
        "10.xx.xx.189"
      ],
      "agent.type": [
        "filebeat"
      ],
      "stream": [
        "stderr"
      ],
      "host.os.kernel": [
        "5.4.247-162.350.amzn2.x86_64"
      ],
      "kubernetes.pod.name": [
        "teleport-auth-6dddf758bc-8hdlk"
      ],
      "kubernetes.labels.app_kubernetes_io/version": [
        "13.2.1"
      ],
      "elastic_agent.snapshot": [
        false
      ],
      "host.id": [
        "ec22fc100e08a79a137d1a3c279a741f"
      ],
      "kubernetes.pod.ip": [
        "10.xx.xx.199"
      ],
      "kubernetes.node.labels.k8s_io/cloud-provider-aws": [
        "7c12cab0f30156fcbaf3c1e778ea0270"
      ],
      "kubernetes.container.name": [
        "teleport"
      ],
      "elastic_agent.id": [
        "5085b08f-d42a-414c-ba5d-458649065e57"
      ],
      "kubernetes.replicaset.name": [
        "teleport-auth-6dddf758bc"
      ],
      "data_stream.namespace": [
        "infra"
      ],
      "host.os.codename": [
        "focal"
      ],
      "kubernetes.namespace_labels.kubernetes_io/metadata_name": [
        "teleport"
      ],
      "kubernetes.namespace_labels.name": [
        "teleport"
      ],
      "message": [
        "{\"addr.remote\":\"xxx.xxx.xxx.211:60712\",\"caller\":\"events/emitter.go:265\",\"cluster_name\":\"teleport.com\",\"code\":\"T3009I\",\"component\":\"audit\",\"ei\":0,\"event\":\"kube.request\",\"kubernetes_cluster\":\"dev\",\"level\":\"info\",\"login\":\"xxx.xxx@xxx.com\",\"message\":\"kube.request\",\"namespace\":\"default\",\"proto\":\"kube\",\"request_path\":\"/apis/batch/v1/namespaces/xxx-xx/jobs\",\"resource_api_group\":\"batch/v1\",\"resource_kind\":\"jobs\",\"resource_namespace\":\"xxx-xxx\",\"response_code\":200,\"server_id\":\"93598ded-262a-45ea-9027-07eb8b0dcbcf\",\"time\":\"2023-09-07T13:10:45.899Z\",\"timestamp\":\"2023-09-07T13:10:45Z\",\"uid\":\"c6f7d765-7adb-4c97-979a-9fb6c64ee08a\",\"user\":\"xxx.xxxx@xxx.xxx\",\"verb\":\"GET\"}\n"
      ],
      "kubernetes.node.labels.kubernetes_io/hostname": [
        "ip-xx-xx-40-107.xxxx.compute.internal"
      ],
      "kubernetes.node.labels.beta_kubernetes_io/arch": [
        "amd64"
      ],
      "host.os.type.keyword": [
        "linux"
      ],
      "@timestamp": [
        "2023-09-07T13:10:45.898Z"
      ],
      "host.os.platform": [
        "ubuntu"
      ],
      "kubernetes.labels.app_kubernetes_io/managed-by": [
        "Helm"
      ],
      "data_stream.dataset": [
        "kubernetes.container_logs"
      ],
      "log.file.path": [
        "/var/log/containers/teleport-auth-6dddf758bc-8hdlk_teleport_teleport-74a61f8737aa44b7970f89b33520bb0182a23458406b0cea1d16b5e380b1b339.log"
      ],
      "kubernetes.node.labels.kubernetes_io/arch": [
        "amd64"
      ],
      "agent.ephemeral_id": [
        "2778ce61-a7d0-4590-86f3-6de50d5775bb"
      ],
      "kubernetes.node.labels.eks_amazonaws_com/capacityType": [
        "ON_DEMAND"
      ],
      "event.dataset": [
        "kubernetes.container_logs"
      ],
      "kubernetes.labels.teleport_dev/majorVersion": [
        "13"
      ],
      "cloud.instance.name": [
        "ip-xx-xx-40-107.xxxx.compute.internal"
      ]
    }
  }
4

Yeah, what I mentioned is the issue.

You have this field in your original message:

"event":"kube.request","kubernetes_cluster"

If you parse the json to the root of the document, this field will conflict with the object event field used by the elastic agent.

You need to parse your json into a target field.

5

Thank you very much. I don't see any more dropping messages in agent logs

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.