Field data type unknown even with mapping

Elastic Stack Elastic Agent
1

Hi,

I'm using an agent policy with a Custom Logs integration to send my logs to Elastic. The logs are coming in fine except for one thing: a couple of the fields are coming in as unknown data types. I have tried to add a mapping to the integration to explicitly map the fields to a type, but it does not appear to be working. I following the instructions here to set up the mappings for my fields.

The mappings on the component template look like so (on the left), but the fields still have unknown data type (on the right):

image
image1643×632 122 KB

Dynamic mapping is also enabled on the component template. Any ideas on what I'm doing wrong?

Thanks!

2

Can you share the json document you have in Elasticsearch?

Go in the expanded document in discover, select the json tab and share the entire document.

Also, did you rollover your data stream after changing the mapping? When you change a template mapping it will only be applied to newly created indices.

3

Here's the json document:

{
  "_index": ".ds-logs-camera_storage_service-default-2024.07.11-000004",
  "_id": "8ulgn5ABT3Hv0nvcorYf",
  "_version": 1,
  "_score": 0,
  "_source": {
    "input": {
      "type": "log"
    },
    "agent": {
      "name": "storage-1.camera-dev.butterflymx.com",
      "id": "64199826-5905-4da4-ae05-cd8f151f88aa",
      "ephemeral_id": "3e7551b8-51a8-481a-b199-9068c28f0bf5",
      "type": "filebeat",
      "version": "8.9.2"
    },
    "@timestamp": "2024-07-11T01:20:27.664Z",
    "ecs": {
      "version": "8.0.0"
    },
    "log": {
      "file": {
        "path": "/var/log/storage-service/combined.log"
      },
      "offset": 1940822106,
      "level": "debug"
    },
    "data_stream": {
      "namespace": "default",
      "type": "logs",
      "dataset": "camera_storage_service"
    },
    "host": {
      "hostname": "storage-1.camera-dev.butterflymx.com",
      "os": {
        "kernel": "5.4.0-176-generic",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.18.0.4"
      ],
      "name": "storage-1.camera-dev.butterflymx.com",
      "mac": [
        "02-42-AC-12-00-04"
      ],
      "architecture": "x86_64"
    },
    "elastic_agent": {
      "id": "64199826-5905-4da4-ae05-cd8f151f88aa",
      "version": "8.9.2",
      "snapshot": false
    },
    "storage-service": {
      "requestId": "86b37de8-3796-4dc2-a7c7-b08cfa34ec95",
      "service": "StorageService"
    },
    "message": "Successfully uploaded snapshot to snapshots/stg-core-na-building-27554/66843e01e9cf80d2bbe9308b/snapshot.png",
    "event": {
      "agent_id_status": "verified",
      "ingested": "2024-07-11T01:20:28Z",
      "dataset": "camera_storage_service"
    }
  },
  "fields": {
    "storage-service.service": [
      "StorageService"
    ],
    "elastic_agent.version": [
      "8.9.2"
    ],
    "host.os.name.text": [
      "Ubuntu"
    ],
    "host.hostname": [
      "storage-1.camera-dev.butterflymx.com"
    ],
    "host.mac": [
      "02-42-AC-12-00-04"
    ],
    "host.ip": [
      "172.18.0.4"
    ],
    "agent.type": [
      "filebeat"
    ],
    "storage-service.requestId": [
      "86b37de8-3796-4dc2-a7c7-b08cfa34ec95"
    ],
    "host.os.version": [
      "20.04.6 LTS (Focal Fossa)"
    ],
    "host.os.kernel": [
      "5.4.0-176-generic"
    ],
    "host.os.name": [
      "Ubuntu"
    ],
    "log.file.path.text": [
      "/var/log/storage-service/combined.log"
    ],
    "log.level": [
      "debug"
    ],
    "agent.name": [
      "storage-1.camera-dev.butterflymx.com"
    ],
    "elastic_agent.snapshot": [
      false
    ],
    "host.name": [
      "storage-1.camera-dev.butterflymx.com"
    ],
    "event.agent_id_status": [
      "verified"
    ],
    "host.os.type": [
      "linux"
    ],
    "elastic_agent.id": [
      "64199826-5905-4da4-ae05-cd8f151f88aa"
    ],
    "data_stream.namespace": [
      "default"
    ],
    "host.os.codename": [
      "focal"
    ],
    "input.type": [
      "log"
    ],
    "log.offset": [
      1940822106
    ],
    "message": [
      "Successfully uploaded snapshot to snapshots/stg-core-na-building-27554/66843e01e9cf80d2bbe9308b/snapshot.png"
    ],
    "data_stream.type": [
      "logs"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "event.ingested": [
      "2024-07-11T01:20:28.000Z"
    ],
    "@timestamp": [
      "2024-07-11T01:20:27.664Z"
    ],
    "agent.id": [
      "64199826-5905-4da4-ae05-cd8f151f88aa"
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "host.containerized": [
      true
    ],
    "host.os.platform": [
      "ubuntu"
    ],
    "data_stream.dataset": [
      "camera_storage_service"
    ],
    "log.file.path": [
      "/var/log/storage-service/combined.log"
    ],
    "agent.ephemeral_id": [
      "3e7551b8-51a8-481a-b199-9068c28f0bf5"
    ],
    "agent.version": [
      "8.9.2"
    ],
    "host.os.family": [
      "debian"
    ],
    "event.dataset": [
      "camera_storage_service"
    ]
  }
}

I did rollover the data stream after changing the mapping. Should that have created a new index?

4

Looks like I just had to be patient and wait for the index to update. The fields are coming through with the expected types now. Thank you!