Running your test case in 18.4.3:
I removed the datastream so everything is created from scratch.
I also removed the "@timestamp" to have recent & unique timestamps.
POST logs-debug-test/_doc
{
"message" : "my log message",
"data_stream" : {
"dataset" : "debug",
"namespace" : "test"
},
"event" : {
"dataset" : "mycustomlogs"
},
"host" : {
"name" : "myhost",
"hostname" : "myhostname"
},
"container" : {
"id" : "12345-345876"
},
"service" :{ "name" : "myservice"}
}
After sending in, it look like:
Result is still the same @stephenb :
Data stream with default logs index template:
GET /_data_stream/logs-debug-test/
{
"data_streams": [
{
"name": "logs-debug-test",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-logs-debug-test-2024.10.11-000001",
"index_uuid": "bloEyFieQUyiu6a5t5IYLQ",
"prefer_ilm": true,
"ilm_policy": "logs",
"managed_by": "Index Lifecycle Management"
}
],
"generation": 1,
"_meta": {
"managed": true,
"description": "default logs template installed by x-pack"
},
"status": "YELLOW",
"template": "logs",
"ilm_policy": "logs",
"next_generation_managed_by": "Index Lifecycle Management",
"prefer_ilm": true,
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false,
"rollover_on_write": false
}
]
}
Index
GET logs-debug-test/
{
".ds-logs-debug-test-2024.10.11-000001": {
"aliases": {},
"mappings": {
"_data_stream_timestamp": {
"enabled": true
},
"dynamic_templates": [
{
"ecs_timestamp": {
"match": "@timestamp",
"mapping": {
"ignore_malformed": false,
"type": "date"
}
}
},
{
"ecs_message_match_only_text": {
"path_match": [
"message",
"*.message"
],
"unmatch_mapping_type": "object",
"mapping": {
"type": "match_only_text"
}
}
},
{
"ecs_non_indexed_keyword": {
"path_match": "event.original",
"mapping": {
"doc_values": false,
"index": false,
"type": "keyword"
}
}
},
{
"ecs_non_indexed_long": {
"path_match": "*.x509.public_key_exponent",
"mapping": {
"doc_values": false,
"index": false,
"type": "long"
}
}
},
{
"ecs_ip": {
"path_match": [
"ip",
"*.ip",
"*_ip"
],
"match_mapping_type": "string",
"mapping": {
"type": "ip"
}
}
},
{
"ecs_wildcard": {
"path_match": [
"*.io.text",
"*.message_id",
"*registry.data.strings",
"*url.path"
],
"unmatch_mapping_type": "object",
"mapping": {
"type": "wildcard"
}
}
},
{
"ecs_path_match_wildcard_and_match_only_text": {
"path_match": [
"*.body.content",
"*url.full",
"*url.original"
],
"unmatch_mapping_type": "object",
"mapping": {
"fields": {
"text": {
"type": "match_only_text"
}
},
"type": "wildcard"
}
}
},
{
"ecs_match_wildcard_and_match_only_text": {
"match": [
"*command_line",
"*stack_trace"
],
"unmatch_mapping_type": "object",
"mapping": {
"fields": {
"text": {
"type": "match_only_text"
}
},
"type": "wildcard"
}
}
},
{
"ecs_path_match_keyword_and_match_only_text": {
"path_match": [
"*.title",
"*.executable",
"*.name",
"*.working_directory",
"*.full_name",
"*file.path",
"*file.target_path",
"*os.full",
"email.subject",
"vulnerability.description",
"user_agent.original"
],
"unmatch_mapping_type": "object",
"mapping": {
"fields": {
"text": {
"type": "match_only_text"
}
},
"type": "keyword"
}
}
},
{
"ecs_date": {
"path_match": [
"*.timestamp",
"*_timestamp",
"*.not_after",
"*.not_before",
"*.accessed",
"created",
"*.created",
"*.installed",
"*.creation_date",
"*.ctime",
"*.mtime",
"ingested",
"*.ingested",
"*.start",
"*.end"
],
"unmatch_mapping_type": "object",
"mapping": {
"type": "date"
}
}
},
{
"ecs_path_match_float": {
"path_match": [
"*.score.*",
"*_score*"
],
"path_unmatch": "*.version",
"unmatch_mapping_type": "object",
"mapping": {
"type": "float"
}
}
},
{
"ecs_usage_double_scaled_float": {
"path_match": "*.usage",
"match_mapping_type": [
"double",
"long",
"string"
],
"mapping": {
"scaling_factor": 1000,
"type": "scaled_float"
}
}
},
{
"ecs_geo_point": {
"path_match": [
"location",
"*.location"
],
"mapping": {
"type": "geo_point"
}
}
},
{
"ecs_flattened": {
"path_match": [
"*structured_data",
"*exports",
"*imports"
],
"match_mapping_type": "object",
"mapping": {
"type": "flattened"
}
}
},
{
"all_strings_to_keywords": {
"match_mapping_type": "string",
"mapping": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
],
"date_detection": false,
"properties": {
"@timestamp": {
"type": "date",
"ignore_malformed": false
},
"container": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"data_stream": {
"properties": {
"dataset": {
"type": "constant_keyword",
"value": "debug"
},
"namespace": {
"type": "constant_keyword",
"value": "test"
},
"type": {
"type": "constant_keyword",
"value": "logs"
}
}
},
"event": {
"properties": {
"dataset": {
"type": "keyword",
"ignore_above": 1024
}
}
},
"host": {
"properties": {
"hostname": {
"type": "keyword",
"ignore_above": 1024
},
"name": {
"type": "keyword",
"fields": {
"text": {
"type": "match_only_text"
}
}
}
}
},
"message": {
"type": "match_only_text"
},
"service": {
"properties": {
"name": {
"type": "keyword",
"fields": {
"text": {
"type": "match_only_text"
}
}
}
}
}
}
},
"settings": {
"index": {
"lifecycle": {
"name": "logs"
},
"codec": "best_compression",
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_hot"
}
}
},
"mapping": {
"total_fields": {
"ignore_dynamic_beyond_limit": "true"
},
"ignore_malformed": "true"
},
"hidden": "true",
"number_of_shards": "1",
"provided_name": ".ds-logs-debug-test-2024.10.11-000001",
"default_pipeline": "logs@default-pipeline",
"creation_date": "1728627754279",
"number_of_replicas": "1",
"uuid": "bloEyFieQUyiu6a5t5IYLQ",
"version": {
"created": "8505000"
}
}
},
"data_stream": "logs-debug-test"
}
}
Best regards
Christof
It seems that I "broke" it during my testing.. Might be related to a different message I recovered from the operational stack, where I forgot to remove the data_stream part.
