I'm using elasticsearch as storage backend for jaeger.
I initialized ilm, then created index. It worked well until 4-th index. On 4-th index ilm not applied anymore.
Steps to reproduce
- Create ilm policy
_ilm/policy/jaeger-ilm-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "4h"
},
"set_priority": {
"priority": 100
}
}
},
"delete": {
"min_age": "1d",
"actions": {
"delete": {}
}
}
}
}
}
- Initialized indexies with jaeger image
docker run -it --rm --net=host -e ES_USE_ILM=true jaegertracing/jaeger-es-rollover:1.24 init http://localhost:2540
Current state of cluster
/_ilm/status
{
"operation_mode": "RUNNING"
}
/*/_ilm/explain?human
{
"indices": {
"jaeger-span-000004": {
"index": "jaeger-span-000004",
"managed": false
},
"jaeger-span-000003": {
"index": "jaeger-span-000003",
"managed": true,
"policy": "jaeger-ilm-policy",
"lifecycle_date": "2021-08-03T13:15:17.482Z",
"lifecycle_date_millis": 1627996517482,
"age": "3.89m",
"phase": "hot",
"phase_time": "2021-08-03T13:05:17.679Z",
"phase_time_millis": 1627995917679,
"action": "complete",
"action_time": "2021-08-03T13:15:17.838Z",
"action_time_millis": 1627996517838,
"step": "complete",
"step_time": "2021-08-03T13:15:17.838Z",
"step_time_millis": 1627996517838,
"phase_execution": {
"policy": "jaeger-ilm-policy",
"phase_definition": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "1m"
},
"set_priority": {
"priority": 100
}
}
},
"version": 1,
"modified_date": "2021-08-03T12:49:02.401Z",
"modified_date_in_millis": 1627994942401
}
},
"jaeger-service-000004": {
"index": "jaeger-service-000004",
"managed": false
},
"jaeger-service-000003": {
"index": "jaeger-service-000003",
"managed": true,
"policy": "jaeger-ilm-policy",
"lifecycle_date": "2021-08-03T13:15:17.282Z",
"lifecycle_date_millis": 1627996517282,
"age": "3.89m",
"phase": "hot",
"phase_time": "2021-08-03T13:05:17.599Z",
"phase_time_millis": 1627995917599,
"action": "complete",
"action_time": "2021-08-03T13:15:17.827Z",
"action_time_millis": 1627996517827,
"step": "complete",
"step_time": "2021-08-03T13:15:17.827Z",
"step_time_millis": 1627996517827,
"phase_execution": {
"policy": "jaeger-ilm-policy",
"phase_definition": {
"min_age": "0ms",
"actions": {
"rollover": {
"max_age": "1m"
},
"set_priority": {
"priority": 100
}
}
},
"version": 1,
"modified_date": "2021-08-03T12:49:02.401Z",
"modified_date_in_millis": 1627994942401
}
}
}
}
/_template/jaeger-*
{
"jaeger-span": {
"order": 0,
"index_patterns": [
"*jaeger-span-*"
],
"settings": {
"index": {
"lifecycle": {
"name": "jaeger-ilm-policy",
"rollover_alias": "jaeger-span-write"
},
"mapping": {
"nested_fields": {
"limit": "50"
}
},
"requests": {
"cache": {
"enable": "true"
}
},
"number_of_shards": "5",
"number_of_replicas": "1"
}
},
"mappings": {
"dynamic_templates": [
{
"span_tags_map": {
"path_match": "tag.*",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
},
{
"process_tags_map": {
"path_match": "process.tag.*",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
}
],
"properties": {
"traceID": {
"ignore_above": 256,
"type": "keyword"
},
"process": {
"properties": {
"tag": {
"type": "object"
},
"serviceName": {
"ignore_above": 256,
"type": "keyword"
},
"tags": {
"dynamic": false,
"type": "nested",
"properties": {
"tagType": {
"ignore_above": 256,
"type": "keyword"
},
"value": {
"ignore_above": 256,
"type": "keyword"
},
"key": {
"ignore_above": 256,
"type": "keyword"
}
}
}
}
},
"startTimeMillis": {
"format": "epoch_millis",
"type": "date"
},
"references": {
"dynamic": false,
"type": "nested",
"properties": {
"traceID": {
"ignore_above": 256,
"type": "keyword"
},
"spanID": {
"ignore_above": 256,
"type": "keyword"
},
"refType": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"flags": {
"type": "integer"
},
"operationName": {
"ignore_above": 256,
"type": "keyword"
},
"parentSpanID": {
"ignore_above": 256,
"type": "keyword"
},
"tags": {
"dynamic": false,
"type": "nested",
"properties": {
"tagType": {
"ignore_above": 256,
"type": "keyword"
},
"value": {
"ignore_above": 256,
"type": "keyword"
},
"key": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"spanID": {
"ignore_above": 256,
"type": "keyword"
},
"duration": {
"type": "long"
},
"startTime": {
"type": "long"
},
"tag": {
"type": "object"
},
"logs": {
"dynamic": false,
"type": "nested",
"properties": {
"fields": {
"dynamic": false,
"type": "nested",
"properties": {
"tagType": {
"ignore_above": 256,
"type": "keyword"
},
"value": {
"ignore_above": 256,
"type": "keyword"
},
"key": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"timestamp": {
"type": "long"
}
}
}
}
},
"aliases": {
"jaeger-span-read": {
}
}
},
"jaeger-service": {
"order": 0,
"index_patterns": [
"*jaeger-service-*"
],
"settings": {
"index": {
"lifecycle": {
"name": "jaeger-ilm-policy",
"rollover_alias": "jaeger-service-write"
},
"mapping": {
"nested_fields": {
"limit": "50"
}
},
"requests": {
"cache": {
"enable": "true"
}
},
"number_of_shards": "5",
"number_of_replicas": "1"
}
},
"mappings": {
"dynamic_templates": [
{
"span_tags_map": {
"path_match": "tag.*",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
},
{
"process_tags_map": {
"path_match": "process.tag.*",
"mapping": {
"ignore_above": 256,
"type": "keyword"
}
}
}
],
"properties": {
"operationName": {
"ignore_above": 256,
"type": "keyword"
},
"serviceName": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"aliases": {
"jaeger-service-read": {
}
}
}
}
It seems to me the issue here jaeger-span.settings.index.lifecycle.name. It has value while created and dissapear after a few index rotations.
Also there is a warnings about templates
legacy template [jaeger-span] has index patterns [*jaeger-span-*] matching patterns from existing composable templates [.slm-history,.watch-history-13,ilm-history,logs,metrics,synthetics] with patterns (.slm-history => [.slm-history-5*],.watch-history-13 => [.watcher-history-13*],ilm-history => [ilm-history-5*],logs => [logs-*-*],metrics => [metrics-*-*],synthetics => [synthetics-*-*]); this template [jaeger-span] may be ignored in favor of a composable template at index creation time
Legacy index templates are deprecated in favor of composable templates