Downsampling policy

Elastic Stack Elasticsearch
1

Hello,
I'm trying to define a policy for my time-series data stream as follows:

"policy": {
      "phases": {
        "warm": {
          "min_age": "0d",
          "actions": {
            "readonly": {},
            "set_priority": {
              "priority": 50
            }
          }
        },
        "cold": {
          "min_age": "2d",
          "actions": {
            "downsample": {
              "fixed_interval": "1h",
              "wait_timeout": "1d"
            },
            "set_priority": {
              "priority": 0
            }
          }
        },
        "hot": {
          "min_age": "0ms",
          "actions": {
            "set_priority": {
              "priority": 100
            },
            "rollover": {
              "max_age": "2d"
            }
          }
        },
        "delete": {
          "min_age": "90d",
          "actions": {
            "delete": {
              "delete_searchable_snapshot": true
            }
          }
        }
      },
      "_meta": {
        "description": "retention policy, IndexCount-5"
      }```

Basically, I want to trigger downsampling once my data stream enters the cold state. So, it should do a rollover after 2 days and enter the warm state, and then move to the cold state after 2 more days.
However, the data stream never gets into the cold state, and hence, never undergoes downsampling.

Below is the output of the explain API:

```{
  "indices": {
    ".ds-my-tsds-2024.02.05-000001": {
      "index": ".ds-my-tsds-2024.02.05-000001",
      "managed": true,
      "policy": "retention_policy_my-tsds",
      "index_creation_date_millis": 1707172560551,
      "time_since_index_creation": "6.89d",
      "lifecycle_date_millis": 1707345782542,
      "age": "4.88d",
      "phase": "warm",
      "phase_time_millis": 1707518582665,
      "action": "migrate",
      "action_time_millis": 1707519182588,
      "step": "check-migration",
      "step_time_millis": 1707519182588,
      "step_info": {
        "message": "Waiting for all shard copies to be active",
        "shards_left_to_allocate": -1,
        "all_shards_active": false,
        "number_of_replicas": 1
      },
      "phase_execution": {
        "policy": "retention_policy_my-tsds",
        "phase_definition": {
          "min_age": "0d",
          "actions": {
            "readonly": {},
            "set_priority": {
              "priority": 50
            }
          }
        },
        "version": 4,
        "modified_date_in_millis": 1707657779774
      }
    },
    ".ds-my-tsds-2024.02.07-000002": {
      "index": ".ds-my-tsds-2024.02.07-000002",
      "managed": true,
      "policy": "retention_policy_my-tsds",
      "index_creation_date_millis": 1707345782601,
      "time_since_index_creation": "4.88d",
      "lifecycle_date_millis": 1707519182388,
      "age": "2.88d",
      "phase": "warm",
      "phase_time_millis": 1707657779671,
      "action": "migrate",
      "action_time_millis": 1707657782475,
      "step": "check-migration",
      "step_time_millis": 1707657782675,
      "step_info": {
        "message": "Waiting for all shard copies to be active",
        "shards_left_to_allocate": -1,
        "all_shards_active": false,
        "number_of_replicas": 1
      },
      "phase_execution": {
        "policy": "retention_policy_my-tsds",
        "phase_definition": {
          "min_age": "0d",
          "actions": {
            "readonly": {},
            "set_priority": {
              "priority": 50
            }
          }
        },
        "version": 4,
        "modified_date_in_millis": 1707657779774
      }
    },
    ".ds-my-tsds-2024.02.09-000003": {
      "index": ".ds-my-tsds-2024.02.09-000003",
      "managed": true,
      "policy": "retention_policy_my-tsds",
      "index_creation_date_millis": 1707519182569,
      "time_since_index_creation": "2.88d",
      "lifecycle_date_millis": 1707692582561,
      "age": "20.97h",
      "phase": "warm",
      "phase_time_millis": 1707692582762,
      "action": "migrate",
      "action_time_millis": 1707700382469,
      "step": "check-migration",
      "step_time_millis": 1707700382469,
      "step_info": {
        "message": "Waiting for all shard copies to be active",
        "shards_left_to_allocate": -1,
        "all_shards_active": false,
        "number_of_replicas": 1
      },
      "phase_execution": {
        "policy": "retention_policy_my-tsds",
        "phase_definition": {
          "min_age": "0d",
          "actions": {
            "readonly": {},
            "set_priority": {
              "priority": 50
            }
          }
        },
        "version": 4,
        "modified_date_in_millis": 1707657779774
      }
    },
    ".ds-my-tsds-2024.02.11-000004": {
      "index": ".ds-my-tsds-2024.02.11-000004",
      "managed": true,
      "policy": "retention_policy_my-tsds",
      "index_creation_date_millis": 1707692582576,
      "time_since_index_creation": "20.97h",
      "lifecycle_date_millis": 1707692582576,
      "age": "20.97h",
      "phase": "hot",
      "phase_time_millis": 1707692582762,
      "action": "rollover",
      "action_time_millis": 1707692582762,
      "step": "check-rollover-ready",
      "step_time_millis": 1707692582762,
      "phase_execution": {
        "policy": "retention_policy_my-tsds",
        "phase_definition": {
          "min_age": "0ms",
          "actions": {
            "set_priority": {
              "priority": 100
            },
            "rollover": {
              "max_age": "2d"
            }
          }
        },
        "version": 4,
        "modified_date_in_millis": 1707657779774
      }
    }
  }
}

Appreciate if anyone can check what am I missing in the above policy.

TIA,
Noam

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.