SLM Policy Not Being Followed?

Elastic Stack Elasticsearch
,
1

Hi All,

I'm attempting to debug an issue, and while attempting to debug said issue, I noticed potentially another issue where SLM isn't properly cleaning up snapshots.

Has anyone seen this issue before or know why it's happening

Elasticsearch: 7.14.1

Steps:

  1. Register an S3 (or maybe other?) repository with Elasticsearch
    • For this these steps I'll call the repository: es-dev-snapshots
  2. Create an index:
PUT /test-index
  1. Create SLM policy
    • Important points:
      • Expire Snapshot after 30 minutes
      • Run every 15 minutes
      • Keep at least 1 snapshot
      • Keep at most 2 snapshots
PUT /_slm/policy/test-slm-policy
{
  "name": "<test-slm-policy-{now/m}>",
  "schedule": "0 */15 * * * ?",
  "repository": "es-dev-snapshots",
  "config": {
    "include_global_state" : false,
    "indices": [
      "test-index"
    ]
  },
  "retention": {
    "expire_after": "30m",
    "min_count": 1,
    "max_count": 2
  }
}
  1. Expect only 2 snapshots to be there at any given time, and the oldest being 30 minutes old.
  2. See far more than 2 snapshots, and snapshots far older than 30 minutes when using: GET /_snapshot/es-dev-snapshots/_all 
{
  "snapshots" : [
    {
      "snapshot" : "test-slm-policy-2021.10.01-klaqknqxspi6i72ogtpeuq",
      "uuid" : "13M17iO3SDSTekkcSwwiXw",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T13:44:59.861Z",
      "start_time_in_millis" : 1633095899861,
      "end_time" : "2021-10-01T13:45:01.862Z",
      "end_time_in_millis" : 1633095901862,
      "duration_in_millis" : 2001,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-yhwjwtgxshmvbd9a_2i07a",
      "uuid" : "tYvqeumXQ8maGcCAYTnCXg",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T13:59:59.845Z",
      "start_time_in_millis" : 1633096799845,
      "end_time" : "2021-10-01T14:00:00.645Z",
      "end_time_in_millis" : 1633096800645,
      "duration_in_millis" : 800,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-k3m5vktsrgks9utvlwbmeq",
      "uuid" : "r1AOqfBbRMOY-vrEZvN34w",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T14:14:59.810Z",
      "start_time_in_millis" : 1633097699810,
      "end_time" : "2021-10-01T14:15:01.811Z",
      "end_time_in_millis" : 1633097701811,
      "duration_in_millis" : 2001,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-7q24a6xmteqqk8laogbn5q",
      "uuid" : "3Kr-QXu5SvKItWp-qjNsjw",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T14:29:59.975Z",
      "start_time_in_millis" : 1633098599975,
      "end_time" : "2021-10-01T14:30:00.376Z",
      "end_time_in_millis" : 1633098600376,
      "duration_in_millis" : 401,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-gseeft4drycxkezf_mdjaw",
      "uuid" : "bt9TbKGcTGO8OYUiSBWFDw",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T14:44:59.971Z",
      "start_time_in_millis" : 1633099499971,
      "end_time" : "2021-10-01T14:45:00.371Z",
      "end_time_in_millis" : 1633099500371,
      "duration_in_millis" : 400,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-cvarllimqms32g5yq06goa",
      "uuid" : "DXuZz4S6QTqrMCP7V1WMWQ",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T14:59:59.949Z",
      "start_time_in_millis" : 1633100399949,
      "end_time" : "2021-10-01T15:00:00.549Z",
      "end_time_in_millis" : 1633100400549,
      "duration_in_millis" : 600,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-_entlmf4qbaez1okv0jtsg",
      "uuid" : "QNzBOo_eQj-tWZ6EsuZn8w",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T15:14:59.944Z",
      "start_time_in_millis" : 1633101299944,
      "end_time" : "2021-10-01T15:15:02.345Z",
      "end_time_in_millis" : 1633101302345,
      "duration_in_millis" : 2401,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "2021.10.01-test-index-test-policy-_85ye5ahryu7s7tbgjvfkg",
      "uuid" : "MAJ2GFy2QEqNHNltsY6-Nw",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T15:15:36.559Z",
      "start_time_in_millis" : 1633101336559,
      "end_time" : "2021-10-01T15:15:36.959Z",
      "end_time_in_millis" : 1633101336959,
      "duration_in_millis" : 400,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-n_qjayw7rpegwyx0sencma",
      "uuid" : "lbADXozHQkqUSV03646bTw",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "restored-test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T15:29:59.915Z",
      "start_time_in_millis" : 1633102199915,
      "end_time" : "2021-10-01T15:30:03.516Z",
      "end_time_in_millis" : 1633102203516,
      "duration_in_millis" : 3601,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-8jknnpv2r3i02krhyu3khw",
      "uuid" : "qt90qHjNSqC4WWWXtnM62Q",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "restored-test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T15:44:59.876Z",
      "start_time_in_millis" : 1633103099876,
      "end_time" : "2021-10-01T15:45:00.276Z",
      "end_time_in_millis" : 1633103100276,
      "duration_in_millis" : 400,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-wyhel6zxtpo9uxyl-uez9q",
      "uuid" : "v4A3tDOySsSAfKnaG4cm9w",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "restored-test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T15:59:59.858Z",
      "start_time_in_millis" : 1633103999858,
      "end_time" : "2021-10-01T16:00:00.258Z",
      "end_time_in_millis" : 1633104000258,
      "duration_in_millis" : 400,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    },
    {
      "snapshot" : "test-slm-policy-2021.10.01-arpofztnr-icv3ntb5s5bq",
      "uuid" : "Bv8evsTlSY-Kg13QCpY6UQ",
      "repository" : "es-dev-snapshots",
      "version_id" : 7140199,
      "version" : "7.14.1",
      "indices" : [
        "restored-test-index"
      ],
      "data_streams" : [ ],
      "include_global_state" : false,
      "metadata" : {
        "policy" : "test-slm-policy"
      },
      "state" : "SUCCESS",
      "start_time" : "2021-10-01T16:14:59.828Z",
      "start_time_in_millis" : 1633104899828,
      "end_time" : "2021-10-01T16:15:01.028Z",
      "end_time_in_millis" : 1633104901028,
      "duration_in_millis" : 1200,
      "failures" : [ ],
      "shards" : {
        "total" : 1,
        "failed" : 0,
        "successful" : 1
      },
      "feature_states" : [ ]
    }
  ]
}
2

SLM retention task is scheduled to run daily , by default on 01.30 am UTC. Is it possible that the the scheduled task hasn’t run yet?

3

That did the trick. Thanks.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.