ILM rollsover the index but it stays in hot phase

I have ILM like this:

    PUT _ilm/policy/logs
    {
      "policy": {
        "phases": {
          "hot": {
            "min_age": "0ms",
            "actions": {
              "rollover": {
                "max_size": "1gb"
              }
            }
          },
          "delete": {
            "min_age": "1d",
            "actions": {
              "delete": {}
            }
          }
        }
      }
    }

And it is applied to template logs, that matches index pattern logs-*. My ILM policy works in very weird way - first, it applies rollover on the index logs-00000X after it reaches 2GB, then it anyway stays in hot phase and never enters the delete phase. What am I doing wrong here?

Please post the output of

GET logs-00000X/_ilm/explain

and

GET logs-00000X/_settings

where logs-00000X has been replaced with the actual name of your index.

Hello, thanks for the support with the issue.

Here's the _ilm/explain:

{
  "indices" : {
    "filebeat-000068" : {
      "index" : "filebeat-000068",
      "managed" : true,
      "policy" : "filebeat",
      "lifecycle_date_millis" : 1582005849352,
      "age" : "2.69h",
      "phase" : "hot",
      "phase_time_millis" : 1581989658439,
      "action" : "complete",
      "action_time_millis" : 1582005857685,
      "step" : "complete",
      "step_time_millis" : 1582005857685,
      "phase_execution" : {
        "policy" : "filebeat",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "1gb"
            }
          }
        },
        "version" : 2,
        "modified_date_in_millis" : 1581335157056
      }
    }
  }
}

And here's the _settings:

{
  "filebeat-000068" : {
    "settings" : {
      "index" : {
        "lifecycle" : {
          "name" : "filebeat",
          "rollover_alias" : "filebeat-alias",
          "indexing_complete" : "true"
        },
        "mapping" : {
          "total_fields" : {
            "limit" : "10000"
          }
        },
        "refresh_interval" : "5s",
        "number_of_shards" : "4",
        "provided_name" : "filebeat-000068",
        "query" : {
          "default_field" : [
            "message",
            "tags",
            "agent.ephemeral_id",
            "agent.id",
            "agent.name",
            "agent.type",
            "agent.version",
            "as.organization.name",
            "client.address",
            "client.as.organization.name",
            "client.domain",
            "client.geo.city_name",
            "client.geo.continent_name",
            "client.geo.country_iso_code",
            "client.geo.country_name",
            "client.geo.name",
            "client.geo.region_iso_code",
            "client.geo.region_name",
            "client.mac",
            "client.user.domain",
            "client.user.email",
            "client.user.full_name",
            "client.user.group.id",
            "client.user.group.name",
            "client.user.hash",
            "client.user.id",
            "client.user.name",
            "cloud.account.id",
            "cloud.availability_zone",
            "cloud.instance.id",
            "cloud.instance.name",
            "cloud.machine.type",
            "cloud.provider",
            "cloud.region",
            "container.id",
            "container.image.name",
            "container.image.tag",
            "container.name",
            "container.runtime",
            "destination.address",
            "destination.as.organization.name",
            "destination.domain",
            "destination.geo.city_name",
            "destination.geo.continent_name",
            "destination.geo.country_iso_code",
            "destination.geo.country_name",
            "destination.geo.name",
            "destination.geo.region_iso_code",
            "destination.geo.region_name",
            "destination.mac",
            "destination.user.domain",
            "destination.user.email",
            "destination.user.full_name",
            "destination.user.group.id",
            "destination.user.group.name",
            "destination.user.hash",
            "destination.user.id",
            "destination.user.name",
            "dns.answers.class",
            "dns.answers.data",
            "dns.answers.name",
            "dns.answers.type",
            "dns.header_flags",
            "dns.id",
            "dns.op_code",
            "dns.question.class",
            "dns.question.name",
            "dns.question.registered_domain",
            "dns.question.type",
            "dns.response_code",
            "dns.type",
            "ecs.version",
            "error.code",
            "error.id",
            "error.message",
            "event.action",
            "event.category",
            "event.code",
            "event.dataset",
            "event.hash",
            "event.id",
            "event.kind",
            "event.module",
            "event.original",
            "event.outcome",
            "event.provider",
            "event.timezone",
            "event.type",
            "file.device",
            "file.directory",
            "file.extension",
            "file.gid",
            "file.group",
            "file.hash.md5",
            "file.hash.sha1",
            "file.hash.sha256",
            "file.hash.sha512",
            "file.inode",
            "file.mode",
            "file.name",
            "file.owner",
            "file.path",
            "file.target_path",
            "file.type",
            "file.uid",
            "geo.city_name",
            "geo.continent_name",
            "geo.country_iso_code",
            "geo.country_name",
            "geo.name",
            "geo.region_iso_code",
            "geo.region_name",
            "group.id",
            "group.name",
            "hash.md5",
            "hash.sha1",
            "hash.sha256",
            "hash.sha512",
            "host.architecture",
            "host.geo.city_name",
            "host.geo.continent_name",
            "host.geo.country_iso_code",
            "host.geo.country_name",
            "host.geo.name",
            "host.geo.region_iso_code",
            "host.geo.region_name",
            "host.hostname",
            "host.id",
            "host.mac",
            "host.name",
            "host.os.family",
            "host.os.full",
            "host.os.kernel",
            "host.os.name",
            "host.os.platform",
            "host.os.version",
            "host.type",
            "host.user.domain",
            "host.user.email",
            "host.user.full_name",
            "host.user.group.id",
            "host.user.group.name",
            "host.user.hash",
            "host.user.id",
            "host.user.name",
            "http.request.body.content",
            "http.request.method",
            "http.request.referrer",
            "http.response.body.content",
            "http.version",
            "log.level",
            "log.logger",
            "log.original",
            "network.application",
            "network.community_id",
            "network.direction",
            "network.iana_number",
            "network.name",
            "network.protocol",
            "network.transport",
            "network.type",
            "observer.geo.city_name",
            "observer.geo.continent_name",
            "observer.geo.country_iso_code",
            "observer.geo.country_name",
            "observer.geo.name",
            "observer.geo.region_iso_code",
            "observer.geo.region_name",
            "observer.hostname",
            "observer.mac",
            "observer.os.family",
            "observer.os.full",
            "observer.os.kernel",
            "observer.os.name",
            "observer.os.platform",
            "observer.os.version",
            "observer.serial_number",
            "observer.type",
            "observer.vendor",
            "observer.version",
            "organization.id",
            "organization.name",
            "os.family",
            "os.full",
            "os.kernel",
            "os.name",
            "os.platform",
            "os.version",
            "process.args",
            "process.executable",
            "process.hash.md5",
            "process.hash.sha1",
            "process.hash.sha256",
            "process.hash.sha512",
            "process.name",
            "process.thread.name",
            "process.title",
            "process.working_directory",
            "server.address",
            "server.as.organization.name",
            "server.domain",
            "server.geo.city_name",
            "server.geo.continent_name",
            "server.geo.country_iso_code",
            "server.geo.country_name",
            "server.geo.name",
            "server.geo.region_iso_code",
            "server.geo.region_name",
            "server.mac",
            "server.user.domain",
            "server.user.email",
            "server.user.full_name",
            "server.user.group.id",
            "server.user.group.name",
            "server.user.hash",
            "server.user.id",
            "server.user.name",
            "service.ephemeral_id",
            "service.id",
            "service.name",
            "service.state",
            "service.type",
            "service.version",
            "source.address",
            "source.as.organization.name",
            "source.domain",
            "source.geo.city_name",
            "source.geo.continent_name",
            "source.geo.country_iso_code",
            "source.geo.country_name",
            "source.geo.name",
            "source.geo.region_iso_code",
            "source.geo.region_name",
            "source.mac",
            "source.user.domain",
            "source.user.email",
            "source.user.full_name",
            "source.user.group.id",
            "source.user.group.name",
            "source.user.hash",
            "source.user.id",
            "source.user.name",
            "tracing.trace.id",
            "tracing.transaction.id",
            "url.domain",
            "url.fragment",
            "url.full",
            "url.original",
            "url.password",
            "url.path",
            "url.query",
            "url.scheme",
            "url.username",
            "user.domain",
            "user.email",
            "user.full_name",
            "user.group.id",
            "user.group.name",
            "user.hash",
            "user.id",
            "user.name",
            "user_agent.device.name",
            "user_agent.name",
            "user_agent.original",
            "user_agent.os.family",
            "user_agent.os.full",
            "user_agent.os.kernel",
            "user_agent.os.name",
            "user_agent.os.platform",
            "user_agent.os.version",
            "user_agent.version",
            "agent.hostname",
            "error.type",
            "timeseries.instance",
            "cloud.project.id",
            "cloud.image.id",
            "host.os.build",
            "host.os.codename",
            "kubernetes.pod.name",
            "kubernetes.pod.uid",
            "kubernetes.namespace",
            "kubernetes.node.name",
            "kubernetes.replicaset.name",
            "kubernetes.deployment.name",
            "kubernetes.statefulset.name",
            "kubernetes.container.name",
            "kubernetes.container.image",
            "jolokia.agent.version",
            "jolokia.agent.id",
            "jolokia.server.product",
            "jolokia.server.version",
            "jolokia.server.vendor",
            "jolokia.url",
            "log.file.path",
            "log.source.address",
            "stream",
            "input.type",
            "syslog.severity_label",
            "syslog.facility_label",
            "process.program",
            "log.flags",
            "user_agent.os.full_name",
            "fileset.name",
            "icmp.code",
            "icmp.type",
            "igmp.type",
            "kafka.topic",
            "kafka.key",
            "apache.access.ssl.protocol",
            "apache.access.ssl.cipher",
            "apache.error.module",
            "user.terminal",
            "user.audit.id",
            "user.audit.name",
            "user.audit.group.id",
            "user.audit.group.name",
            "user.effective.id",
            "user.effective.name",
            "user.effective.group.id",
            "user.effective.group.name",
            "user.filesystem.id",
            "user.filesystem.name",
            "user.filesystem.group.id",
            "user.filesystem.group.name",
            "user.owner.id",
            "user.owner.name",
            "user.owner.group.id",
            "user.owner.group.name",
            "user.saved.id",
            "user.saved.name",
            "user.saved.group.id",
            "user.saved.group.name",
            "fields.*"
          ]
        },
        "creation_date" : "1581989639415",
        "number_of_replicas" : "1",
        "uuid" : "Jib9LcGIQySFKjS3U4_y9A",
        "version" : {
          "created" : "7050299"
        }
      }
    }
  }
}

PS. I'm using filebeat to push logs, but I've setup the ILM policy myself with API calls, as filebeat doesn't support multiple ILM policies for multiple indices.

I notice that the ILM policy is filebeat, and not logs as in your original message. Can you please check that the filebeat policy has a delete phase?

The first post refers to filebeat policy. Sorry for the confusion, I've changed the name in the post on the forum.

Here's a bit more information about the index:

and about the template:

and the ILM: