APM-Server ILM policy is not applied

I have APM Server running with following error

apm-server.yml
# Source : https://github.com/elastic/apm-server/blob/master/apm-server.docker.yml
################################ APM Server ################################
apm-server:
  host: "0.0.0.0:8200"
  frontend:
    enabled: false

#================================= APM Server - RUM Real User Monitoring =================================

# Enable Real User Monitoring (RUM) Support. By default RUM is disabled.
# RUM does not support token based authorization. Enabled RUM endpoints will not require any authorization
# token configured for other endpoints.
rum:
  enabled: false

  event_rate:

    # Defines the maximum amount of events allowed to be sent to the APM Server RUM
    # endpoint per IP per second. Defaults to 300.
    limit: 300

    # An LRU cache is used to keep a rate limit per IP for the most recently seen IPs.
    # This setting defines the number of unique IPs that can be tracked in the cache.
    # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
    lru_size: 1000

  #-- General RUM settings

  # A list of permitted origins for real user monitoring.
  # User-agents will send an origin header that will be validated against this list.
  # An origin is made of a protocol scheme, host and port, without the url path.
  # Allowed origins in this setting can have * to match anything (eg.: http://*.example.com)
  # If an item in the list is a single '*', everything will be allowed.
  allow_origins : ['*']

  # A list of Access-Control-Allow-Headers to allow RUM requests, in addition to "Content-Type",
  # "Content-Encoding", and "Accept"
  allow_headers : []

  # Custom HTTP headers to add to RUM responses, e.g. for security policy compliance.
  #response_headers :
  #  X-My-Header: Contents of the header

  # Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes.
  # If the regexp matches, the stacktrace frame is considered to be a library frame.
  #library_pattern: "node_modules|bower_components|~"

  # Regexp to be matched against a stacktrace frame's `file_name`.
  # If the regexp matches, the stacktrace frame is not used for calculating error groups.
  # The default pattern excludes stacktrace frames that have a filename starting with '/webpack'
  #exclude_from_grouping: "^/webpack"

  # If a source map has previously been uploaded, source mapping is automatically applied.
  # to all error and transaction documents sent to the RUM endpoint.
  source_mapping:

    # Sourcemapping is enabled by default.
    enabled: true

    # Source maps are always fetched from Elasticsearch, by default using the output.elasticsearch configuration.
    # A different instance must be configured when using any other output.
    # This setting only affects sourcemap reads - the output determines where sourcemaps are written.
    elasticsearch:
      # Array of hosts to connect to.
      # Scheme and port can be left out and will be set to the default (`http` and `9200`).
      # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`.
      # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`.
      hosts: ["elasticsearch:9200"]

      # Protocol - either `http` (default) or `https`.
      protocol: "http"

      # Authentication credentials - either API key or username/password.
      #api_key: "id:api_key"
      username: "elastic"
      password: "changeme"

    # The `cache.expiration` determines how long a source map should be cached before fetching it again from Elasticsearch.
    # Note that values configured without a time unit will be interpreted as seconds.
    cache:
      expiration: 5m

    # Source maps are stored in a separate index.
    # If the default index pattern for source maps at 'outputs.elasticsearch.indices'
    # is changed, a matching index pattern needs to be specified here.
    index_pattern: "apm-*-sourcemap*"

#================================= APM Server - Agent Configuration =================================

# When using APM agent configuration, information fetched from Kibana will be cached in memory for some time.
# Specify cache key expiration via this setting. Default is 30 seconds.
agent.config.cache.expiration: 30s
kibana:
  enabled: true
  host: "kibana:5601"
  protocol: "http"

#================================= APM Server - ILM Index Lifecycle Management =================================

ilm:
  enabled: "true"
  setup:
    enabled: true
    overwrite: false
    # Set `require_policy` to `false` when policies are set up outside of APM Server but referenced here.
    # Default value is `true`.
    require_policy: true
    mapping:
      - event_type: "error"
        policy_name: "apm-rollover-7-days"
        index_suffix: ""
      - event_type: "span"
        policy_name: "apm-rollover-7-days"
        index_suffix: ""
      - event_type: "transaction"
        policy_name: "apm-rollover-7-days"
        index_suffix: ""
      - event_type: "metric"
        policy_name: "apm-rollover-7-days"
        index_suffix: ""
      - event_type: "onboarding"
        policy_name: "apm-rollover-7-days"
        index_suffix: ""

    # Configured policies are added to pre-defined default policies.
    policies:
      - name: "apm-rollover-7-days"
        policy:
          phases:
            hot:
              actions:
                rollover:
                  max_size: "50gb"
                  max_age: "7d"
                set_priority:
                  priority: 100
            delete:
              min_age: "7d"
              actions:
                delete: {}

#================================= General =================================

queue:
  mem:
    events: 4096
    flush.min_events: 2048
    flush.timeout: 1s
max_procs: 4

#================================= Template =================================

setup.template.enabled: true
setup.template.name: "apm-%{[observer.version]}"
setup.template.pattern: "apm-%{[observer.version]}-*"
# Path to fields.yml file to generate the template.
#setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template.
#setup.template.overwrite: false
# Elasticsearch template settings.
setup.template.settings:
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
  index:
    number_of_shards: 1
    codec: best_compression
    number_of_routing_shards: 30
    mapping.total_fields.limit: 2000

#================================ Outputs ====================================

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  enabled: true
  compression_level: 9
  protocol: "http"
  worker: 1
  index: "apm-%{[observer.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "apm-%{[observer.version]}-sourcemap"
      when.contains:
        processor.event: "sourcemap"
    - index: "apm-%{[observer.version]}-error-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "error"
    - index: "apm-%{[observer.version]}-transaction-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "transaction"
    - index: "apm-%{[observer.version]}-span-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "span"
    - index: "apm-%{[observer.version]}-metric-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "metric"
    - index: "apm-%{[observer.version]}-onboarding-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "onboarding"

  max_retries: 3
  bulk_max_size: 60
  backoff.init: 1s
  backoff.max: 60s

#================================= Logging =================================

logging.level: info
logging.selectors: ["*"]
logging.to_syslog: false
logging.metrics.enabled: false
logging.metrics.period: 10s

#=============================== HTTP Endpoint ===============================

# Stats can be access through http://localhost:5066/stats. For pretty JSON output append ?pretty to the URL.
http.enabled: true
http.host: localhost
http.port: 5066

With this configuration I want 7 Days old APM data to be deleted. But when I use GET _ilm/policy in dev_tools console. but didn't find ILM policy apm-rollover-7-days created in the above config. Whats is wrong in my configuration ?

Versions :
apm-server:7.12.0
kibana:7.11.2
elasticsearch:7.11.2

Hi @rp346. Were you trying to update an existing policy or template? If so, ilm.setup.overwrite must be set to true, otherwise the default template and policies will not be overwritten.

Also, it looks like the "apm-rollover-7-days" policy you've defined will delete data after 1 day:

            delete:
              min_age: "1d"
              actions:
                delete: {}

I am trying to define new custom ILM policy, which will delete apm data older than 7d. This is a new policy thats why I have set ilm.setup.overwrite to false.

My concern now is why policy with name apm-rollover-7-days is not getting created.

If I create new ILM policy through Kibana UI, then it shows up in the output of GET _ilm/policy

(I set 1d just for testing without changing policy name instead of 7d. I have fixed this in above configuration. )

Hey again! Thanks for the extra info. I tried running your apm-server.yml code (thanks for supplying that, by the way) and I think this is a yaml formatting issue.

  1. ilm.*, rum.*, and agent config settings need to be nested under apm-server. You currently have all of these as top-level settings.
  2. The mapping for the onboarding document needs to be removed from the ILM settings. It's not supported (and not needed).

Fixing both of those issues loads the custom apm-rollover-7-days policy for me:

Here's what the fixed apm-server.yaml file looks like:

# Source : https://github.com/elastic/apm-server/blob/master/apm-server.docker.yml
################################ APM Server ################################
apm-server:
  host: "0.0.0.0:8200"
  frontend:
    enabled: false

#================================= APM Server - ILM Index Lifecycle Management =================================
  ilm:
    enabled: "true"
    setup:
      enabled: true
      overwrite: false
      # Set `require_policy` to `false` when policies are set up outside of APM Server but referenced here.
      # Default value is `true`.
      require_policy: true
      mapping:
        - event_type: "error"
          policy_name: "apm-rollover-7-days"
          index_suffix: ""
        - event_type: "span"
          policy_name: "apm-rollover-7-days"
          index_suffix: ""
        - event_type: "transaction"
          policy_name: "apm-rollover-7-days"
          index_suffix: ""
        - event_type: "metric"
          policy_name: "apm-rollover-7-days"
          index_suffix: ""

      # Configured policies are added to pre-defined default policies.
      policies:
        - name: "apm-rollover-7-days"
          policy:
            phases:
              hot:
                actions:
                  rollover:
                    max_size: "50gb"
                    max_age: "7d"
                  set_priority:
                    priority: 100
              delete:
                min_age: "7d"
                actions:
                  delete: {}

#================================= APM Server - RUM Real User Monitoring =================================

  # Enable Real User Monitoring (RUM) Support. By default RUM is disabled.
  # RUM does not support token based authorization. Enabled RUM endpoints will not require any authorization
  # token configured for other endpoints.
  rum:
    enabled: false

    event_rate:

      # Defines the maximum amount of events allowed to be sent to the APM Server RUM
      # endpoint per IP per second. Defaults to 300.
      limit: 300

      # An LRU cache is used to keep a rate limit per IP for the most recently seen IPs.
      # This setting defines the number of unique IPs that can be tracked in the cache.
      # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
      lru_size: 1000

    #-- General RUM settings

    # A list of permitted origins for real user monitoring.
    # User-agents will send an origin header that will be validated against this list.
    # An origin is made of a protocol scheme, host and port, without the url path.
    # Allowed origins in this setting can have * to match anything (eg.: http://*.example.com)
    # If an item in the list is a single '*', everything will be allowed.
    allow_origins : ['*']

    # A list of Access-Control-Allow-Headers to allow RUM requests, in addition to "Content-Type",
    # "Content-Encoding", and "Accept"
    allow_headers : []

    # Custom HTTP headers to add to RUM responses, e.g. for security policy compliance.
    #response_headers :
    #  X-My-Header: Contents of the header

    # Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes.
    # If the regexp matches, the stacktrace frame is considered to be a library frame.
    #library_pattern: "node_modules|bower_components|~"

    # Regexp to be matched against a stacktrace frame's `file_name`.
    # If the regexp matches, the stacktrace frame is not used for calculating error groups.
    # The default pattern excludes stacktrace frames that have a filename starting with '/webpack'
    #exclude_from_grouping: "^/webpack"

    # If a source map has previously been uploaded, source mapping is automatically applied.
    # to all error and transaction documents sent to the RUM endpoint.
    source_mapping:

      # Sourcemapping is enabled by default.
      enabled: true

      # Source maps are always fetched from Elasticsearch, by default using the output.elasticsearch configuration.
      # A different instance must be configured when using any other output.
      # This setting only affects sourcemap reads - the output determines where sourcemaps are written.
      elasticsearch:
        # Array of hosts to connect to.
        # Scheme and port can be left out and will be set to the default (`http` and `9200`).
        # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`.
        # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`.
        hosts: ["localhost:9200"]

        # Protocol - either `http` (default) or `https`.
        protocol: "http"

        # Authentication credentials - either API key or username/password.
        #api_key: "id:api_key"
        username: "admin"
        password: "changeme"

      # The `cache.expiration` determines how long a source map should be cached before fetching it again from Elasticsearch.
      # Note that values configured without a time unit will be interpreted as seconds.
      cache:
        expiration: 5m

      # Source maps are stored in a separate index.
      # If the default index pattern for source maps at 'outputs.elasticsearch.indices'
      # is changed, a matching index pattern needs to be specified here.
      index_pattern: "apm-*-sourcemap*"

#================================= APM Server - Agent Configuration =================================

  # When using APM agent configuration, information fetched from Kibana will be cached in memory for some time.
  # Specify cache key expiration via this setting. Default is 30 seconds.
  agent.config.cache.expiration: 30s
  kibana:
    enabled: true
    host: "kibana:5601"
    protocol: "http"

#================================= General =================================

queue:
  mem:
    events: 4096
    flush.min_events: 2048
    flush.timeout: 1s
max_procs: 4

#================================= Template =================================

setup.template.enabled: true
setup.template.name: "apm-%{[observer.version]}"
setup.template.pattern: "apm-%{[observer.version]}-*"
# Path to fields.yml file to generate the template.
#setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template.
#setup.template.overwrite: false
# Elasticsearch template settings.
setup.template.settings:
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
  index:
    number_of_shards: 1
    codec: best_compression
    number_of_routing_shards: 30
    mapping.total_fields.limit: 2000

#================================ Outputs ====================================

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  enabled: true
  compression_level: 9
  protocol: "http"
  worker: 1
  index: "apm-%{[observer.version]}-%{+yyyy.MM.dd}"
  indices:
    - index: "apm-%{[observer.version]}-sourcemap"
      when.contains:
        processor.event: "sourcemap"
    - index: "apm-%{[observer.version]}-error-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "error"
    - index: "apm-%{[observer.version]}-transaction-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "transaction"
    - index: "apm-%{[observer.version]}-span-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "span"
    - index: "apm-%{[observer.version]}-metric-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "metric"
    - index: "apm-%{[observer.version]}-onboarding-%{+yyyy.MM.dd}"
      when.contains:
        processor.event: "onboarding"

  max_retries: 3
  bulk_max_size: 60
  backoff.init: 1s
  backoff.max: 60s

#================================= Logging =================================

logging.level: info
logging.selectors: ["*"]
logging.to_syslog: false
logging.metrics.enabled: false
logging.metrics.period: 10s

#=============================== HTTP Endpoint ===============================

# Stats can be access through http://localhost:5066/stats. For pretty JSON output append ?pretty to the URL.
http.enabled: true
http.host: localhost
http.port: 5066

I hope this helps!