Filebeat ILM Policy Not Working

Elastic Stack Beats
,
1

Hi,

I am using Filebeat to harvest logs from servers. I want to set up a cleanup policy, and I have created one, but it is not taking effect. Can anyone help me figure out if I am missing something? I am using the same Filebeat configuration on two other servers as well.

path.data: /central-data/jfrog/filebeat-8.17.1-linux-x86_64/data
filebeat.inputs:
  - type: filestream
    enabled: true
    paths:
      - /central-data/jfrog/artifactory/var/log/artifactory-request-out.log
    processors:
      - dissect:
          tokenizer: "%{timestamp}|%{req_trace_id}|%{remote_repo_name}|%{req_user}|%{req_type}|%{remote_url}|%{res_status|integer}|%{req_content_length|long}|%{res_content_length|long}|%{duration|long}"
          target_prefix: ""
      - add_fields:
          target: ''
          fields:
            req_user: "anonymous"
          when:
            equals:
              req_user: ""
      - if:
          contains.req_user: "@"
        then:
          - dissect:
              tokenizer: "%{req_userr}@%{domain}"
              target_prefix: ""
        else:
          - add_fields:
              target: ''
              fields:
                domain: ""
    fields:
      record_type: "jfrt_remote_repo"
    fields_under_root: true

  # Artifactory Request
  - type: filestream
    id: artifactory_request
    enabled: true
    paths:
      - /central-data/jfrog/artifactory/var/log/artifactory-request.log
    processors:
      - dissect:
          tokenizer: "%{timestamp}|%{req_trace_id}|%{req_remote_address|ip}|%{req_user}|%{req_type}|%{req_url}|%{res_status|integer}|%{req_content_length|long}|%{res_content_length|long}|%{res_duration|long}|%{req_user_agent}"
          target_prefix: ""
      - if:
          or:
            - contains.req_remote_address: "127.0.0.1"
            - contains.req_user_agent: "JFrog-Router"
        then:
          - add_fields:
              target: ''
              fields:
                req_call_type: "internal"
        else:
          - add_fields:
              target: ''
              fields:
                req_call_type: "external"
      - if:
          contains.req_user: "@"
        then:
          - dissect:
              tokenizer: "%{req_userr}@%{domain}"
              target_prefix: ""
        else:
          - add_fields:
              target: ''
              fields:
                domain: ""
    fields:
      record_type: "jfrt"
    fields_under_root: true
  # Access Request
  - type: filestream
    id: access_request
    enabled: true
    paths:
      - /central-data/jfrog/artifactory/var/log/access-request.log
    processors:
      - dissect:
          tokenizer: "%{timestamp}|%{req_trace_id}|%{req_remote_address|ip}|%{req_user}|%{req_type}|%{req_url}|%{res_status|integer}|%{req_content_length|long}|%{res_content_length|long}|%{res_duration|long}|%{req_user_agent}"
          target_prefix: ""
      - if:
          or:
            - contains.req_remote_address: "127.0.0.1"
            - contains.req_user_agent: "JFrog-Router"
        then:
          - add_fields:
              target: ''
              fields:
                req_call_type: "internal"
        else:
          - add_fields:
              target: ''
              fields:
                req_call_type: "external"
      - if:
          contains.req_user: "@"
        then:
          - dissect:
              tokenizer: "%{req_userr}@%{domain}"
              target_prefix: ""
        else:
          - add_fields:
              target: ''
              fields:
                domain: ""
    fields:
      record_type: "jfac"
    fields_under_root: true

  - type: filestream
    id: audit_security_audit_log
    enabled: true
    paths:
      - /central-data/jfrog/artifactory/var/log/access-security-audit.log
    processors:
    - dissect:
        tokenizer: "%{timestamp}|%{trace_id}|%{user_ip}|%{user}|%{logged_principal}|%{entity_name}|%{event_typee}|%{eevent}|%{json_data}"
        field: "message"
        target_prefix: "dissected"

    - decode_json_fields:
        fields: ["dissected.json_data"]
        target: ""
        overwrite_keys: true
    fields:
      record_type: "jf_access_audit"
    fields_under_root: true

  # Artifactory Access Log
  - type: filestream
    id: artifactory_access
    enabled: true
    paths:
      - "/central-data/jfrog/artifactory/var/log/artifactory-access.log"
    processors:
      - dissect:
          tokenizer: "%{timestamp} [%{trace_id}] [%{action_response}] %{repo_path} %{msg} for client : %{username} %{ip_delimiter} %{ip|ip} [%{type}]"
          target_prefix: ""
      - drop_fields:
          fields: ["ip_delimiter"]
          ignore_missing: true
    fields:
      record_type: "jfrt-ac"
    fields_under_root: true

filebeat.modules:
  - module: nginx
    access:
      enabled: true
      var.paths: ["/var/log/nginx/*access.log*"]
    error:
      enabled: true
      var.paths: ["/var/log/nginx/error.log*"]

set.ilm.enabled: true
setup.ilm.policy_name: "compress_and_delete_policy"
#setup.dashboards.enabled: true
setup.kibana:
  host: "Kibana LB"

output.elasticsearch:
  hosts: ["ES LB"]
  username: "admin"
  password: "******"
  indices:
    - index: "active_jfrt_request_data-%{+yyyy.MM.dd}"
      when.or:
        - equals:
            record_type: "jfac"
        - equals:
            record_type: "jfrt"
    - index: "active_jfrt_outbound_data-%{+yyyy.MM.dd}"
      when.or:
        - equals:
            record_type: "jfrt_remote_repo"
    - index: "active_jfac_audit-%{+yyyy.MM.dd}"
      when.or:
        - equals:
            record_type: "jf_access_audit"
    - index: "active_jfac-%{+yyyy.MM.dd}"
      when.or:
        - equals:
            record_type: "jfrt-ac"
    - index: "active_nginx_logs-%{+yyyy.MM.dd}"

compress_and_delete_policy

put _ilm/policy/compress_and_delete_policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "7d"
          }
        }
      },
      "delete": {
        "min_age": "7d",
        "actions": {
          "delete": {
            "delete_searchable_snapshot": true
          }
        }
      }
    }
  }
}

Can anyone provide me the steps to configure the policy and apply to this indices.

Thanks!!