Filebeat process different log paths and write data to seperate index,Without use of logstash and follow ILM/rollover alias defined in template

Elastic Stack Beats
12

@PRASHANT_MEHTA

Looks at this carefully this is minimal but does everything you ask.

This is the order you need to do this as well ...

BTW timeseries indices are of the pattern

my-index-yyyy.mm.dd-000001

# Create Common ILM Policy
PUT _ilm/policy/mis-monitoring-common-policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_age": "1d",
            "max_size": "50gb"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "delete": {
        "min_age": "7d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

# Create template with correct, pattern, alias and ILM Policy
PUT _index_template/mis-log
{
  "index_patterns": ["mis-logs-*"],
  "template": {
    "settings": {
      "index": {
        "lifecycle": {
          "name": "mis-monitoring-common-policy",
          "rollover_alias": "mis-log"
        },
        "number_of_shards": "1",
        "number_of_replicas": "0"
      }
    }
  }
}

# Create template with correct, pattern, alias and ILM Policy
PUT _index_template/mis-monitoring-usecases
{
  "index_patterns": ["mis-monitoring-usecases-*"],
  "template": {
    "settings": {
      "index": {
        "lifecycle": {
          "name": "mis-monitoring-common-policy",
          "rollover_alias": "mis-monitoring-usecases"
        },
        "number_of_shards": "1",
        "number_of_replicas": "0"
      }
    }
  }
}

# Create Initial Managed index 
PUT mis-logs-2023.05.06-000001
{
  "aliases": {
    "mis-logs":{
      "is_write_index": true 
    }
  }
}

# Create Initial Managed index 
PUT mis-monitoring-usecases-2023.05.06-000001
{
  "aliases": {
    "mis-monitoring-usecases":{
      "is_write_index": true 
    }
  }
}

My complete working filebeat.yml this is not a snippet

filebeat.inputs:

- type: log
  enabled: true
  paths:
    - /Users/sbrown/workspace/sample-data/discuss/multiple-logs/logs/*.log
  index: mis-monitoring-usecases

- type: log
  enabled: true
  paths:
    - /Users/sbrown/workspace/sample-data/discuss/multiple-logs/otherlogs/*.log
  index: mis-logs

# ======================= Elasticsearch template setting =======================

setup.ilm.enabled: false
setup.template.enabled: false

setup.kibana:
# ---------------------------- Elasticsearch Output ----------------------------
output.elasticsearch:
  # Array of hosts to connect to.
  hosts: ["localhost:9200"]

./filebeat setup -e

filebeat -e

Results

GET _cat/indices/mis-*/?v
health status index                                     uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   mis-monitoring-usecases-2023.05.06-000001 v6zg9FSwRSK8B6LzzOg8Pw   1   0         11            0     16.6kb         16.6kb
green  open   mis-logs-2023.05.06-000001                Gx8CtLOERs-zjQXElVaCLA   1   0         11            0     16.7kb         16.7kb

GET mis-logs-2023.05.06-000001/_ilm/explain
{
  "indices" : {
    "mis-logs-2023.05.06-000001" : {
      "index" : "mis-logs-2023.05.06-000001",
      "managed" : true,
      "policy" : "mis-monitoring-common-policy",
      "lifecycle_date_millis" : 1683418881220,
      "age" : "1.05m",
      "phase" : "hot",
      "phase_time_millis" : 1683418881304,
      "action" : "unfollow",
      "action_time_millis" : 1683418881435,
      "step" : "wait-for-follow-shard-tasks",
      "step_time_millis" : 1683418881522,
      "phase_execution" : {
        "policy" : "mis-monitoring-common-policy",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "1d"
            },
            "set_priority" : {
              "priority" : 100
            }
          }
        },
        "version" : 2,
        "modified_date_in_millis" : 1683418881161
      }
    }
  }
}

Test rollover it works...

POST mis-logs/_rollover
{
  "acknowledged" : true,
  "shards_acknowledged" : true,
  "old_index" : "mis-logs-2023.05.06-000001",
  "new_index" : "mis-logs-2023.05.06-000002",
  "rolled_over" : true,
  "dry_run" : false,
  "conditions" : { }
}