Possibility to use single index template for all application indices

Hi Team,

I have multiple applications running and each has their application logs captured by filebeat and it sends to logstash.

Below is the logstash configuration /etc/logstash/conf.d/logstash.conf file. ( showing only for 3 applications )

input {
  beats {
    port => 5044
  }
}

filter {
if [log_type] == "portal-api_app_server" and [app_id] == "node"
  {
    grok { match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:json_message}"  } } json { source =>  "json_message" }
    mutate {
             replace => {
               "[type]" => "portal-api_app_server"
             }
           }
  }
if [log_type] == "federate_ping_server" and [app_id] == "pf"
  {
    mutate { gsub => ["message","\|"," "] } grok { patterns_dir => ["/etc/logstash/patterns"] match => { "message" => "%{MY_DATE_PATTERN:timestamp}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{UUID:ConsentID}%{SPACE}%{WORD:TransactionID}%{SPACE}%{WORD:TraceID}%{SPACE}%{GREEDYDATA:messagetext}" } }
    mutate {
             replace => {
               "[type]" => "federate_ping_server"
             }
           }
  }
if [log_type] == "directory_ping_server" and [app_id] == "pd"
  {
    mutate { gsub => ["message","\|"," "] } grok { patterns_dir => ["/etc/logstash/patterns"] match => { "message" => "%{MY_DATE_PATTERN:timestamp}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{UUID:ConsentID}%{SPACE}%{WORD:TransactionID}%{SPACE}%{WORD:TraceID}%{SPACE}%{GREEDYDATA:messagetext}" } }
    mutate {
             replace => {
               "[type]" => "directory_ping_server"
             }
           }
  }
}  
output {
 if [log_type] == "portal-api_app_server" {
  elasticsearch {
    hosts => ['http://10.10.10.242:9200']
        user => elastic
    password => "${es_pwd}"
     index => "portal-api"
     template_name => "portal-api"
     template_overwrite => "false"
      }
 }
   if [log_type] == "federate_ping_server" {
  elasticsearch {
    hosts => ['http://10.10.10.242:9200']
        user => elastic
    password => "${es_pwd}"
     index => "federate"
     template_name => "federate"
     template_overwrite => "false"
      }
 }
 if [log_type] == "directory_ping_server" {
  elasticsearch {
    hosts => ['http://10.10.10.242:9200']
        user => elastic
    password => "${es_pwd}"
     index => "directory"
     template_name => "directory"
     template_overwrite => "false"
      }
 }
 elasticsearch {
    hosts => ['http://10.10.10.242:9200']
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM}"
    user => elastic
    password => "${es_pwd}"
  }
}

Below is the index_template for above three,

 {
      "name" : "portal-api_template",
      "index_template" : {
        "index_patterns" : [
          "portal-api*"
        ],
        "template" : {
          "settings" : {
            "index" : {
              "lifecycle" : {
                "name" : "testpolicy",
                "rollover_alias" : "portal-api"
              },
              "number_of_shards" : "1",
              "number_of_replicas" : "0"
            }
          },
          "mappings" : {
            "_routing" : {
              "required" : false
            },
            "dynamic_date_formats" : [
              "strict_date_optional_time",
              "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
            ],
            "numeric_detection" : true,
            "_source" : {
              "excludes" : [ ],
              "includes" : [ ],
              "enabled" : true
            },
            "dynamic" : true,
            "dynamic_templates" : [ ],
            "date_detection" : true
          },
          "aliases" : {
            "aliases" : { }
          }
        },
        "composed_of" : [ ],
        "priority" : 600,
        "version" : 2
      }
    },
{
      "name" : "federate_template",
      "index_template" : {
        "index_patterns" : [
          "federate*"
        ],
        "template" : {
          "settings" : {
            "index" : {
              "lifecycle" : {
                "name" : "testpolicy",
                "rollover_alias" : "federate"
              },
              "number_of_shards" : "1",
              "number_of_replicas" : "0"
            }
          },
          "mappings" : {
            "_routing" : {
              "required" : false
            },
            "dynamic_date_formats" : [
              "strict_date_optional_time",
              "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
            ],
            "numeric_detection" : true,
            "_source" : {
              "excludes" : [ ],
              "includes" : [ ],
              "enabled" : true
            },
            "dynamic" : true,
            "dynamic_templates" : [ ],
            "date_detection" : true
          },
          "aliases" : {
            "aliases" : { }
          }
        },
        "composed_of" : [ ],
        "priority" : 600,
        "version" : 2
      }
    },
{
      "name" : "directory_template",
      "index_template" : {
        "index_patterns" : [
          "directory*"
        ],
        "template" : {
          "settings" : {
            "index" : {
              "lifecycle" : {
                "name" : "testpolicy",
                "rollover_alias" : "directory"
              },
              "number_of_shards" : "1",
              "number_of_replicas" : "0"
            }
          },
          "mappings" : {
            "_routing" : {
              "required" : false
            },
            "dynamic_date_formats" : [
              "strict_date_optional_time",
              "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
            ],
            "numeric_detection" : true,
            "_source" : {
              "excludes" : [ ],
              "includes" : [ ],
              "enabled" : true
            },
            "dynamic" : true,
            "dynamic_templates" : [ ],
            "date_detection" : true
          },
          "aliases" : {
            "aliases" : { }
          }
        },
        "composed_of" : [ ],
        "priority" : 600,
        "version" : 2
      }
    },

As you can see each application is having its own index_template. I created these multiple index_template as each index name is different and therefore different index_patterns, can i use single index_template (as other settings are same for all) for all applications? Is there any possibility to mention multiple index_patterns to match multiple indices?

Thanks,

Hi @prat

Yes See Here

You will notice that index_patterns takes an array so yes you can just make a comma separated list of index patterns to match.

From the docs

index_patterns
(Required, array of strings) Array of wildcard (*) expressions used to match the names of data streams and indices during creation.

Elasticsearch includes several built-in index templates. To avoid naming collisions with these templates, see Avoid index pattern collisions.

Hi @stephenb, Thanks for prompt response.

Ok so i only need one index_template and need to mentioned all index_patterns in an array and need to mentioned the same index_template to each index in output section of logstash config. Is this correct?

Thanks,

Hi @stephenb,

Could you please confirm.

Thanks,

Hi @prat

Yes, Perhaps give it a try... :slight_smile:

If you are using ILM you will need to pay attention to the write aliases etc, if you are not then you should be fine.

You could also accomplish the same with composable templates with all the mapping details etc in components and then the specifics of the index Patterns, ILM policies in the actual _index_template that is a modular approach.

See Here

So you have options, I would try and see which one works for you best.

Hi @stephenb,

Thanks for your update.

Yes i am using ILM, aliases in index_template. Below is the example of index_template for one the application which has ILM, alias settings.

Below is the actual file which _index_template API is using and applying through PUT method.

Not sure why alias and ILM things are not shown in above federate_template output. I have taken that output from kibana using GET _index_template/federate_template.

# cat federate_template.json

{
  "version": 2,
  "priority": 600,
  "template": {
    "settings": {
      "index.number_of_shards": 1,
      "index.number_of_replicas": 0,
      "index.lifecycle.name": "testpolicy",      
      "index.lifecycle.rollover_alias": "federate" 
    },
    "mappings": {
      "_source": {
        "enabled": true,
        "includes": [],
        "excludes": []
      },
      "_routing": {
        "required": false
      },
      "dynamic": true,
      "numeric_detection": true,
      "date_detection": true,
      "dynamic_date_formats": [
        "strict_date_optional_time",
        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
      ],
      "dynamic_templates": []
    },
    "aliases": {
      "aliases": {
        "alias1": {
          "aliases": "federate"
        }
      }
    }
  },
  "index_patterns": [
    "federate*"
  ]
}

If all these index_templates have different aliases, index.lifecycle.rollover_alias , how can we achieve that in single index_template ?

I have not used component_template as i do not have any mappings requirement.

Thanks,

If you have different ILM policies and aliases then you'll need separate templates.

Your templates aren't too complicated so I'm not really sure what the issue is of having three separate templates.

Since you have different settings like ILM and alias then they require separate templates.

BTW You do have a mapping section... If this was complex then you could use the _component_template for the mapping section and then only parent template would be different where you define the ILM a list and some of the other parameters.

Hi @stephenb,

There are around 8-10 applications and i am just trying to reduce the duplicate configuration.

but even in this case, If i understood above correctly, total no. index_templates still remain same right..? i.e ~8-10 index_templates.

Is there any way to reduce below configuration? (may be by using array?) using multiple-actions mentioned over here?

I am using individual files for each applications to apply write_index: true.

portal-api_alias.json

{
  "aliases": {
    "portal-api":{
      "is_write_index": true 
    }
  }
}

federate_alias.json

{
  "aliases": {
    "federate":{
      "is_write_index": true 
    }
  }
}

and need to create below tasks for each above alias files

- name: create portal-api initial index write true
  uri:
   method: PUT
   url: "http://{{ elasticsearch1_private_ip }}:{{ elasticsearch_port }}/portal-api-000001?pretty"
   body: "{{ lookup('file', '{{ store }}/portal-api.json') }}"
   body_format: json
   user: "{{ elasticsearch_username }}"
   password: "{{ elasticsearch_password }}"
   status_code: 200

- name: create federate initial index write true
  uri:
   method: PUT
   url: "http://{{ elasticsearch1_private_ip }}:{{ elasticsearch_port }}/federate-000001?pretty"
   body: "{{ lookup('file', '{{ store }}/federate_alias.json') }}"
   body_format: json
   user: "{{ elasticsearch_username }}"
   password: "{{ elasticsearch_password }}"
   status_code: 200

Thanks,

Perhaps someone else can comment but I don't think using aliases is going to solve that problem.

As far as I know, In the end if you have N different Indices, that you want N write/rollover aliases and N ILM Policies you are going to need separate templates, there is no "ordered arrays" or something like that in the settings portion of a _index_template.

Please file a feature request if you like, perhaps it would be useful for other folks as well.

Also there are plenty of users / customers that automate the deployment of templates with CI / Pipelines, scripting or other automation technologies.. some have many more than 8-10 templates.

Perhaps someone else will have a different idea.

Hi @stephenb,

Thanks for your reply. I won't stretch the case anymore.

Not sure which problem you are highlighting?

My question is not about automation etc..

I am creating below aliases as its mentioned in below link,

bootstrap an initial index and designate it as the write index for the rollover alias specified in your index template.

Like you said there is a way to include multiple index_patterns in an array inside single index_template. Is there any way to include below aliases inside an array to reduce no. of individual files?

As you can see below for each index i am creating individual alias file.

portal-api_alias.json

{
  "aliases": {
    "portal-api":{
      "is_write_index": true 
    }
  }
}

federate_alias.json

{
  "aliases": {
    "federate":{
      "is_write_index": true 
    }
  }
}

ad-spa_alias.json

{
  "aliases": {
    "ad-spa":{
      "is_write_index": true 
    }
  }
}

Thanks,

Hi @stephenb,

Can you please update on last reply?

Specifically for below,

Thanks,