Possibility to use single index template for all application indices

Elastic Stack Elasticsearch
1

Hi Team,

I have multiple applications running and each has their application logs captured by filebeat and it sends to logstash.

Below is the logstash configuration /etc/logstash/conf.d/logstash.conf file. ( showing only for 3 applications )

input {
  beats {
    port => 5044
  }
}

filter {
if [log_type] == "portal-api_app_server" and [app_id] == "node"
  {
    grok { match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:json_message}"  } } json { source =>  "json_message" }
    mutate {
             replace => {
               "[type]" => "portal-api_app_server"
             }
           }
  }
if [log_type] == "federate_ping_server" and [app_id] == "pf"
  {
    mutate { gsub => ["message","\|"," "] } grok { patterns_dir => ["/etc/logstash/patterns"] match => { "message" => "%{MY_DATE_PATTERN:timestamp}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{UUID:ConsentID}%{SPACE}%{WORD:TransactionID}%{SPACE}%{WORD:TraceID}%{SPACE}%{GREEDYDATA:messagetext}" } }
    mutate {
             replace => {
               "[type]" => "federate_ping_server"
             }
           }
  }
if [log_type] == "directory_ping_server" and [app_id] == "pd"
  {
    mutate { gsub => ["message","\|"," "] } grok { patterns_dir => ["/etc/logstash/patterns"] match => { "message" => "%{MY_DATE_PATTERN:timestamp}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{UUID:ConsentID}%{SPACE}%{WORD:TransactionID}%{SPACE}%{WORD:TraceID}%{SPACE}%{GREEDYDATA:messagetext}" } }
    mutate {
             replace => {
               "[type]" => "directory_ping_server"
             }
           }
  }
}  
output {
 if [log_type] == "portal-api_app_server" {
  elasticsearch {
    hosts => ['http://10.10.10.242:9200']
        user => elastic
    password => "${es_pwd}"
     index => "portal-api"
     template_name => "portal-api"
     template_overwrite => "false"
      }
 }
   if [log_type] == "federate_ping_server" {
  elasticsearch {
    hosts => ['http://10.10.10.242:9200']
        user => elastic
    password => "${es_pwd}"
     index => "federate"
     template_name => "federate"
     template_overwrite => "false"
      }
 }
 if [log_type] == "directory_ping_server" {
  elasticsearch {
    hosts => ['http://10.10.10.242:9200']
        user => elastic
    password => "${es_pwd}"
     index => "directory"
     template_name => "directory"
     template_overwrite => "false"
      }
 }
 elasticsearch {
    hosts => ['http://10.10.10.242:9200']
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM}"
    user => elastic
    password => "${es_pwd}"
  }
}

Below is the index_template for above three,

 {
      "name" : "portal-api_template",
      "index_template" : {
        "index_patterns" : [
          "portal-api*"
        ],
        "template" : {
          "settings" : {
            "index" : {
              "lifecycle" : {
                "name" : "testpolicy",
                "rollover_alias" : "portal-api"
              },
              "number_of_shards" : "1",
              "number_of_replicas" : "0"
            }
          },
          "mappings" : {
            "_routing" : {
              "required" : false
            },
            "dynamic_date_formats" : [
              "strict_date_optional_time",
              "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
            ],
            "numeric_detection" : true,
            "_source" : {
              "excludes" : [ ],
              "includes" : [ ],
              "enabled" : true
            },
            "dynamic" : true,
            "dynamic_templates" : [ ],
            "date_detection" : true
          },
          "aliases" : {
            "aliases" : { }
          }
        },
        "composed_of" : [ ],
        "priority" : 600,
        "version" : 2
      }
    },

{
      "name" : "federate_template",
      "index_template" : {
        "index_patterns" : [
          "federate*"
        ],
        "template" : {
          "settings" : {
            "index" : {
              "lifecycle" : {
                "name" : "testpolicy",
                "rollover_alias" : "federate"
              },
              "number_of_shards" : "1",
              "number_of_replicas" : "0"
            }
          },
          "mappings" : {
            "_routing" : {
              "required" : false
            },
            "dynamic_date_formats" : [
              "strict_date_optional_time",
              "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
            ],
            "numeric_detection" : true,
            "_source" : {
              "excludes" : [ ],
              "includes" : [ ],
              "enabled" : true
            },
            "dynamic" : true,
            "dynamic_templates" : [ ],
            "date_detection" : true
          },
          "aliases" : {
            "aliases" : { }
          }
        },
        "composed_of" : [ ],
        "priority" : 600,
        "version" : 2
      }
    },

{
      "name" : "directory_template",
      "index_template" : {
        "index_patterns" : [
          "directory*"
        ],
        "template" : {
          "settings" : {
            "index" : {
              "lifecycle" : {
                "name" : "testpolicy",
                "rollover_alias" : "directory"
              },
              "number_of_shards" : "1",
              "number_of_replicas" : "0"
            }
          },
          "mappings" : {
            "_routing" : {
              "required" : false
            },
            "dynamic_date_formats" : [
              "strict_date_optional_time",
              "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
            ],
            "numeric_detection" : true,
            "_source" : {
              "excludes" : [ ],
              "includes" : [ ],
              "enabled" : true
            },
            "dynamic" : true,
            "dynamic_templates" : [ ],
            "date_detection" : true
          },
          "aliases" : {
            "aliases" : { }
          }
        },
        "composed_of" : [ ],
        "priority" : 600,
        "version" : 2
      }
    },

As you can see each application is having its own index_template. I created these multiple index_template as each index name is different and therefore different index_patterns, can i use single index_template (as other settings are same for all) for all applications? Is there any possibility to mention multiple index_patterns to match multiple indices?

Thanks,

2

Hi @prat

Yes See Here

You will notice that index_patterns takes an array so yes you can just make a comma separated list of index patterns to match.

From the docs

index_patterns
(Required, array of strings) Array of wildcard (*) expressions used to match the names of data streams and indices during creation.

Elasticsearch includes several built-in index templates. To avoid naming collisions with these templates, see Avoid index pattern collisions.

3

Hi @stephenb, Thanks for prompt response.

Ok so i only need one index_template and need to mentioned all index_patterns in an array and need to mentioned the same index_template to each index in output section of logstash config. Is this correct?

Thanks,

4

Hi @stephenb,

Could you please confirm.

Thanks,

5

Hi @prat

Yes, Perhaps give it a try... :slight_smile:

If you are using ILM you will need to pay attention to the write aliases etc, if you are not then you should be fine.

You could also accomplish the same with composable templates with all the mapping details etc in components and then the specifics of the index Patterns, ILM policies in the actual _index_template that is a modular approach.

See Here

So you have options, I would try and see which one works for you best.

6

Hi @stephenb,

Thanks for your update.

Yes i am using ILM, aliases in index_template. Below is the example of index_template for one the application which has ILM, alias settings.

Below is the actual file which _index_template API is using and applying through PUT method.

Not sure why alias and ILM things are not shown in above federate_template output. I have taken that output from kibana using GET _index_template/federate_template.

# cat federate_template.json 

{
  "version": 2,
  "priority": 600,
  "template": {
    "settings": {
      "index.number_of_shards": 1,
      "index.number_of_replicas": 0,
      "index.lifecycle.name": "testpolicy",      
      "index.lifecycle.rollover_alias": "federate" 
    },
    "mappings": {
      "_source": {
        "enabled": true,
        "includes": [],
        "excludes": []
      },
      "_routing": {
        "required": false
      },
      "dynamic": true,
      "numeric_detection": true,
      "date_detection": true,
      "dynamic_date_formats": [
        "strict_date_optional_time",
        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
      ],
      "dynamic_templates": []
    },
    "aliases": {
      "aliases": {
        "alias1": {
          "aliases": "federate"
        }
      }
    }
  },
  "index_patterns": [
    "federate*"
  ]
}

If all these index_templates have different aliases, index.lifecycle.rollover_alias , how can we achieve that in single index_template ?

I have not used component_template as i do not have any mappings requirement.

Thanks,

7

If you have different ILM policies and aliases then you'll need separate templates.

Your templates aren't too complicated so I'm not really sure what the issue is of having three separate templates.

Since you have different settings like ILM and alias then they require separate templates.

BTW You do have a mapping section... If this was complex then you could use the _component_template for the mapping section and then only parent template would be different where you define the ILM a list and some of the other parameters.

8

Hi @stephenb,

There are around 8-10 applications and i am just trying to reduce the duplicate configuration.

but even in this case, If i understood above correctly, total no. index_templates still remain same right..? i.e ~8-10 index_templates.

Is there any way to reduce below configuration? (may be by using array?) using multiple-actions mentioned over here?

I am using individual files for each applications to apply write_index: true.

portal-api_alias.json

{
  "aliases": {
    "portal-api":{
      "is_write_index": true 
    }
  }
}

federate_alias.json

{
  "aliases": {
    "federate":{
      "is_write_index": true 
    }
  }
}

and need to create below tasks for each above alias files

- name: create portal-api initial index write true
  uri:
   method: PUT
   url: "http://{{ elasticsearch1_private_ip }}:{{ elasticsearch_port }}/portal-api-000001?pretty"
   body: "{{ lookup('file', '{{ store }}/portal-api.json') }}"
   body_format: json
   user: "{{ elasticsearch_username }}"
   password: "{{ elasticsearch_password }}"
   status_code: 200

- name: create federate initial index write true
  uri:
   method: PUT
   url: "http://{{ elasticsearch1_private_ip }}:{{ elasticsearch_port }}/federate-000001?pretty"
   body: "{{ lookup('file', '{{ store }}/federate_alias.json') }}"
   body_format: json
   user: "{{ elasticsearch_username }}"
   password: "{{ elasticsearch_password }}"
   status_code: 200

Thanks,

9

Perhaps someone else can comment but I don't think using aliases is going to solve that problem.

As far as I know, In the end if you have N different Indices, that you want N write/rollover aliases and N ILM Policies you are going to need separate templates, there is no "ordered arrays" or something like that in the settings portion of a _index_template.

Please file a feature request if you like, perhaps it would be useful for other folks as well.

Also there are plenty of users / customers that automate the deployment of templates with CI / Pipelines, scripting or other automation technologies.. some have many more than 8-10 templates.

Perhaps someone else will have a different idea.

10

Hi @stephenb,

Thanks for your reply. I won't stretch the case anymore.

Not sure which problem you are highlighting?

My question is not about automation etc..

I am creating below aliases as its mentioned in below link,

bootstrap an initial index and designate it as the write index for the rollover alias specified in your index template.

Like you said there is a way to include multiple index_patterns in an array inside single index_template. Is there any way to include below aliases inside an array to reduce no. of individual files?

As you can see below for each index i am creating individual alias file.

portal-api_alias.json

{
  "aliases": {
    "portal-api":{
      "is_write_index": true 
    }
  }
}

federate_alias.json

{
  "aliases": {
    "federate":{
      "is_write_index": true 
    }
  }
}

ad-spa_alias.json

{
  "aliases": {
    "ad-spa":{
      "is_write_index": true 
    }
  }
}

Thanks,

11

Hi @stephenb,

Can you please update on last reply?

Specifically for below,

Thanks,

12

Not in the way I think you are trying to do.

You can I add multiple aliases through the component see here

# Component template with index aliases
PUT _component_template/my-aliases
{
  "template": {
    "aliases": {
      "my-alias": {}
    }
  }
}

# Index template with index aliases
PUT _index_template/my-index-template
{
  "index_patterns": [
    "my-index-*"
  ],
  "composed_of": [
    "my-aliases",
    "my-mappings",
    "my-settings"
  ],
  "template": {
    "aliases": {
      "yet-another-alias": {}
    }
  }
}

It seems that you're looking for an ordered array of index patterns and then an order array aliases that correlate 1 to 1 with your index pattern... I don't see / think that capability exists.

Perhaps if you try adding the aliases through the components you may get what you want, but I don't think that is what you want.

I still don't think that's going to accomplish what you are looking for but perhaps it will, perhaps try it?

13

I am creating those individual alias for respective indices for making them write_index.

Yes. As you said using ordered array it is possible for index patterns but even that is not possible when we have ILM etc.. settings.

So was looking similar thing for write alias also.

Currently, I am applying multiple index template (as it not be single index template as concluded above) and each one of them defining its alias like below, (so multiple .json files for this)

    .
    .
    "aliases": {
      "aliases": {
         "alias1": {
            "aliases": "ad-spa"
         }
       }
    }

and then applying write_alias for each alias (so multiple .json files for this)

{
  "aliases": {
    "ad-spa":{
      "is_write_index": true 
    }
  }
}

so my last question was on reducing this no. of individual write_alias files if possible,

Can we do it like below?

so alias will still first get defined in index template and for making them write_index, can we defined them in one single file?

POST _aliases
{
  "actions": [
    {
      "add": {
        "index": "federate",
        "alias": "federate",
        "is_write_index": true
      }
    },
    {
      "add": {
        "index": "portal-api",
        "alias": "portal-api",
        "is_write_index": true
    },
    {
      "add": {
        "index": "ad-spa",
        "alias": "ad-spa",
        "is_write_index": true
      }
    }
  ]
}
14

First the index and alias name can not be the same in your sample above...

But yes you can create as many aliases as you like in a single POST _aliases action

You could also do as I suggested above put all the aliases in a single component and include that as part of the composite.

See Here

In fact it looks like perhaps it can be parameterized ...

Wow I just tried this.... and it works... I think this is may be what you want! Each of the indices got their own correct aliases! Huh .... TIL! (Today I Learned!)

PUT _component_template/my_alias_template
{
  "template": {
    "aliases": {
      "{index}-writer-alias": {
        "is_write_index": true
      }
    }
  }
}

PUT _index_template/discuss_1
{
  "index_patterns": [
    "disc*",
    "bar*"
  ],
  "template": {
    "settings": {
      "number_of_shards": 1
    },
    "mappings": {
      "_source": {
        "enabled": true
      },
      "properties": {
        "host_name": {
          "type": "keyword"
        },
        "created_at": {
          "type": "date",
          "format": "EEE MMM dd HH:mm:ss Z yyyy"
        }
      }
    }
  },
  "priority": 500,
  "composed_of": [
    "my_alias_template"
  ],
  "version": 3,
  "_meta": {
    "description": "my custom"
  }
}

DELETE discuss

POST discuss/_doc
{
  "log.level": "INFO",
  "log.logger": "org.elasticsearch.bootstrap.Bootstrap",
  "log.file.path": "/var/log/fun-times.log"
}

GET discuss  <!-- Show Correct Alias 

POST bar/_doc
{
  "log.level": "INFO",
  "log.logger": "org.elasticsearch.bootstrap.Bootstrap",
  "log.file.path": "/var/log/fun-times.log"
}

GET bar  <!-- Show Correct Alias
15

You can also probably collapse that into a single template.

PUT _index_template/discuss_1
{
  "index_patterns": [
    "disc*",
    "bar*"
  ],
  "template": {
    "settings": {
      "number_of_shards": 1
    },
    "aliases": {
      "{index}-writer-alias": {
        "is_write_index": true
      }
    },
    "mappings": {
      "_source": {
        "enabled": true
      },
      "properties": {
        "host_name": {
          "type": "keyword"
        },
        "created_at": {
          "type": "date",
          "format": "EEE MMM dd HH:mm:ss Z yyyy"
        }
      }
    }
  },
  "priority": 500,
  "version": 3,
  "_meta": {
    "description": "my custom"
  }
}
16

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.