Trying to build index template from dynamicly generated template

Elastic Stack Elasticsearch
1

I want to apply lifecycle policy on some of the indexes.
So I think I need to create an index template, otherwise I will have to contiunally apply the policy to newly created indexes.
I pulled an index template from a dynamicly created index:

{
  "mappings": {
    "doc": {
      "properties": {
        "@timestamp": {
          "type": "date"
        },
        "@version": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ErrorCode": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Exception": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Exception type": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Instance name": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Memory usage": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Module": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Procedure": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Time": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Username": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "Version": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "beat": {
          "properties": {
            "hostname": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            },
            "version": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "fields": {
          "properties": {
            "logtype": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "host": {
          "properties": {
            "name": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "input": {
          "properties": {
            "type": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "log": {
          "properties": {
            "flags": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "message": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "offset": {
          "type": "long"
        },
        "prospector": {
          "properties": {
            "type": {
              "type": "text",
              "fields": {
                "keyword": {
                  "type": "keyword",
                  "ignore_above": 256
                }
              }
            }
          }
        },
        "source": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "tags": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        }
      }
    }
  }
}

I then attempted to 'qualify' it a bit, changing text fields to more proper fields, and removed unwanted keyword markings, and removed fields that I don't want.
That gave me this:

PUT _index_template/portraiterrors?include_type_name
{
  "version": 1,
  "priority": 100,
  "template": {
    "settings": {
      "number_of_replicas": 2,
      "number_of_shards": 2
    },
    "mappings": {
      "mappings": {
        "_source": {
          "enabled": true,
          "includes": [],
          "excludes": []
        },
        "_routing": {
          "required": false
        },
        "dynamic": false,
        "dynamic_templates": [],
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "integer",
            "index": true,
            "ignore_malformed": false,
            "coerce": true,
            "doc_values": true,
            "store": false
          },
          "ErrorCode": {
            "type": "integer",
            "index": true,
            "ignore_malformed": false,
            "coerce": true,
            "doc_values": true,
            "store": false
          },
          "Exception": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "Exception type": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "Instance name": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "Memory usage": {
            "type": "text"
          },
          "Module": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "Procedure": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "Time": {
            "type": "text"
          },
          "Username": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "Version": {
            "type": "text"
          },
          "beat": {
            "type": "object",
            "properties": {
              "hostname": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "name": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "version": {
                "type": "text"
              }
            }
          },
          "fields": {
            "type": "object",
            "properties": {
              "logtype": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "host": {
            "type": "object",
            "properties": {
              "name": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "message": {
            "type": "text"
          },
          "offset": {
            "type": "long"
          },
          "prospector": {
            "type": "object",
            "properties": {
              "type": {
                "type": "text",
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          }
        }
      }
    }
  },
  "index_patterns": [
    "portraiterrors-"
  ]
}

But I get these errors:

{
  "statusCode": 400,
  "error": "Bad Request",
  "message": "composable template [simulate_template_nnskae7_t-efcurjf6ecnw] template after composition is invalid",
  "attributes": {
    "error": {
      "root_cause": [
        {
          "type": "illegal_argument_exception",
          "reason": "composable template [simulate_template_nnskae7_t-efcurjf6ecnw] template after composition is invalid"
        }
      ],
      "type": "illegal_argument_exception",
      "reason": "composable template [simulate_template_nnskae7_t-efcurjf6ecnw] template after composition is invalid",
      "caused_by": {
        "type": "illegal_argument_exception",
        "reason": "invalid composite mappings for [simulate_template_nnskae7_t-efcurjf6ecnw]",
        "caused_by": {
          "type": "mapper_parsing_exception",
          "reason": "Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [mappings : {_routing={required=false}, _source={excludes=[], includes=[], enabled=true}, dynamic=false, dynamic_templates=[], properties={offset={type=long}, prospector={type=object, properties={type={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Exception type={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Time={type=text}, Instance name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, message={type=text}, Exception={type=text, fields={keyword={ignore_above=256, type=keyword}}}, @timestamp={type=date}, Username={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Version={type=text}, Memory usage={type=text}, @version={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, beat={type=object, properties={hostname={type=text, fields={keyword={ignore_above=256, type=keyword}}}, name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, version={type=text}}}, host={type=object, properties={name={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Procedure={type=text, fields={keyword={ignore_above=256, type=keyword}}}, ErrorCode={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, Module={type=text, fields={keyword={ignore_above=256, type=keyword}}}, fields={type=object, properties={logtype={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}}}]",
          "caused_by": {
            "type": "mapper_parsing_exception",
            "reason": "Root mapping definition has unsupported parameters:  [mappings : {_routing={required=false}, _source={excludes=[], includes=[], enabled=true}, dynamic=false, dynamic_templates=[], properties={offset={type=long}, prospector={type=object, properties={type={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Exception type={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Time={type=text}, Instance name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, message={type=text}, Exception={type=text, fields={keyword={ignore_above=256, type=keyword}}}, @timestamp={type=date}, Username={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Version={type=text}, Memory usage={type=text}, @version={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, beat={type=object, properties={hostname={type=text, fields={keyword={ignore_above=256, type=keyword}}}, name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, version={type=text}}}, host={type=object, properties={name={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Procedure={type=text, fields={keyword={ignore_above=256, type=keyword}}}, ErrorCode={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, Module={type=text, fields={keyword={ignore_above=256, type=keyword}}}, fields={type=object, properties={logtype={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}}}]"
          }
        }
      }
    },
    "causes": [
      "invalid composite mappings for [simulate_template_nnskae7_t-efcurjf6ecnw]",
      "Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [mappings : {_routing={required=false}, _source={excludes=[], includes=[], enabled=true}, dynamic=false, dynamic_templates=[], properties={offset={type=long}, prospector={type=object, properties={type={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Exception type={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Time={type=text}, Instance name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, message={type=text}, Exception={type=text, fields={keyword={ignore_above=256, type=keyword}}}, @timestamp={type=date}, Username={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Version={type=text}, Memory usage={type=text}, @version={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, beat={type=object, properties={hostname={type=text, fields={keyword={ignore_above=256, type=keyword}}}, name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, version={type=text}}}, host={type=object, properties={name={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Procedure={type=text, fields={keyword={ignore_above=256, type=keyword}}}, ErrorCode={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, Module={type=text, fields={keyword={ignore_above=256, type=keyword}}}, fields={type=object, properties={logtype={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}}}]",
      "Root mapping definition has unsupported parameters:  [mappings : {_routing={required=false}, _source={excludes=[], includes=[], enabled=true}, dynamic=false, dynamic_templates=[], properties={offset={type=long}, prospector={type=object, properties={type={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Exception type={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Time={type=text}, Instance name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, message={type=text}, Exception={type=text, fields={keyword={ignore_above=256, type=keyword}}}, @timestamp={type=date}, Username={type=text, fields={keyword={ignore_above=256, type=keyword}}}, Version={type=text}, Memory usage={type=text}, @version={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, beat={type=object, properties={hostname={type=text, fields={keyword={ignore_above=256, type=keyword}}}, name={type=text, fields={keyword={ignore_above=256, type=keyword}}}, version={type=text}}}, host={type=object, properties={name={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}, Procedure={type=text, fields={keyword={ignore_above=256, type=keyword}}}, ErrorCode={coerce=true, index=true, ignore_malformed=false, store=false, type=integer, doc_values=true}, Module={type=text, fields={keyword={ignore_above=256, type=keyword}}}, fields={type=object, properties={logtype={type=text, fields={keyword={ignore_above=256, type=keyword}}}}}}}]"
    ]
  }
}

Is that because I made a mistake in this, or because I can't apply it 'after the fact' with the corrections I made?

2

Hi, you can create or update an index template anytime (it will only affect future indices, not existing ones) . You have an extra level of mappings object. Just remove it (and also include_type_name unless you're on a very old version and should not be usable with _index_template ) and it will work !

1 Like
3

Ahh yes, thankyou Vincent!

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.