Elasticsearch Index mappings( version 6.5.4)

Elastic Stack Logstash
1

I am using filebeat to fetch logs, logstash to filter, and pushing them to elasticsearch.
I have created an index on elasticsearch import-export-logger.

This id mapping of import-export-logger.

{
  "import-export-logger" : {
    "aliases" : { },
    "mappings" : {
      "_doc" : {
        "properties" : {
          "@timestamp" : {
            "type" : "date"
          },
          "currentMilestone" : {
            "type" : "integer"
          },
          "date" : {
            "type" : "date",
            "format" : "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "description" : {
            "type" : "text"
          },
          "exportFrequency" : {
            "type" : "keyword"
          },
          "exportQueueType" : {
            "type" : "keyword"
          },
          "facilityCode" : {
            "type" : "keyword"
          },
          "failedImportMilestone" : {
            "type" : "integer"
          },
          "fileURL" : {
            "type" : "text"
          },
          "jobCreatedDate" : {
            "type" : "date",
            "format" : "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "jobID" : {
            "type" : "keyword"
          },
          "jobName" : {
            "type" : "keyword"
          },
          "jobType" : {
            "type" : "keyword"
          },
          "message" : {
            "type" : "text"
          },
          "requestID" : {
            "type" : "keyword"
          },
          "status" : {
            "type" : "keyword"
          },
          "successfulImportMilestone" : {
            "type" : "integer"
          },
          "tenantCode" : {
            "type" : "keyword"
          },
          "totalMilestone" : {
            "type" : "integer"
          },
          "userName" : {
            "type" : "keyword"
          }
        }
      }
    },
    "settings" : {
      "index" : {
        "creation_date" : "1657625688183",
        "number_of_shards" : "5",
        "number_of_replicas" : "1",
        "uuid" : "TWK_Ha4ZR9SUSDCwXyLeGA",
        "version" : {
          "created" : "6050499"
        },
        "provided_name" : "import-export-logger"
      }
    }
  }
}

This is the output section of logstash config file

elasticsearch {
		hosts => [ "localhost:9200" ]
		index => "import-export-logger-%{index-name}"
		http_compression => true
	}

Here I want elasticsearch to create a new index but with the mapping of import-export-logger index rather tan creating new mapping on its own.

Mapping for output index import-export-logger-2020-11-06 is

{
  "import-export-logger-2020-11-06" : {
    "mappings" : {
      "doc" : {
        "properties" : {
          "@timestamp" : {
            "type" : "date"
          },
          "beat" : {
            "properties" : {
              "hostname" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "name" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              },
              "version" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              }
            }
          },
          "created_timestamp" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "currentMilestone" : {
            "type" : "long"
          },
          "current_timestamp" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "description" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "exportFrequency" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "exportQueueType" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "facilityCode" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "failedImportMilestone" : {
            "type" : "long"
          },
          "fields" : {
            "properties" : {
              "target_index" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "type" : "keyword",
                    "ignore_above" : 256
                  }
                }
              }
            }
          },
          "fileURL" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "index-name" : {
            "type" : "date"
          },
          "jobID" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "jobName" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "jobType" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "message" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "requestID" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "source" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "status" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "successfulImportMilestone" : {
            "type" : "long"
          },
          "tags" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "tenantCode" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "totalMilestone" : {
            "type" : "long"
          },
          "userName" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          }
        }
      }
    }
  }
}

Can anyone help me with how should I configure my files to get the desired mapping

2

Hi @gyrao_72 Welcome to the community!

First 6.5 is incredibly old and EOL and you should update as a matter of urgency.

2nd Did you create an index template?

You did not show the actual command you use to create your template did you just create a single mapping... unclear what you did..

What you need to do is great an Index Template, which matches the index name and applies the template

something like.... this is how a mappings gets applied to a pattern of index names

PUT _template/import-export-logger-template
{
  "index_patterns": ["import-export-logger-*"],  <!-- This is how your index gets match to the mapping 
  "settings": {
    "number_of_shards": 1
  },
  "mappings": {
3

Hi @stephenb

Thank you for your early reply.
Creating template solution worked.

And the reason why I am using v6.5.4 is that my organization is using this version but maybe in the next quarter they will be upgrading to a newer version

Thank you once again

1 Like
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.