Trying to understand the logic behind the ES index mapping using heka - in existing project. I have following template to create index in ES.
# cat es-index-template.json
{
"template": "logs-*",
"mappings": {
"_default_": {
"properties": {
"timestamp": {
"type": "date",
"format":"dateOptionalTime"
},
"host": {"type": "string", "index": "not_analyzed"},
"app": {"type": "string", "index": "not_analyzed"},
"app_instance": {"type": "integer"},
"msg": {"type": "string"},
"msg_id": {"type": "string", "index": "not_analyzed"},
"priority": {"type": "integer"},
"trace_id": {"type": "string", "index": "not_analyzed"},
"upstream_id": {"type": "string", "index": "not_analyzed"},
"hop_id": {"type": "string", "index": "not_analyzed"},
"pid": {"type": "integer"},
"tid": {"type": "long"},
"user_id": {"type": "string", "index": "not_analyzed"},
"http_status": {"type": "string", "index": "not_analyzed"},
"http_path": {"type": "string"},
"http_method": {"type": "string", "index": "not_analyzed"},
"facility": {"type": "string", "index": "not_analyzed"}
}
}
}
}
When I checked the mapping of ES and found that log-* index is having only one document type "log_message" in it. I checked the _mapping of document and index both are giving me the same results only difference is that when I am issuing the following command without document type it's showing me the mapping of _all also for index. I read about _search and understand that it's the json input data before the analyzer parse it. How come this mapping is different than the one mentioned in heka file? Is checking the mapping of _search is equivalent to check the _mapping of indexed data.
root@ovthick_121:~# curl http://172.16.0.64:9200/logs-2017.11.28/_mapping/log_message?pretty
{
"logs-2017.11.28" : {
"mappings" : {
"log_message" : {
"properties" : {
"akkaSource" : {
"type" : "string"
},
"akkaTimestamp" : {
"type" : "string"
},
"app" : {
"type" : "string",
"index" : "not_analyzed"
},
"app_instance" : {
"type" : "integer"
},
"code_file" : {
"type" : "string"
},
"code_func" : {
"type" : "string"
},
"code_line" : {
"type" : "long"
},
"container" : {
"type" : "string"
},
"desired_orch_state" : {
"type" : "string"
},
"dst_connect_time" : {
"type" : "string"
},
"dst_ip" : {
"type" : "string"
},
"dst_port" : {
"type" : "string"
},
"dst_response_time" : {
"type" : "string"
},
"duration" : {
"type" : "double"
},
"facility" : {
"type" : "string",
"index" : "not_analyzed"
},
"haproxy_backend" : {
"type" : "string"
},
"haproxy_termination_state" : {
"type" : "string"
},
"hop_id" : {
"type" : "string",
"index" : "not_analyzed"
},
"host" : {
"type" : "string",
"index" : "not_analyzed"
},
"http_method" : {
"type" : "string",
"index" : "not_analyzed"
},
"http_path" : {
"type" : "string"
},
"http_query" : {
"type" : "string"
},
"http_status" : {
"type" : "string",
"index" : "not_analyzed"
},
"level" : {
"type" : "string"
},
"msg" : {
"type" : "string"
},
"msg_id" : {
"type" : "string",
"index" : "not_analyzed"
},
"namespace" : {
"type" : "string"
},
"oplog" : {
"type" : "boolean"
},
"orch_state" : {
"type" : "string"
},
"pid" : {
"type" : "integer"
},
"priority" : {
"type" : "integer"
},
"product_id" : {
"type" : "string"
},
"relationship_id" : {
"type" : "string"
},
"relationship_type" : {
"type" : "string"
},
"request_bytes" : {
"type" : "string"
},
"resource_id" : {
"type" : "string"
},
"resource_label" : {
"type" : "string"
},
"resource_tenant_id" : {
"type" : "string"
},
"resource_type" : {
"type" : "string"
},
"response_bytes" : {
"type" : "string"
},
"retries" : {
"type" : "string"
},
"session_duration" : {
"type" : "string"
},
"sourceActorSystem" : {
"type" : "string"
},
"sourceThread" : {
"type" : "string"
},
"source_id" : {
"type" : "string"
},
"src_ip" : {
"type" : "string"
},
"src_port" : {
"type" : "string"
},
"syslog" : {
"type" : "string"
},
"target_id" : {
"type" : "string"
},
"tenant_id" : {
"type" : "string"
},
"thread" : {
"type" : "string"
},
"tid" : {
"type" : "long"
},
"timestamp" : {
"type" : "date",
"format" : "dateOptionalTime"
},
"tls_ciphers" : {
"type" : "string"
},
"tls_version" : {
"type" : "string"
},
"trace_id" : {
"type" : "string",
"index" : "not_analyzed"
},
"upstream_id" : {
"type" : "string",
"index" : "not_analyzed"
},
"user_id" : {
"type" : "string",
"index" : "not_analyzed"
},
"user_name" : {
"type" : "string"
}
}
}
}
}
}