Hi, I've encountered a problem using kibana 5.2.2
I have a index and many fields in it, and I can find them in the discover page. But it is strange that some of them are "sometimes" out of the "_source" block and not searchable and I can't find them when I create a visualize. Here're 2 forms I've found in discover page:
"_index": "fw-au-ingest-service-log-2017.06",
"_type": "log",
"_id": "AVzO9EiAC7mbjsXAV3CA",
"_score": null,
"_source": {
"NewInsertedASUserQps": 71.8,
"L1MergerProcessTime": 85303.41514726507,
"ToL1MergerQueueTime": 97981.33995163241,
"ASWriteQps": 4184.7,
"KafkaReadPayloadSize": 112.78925986113163,
"KafkaReadPayloadThroughput": 458082.3,
"type": "log",
"L2MergerProcessTime": 74669.47422680413,
"path": "/export/audience_ingest_service/workdir_2/audience_monitor_20170622.LOG",
"ASWriteSize": 1708.9726384209143,
"ToL2ParserQueueTime": 94.98969072164948,
"host": "auingest02.fwmrm.net",
"LABEL": "2",
"ASReadLatency": 7477.176254619144,
"ASWriteLatency": 6169.744598986713,
"KafkaReadPayloadQps": 4061.4,
"ASReadThroughput": 7018301.1,
"L1ParserProcessTime": 713.0639661426844,
"offset": 10058623,
"input_type": "log",
"ASWriteConflictSize": 50132,
"BusyL1MergerThreadNum": 21,
"RecordIngestLifetime": 190056.67592010446,
"CategoryString": "StaticMonitor",
"fields": {
"tag": "audience_ingest_service"
},
"ASWriteConflictThroughput": 5013.2,
"ASWriteFailCount": 1420,
"ToL1ParserQueueTime": 5602.192212817412,
"KafkaReadSize": 181.17508740828285,
"ASReadQps": 4194.7,
"@version": "1",
"KafkaReadQps": 4061.4,
"KafkaReadThroughput": 735824.5,
"BusyL1ParserThreadNum": 5,
"timestamp": "2017/06/22 08:38:50",
"ASWriteLatency_Max": 223428,
"RecordLifetime_Max": 381,
"ASWriteThroughput": 7151537.8,
"ASWriteConflictQps": 0.1,
"RecordLifetime_Min": 0,
"@timestamp": "2017-06-22T08:38:53.183Z",
"ASReadSize": 1673.1354089684603,
"ASReadLatency_Max": 333924,
"RecordFlyTime": 296.6113556125163,
"RecordLifetime": 296.82959326788216,
"L2ParserProcessTime": 14585.350515463917,
"ToL2MergerQueueTime": 444.1443298969072
},
"fields": {
"@timestamp": [
1498120733183
],
"timestamp": [
1498120730000
]
},
"sort": [
1498120733183
]
}
or
"_index": "fw-au-ingest-service-log-2017.06",
"_type": "log",
"_id": "AVzO9C28C7mbjsXAV1WG",
"_score": null,
"_source": {
"NewInsertedASUserQps": 65.3,
"L1MergerProcessTime": 3189.897819063005,
"ToL1MergerQueueTime": 506.83509188206784,
"ASWriteQps": 4001.9,
"KafkaReadPayloadSize": 112.34488590468497,
"ASWriteFailCount": 364,
"KafkaReadPayloadThroughput": 445065.5,
"ToL1ParserQueueTime": 112.45315024232633,
"KafkaReadSize": 181.11619042810986,
"type": "log",
"L2MergerProcessTime": 1382.9807692307693,
"path": "/export/audience_ingest_service/workdir_1/audience_monitor_20170622.LOG",
"ASWriteSize": 1617.957270296609,
"ASReadQps": 4036.3,
"ToL2ParserQueueTime": 12.865384615384615,
"@version": "1",
"host": "auingest03.fwmrm.net",
"KafkaReadQps": 3961.6,
"KafkaReadThroughput": 717509.9,
"LABEL": "1",
"ASReadLatency": 566.4623293610484,
"ASWriteLatency": 685.3377645618332,
"KafkaReadPayloadQps": 3961.6,
"timestamp": "2017/06/22 08:38:42",
"ASWriteLatency_Max": 19908,
"ASReadThroughput": 6432002.8,
"L1ParserProcessTime": 52.298944870759286,
"RecordLifetime_Max": 1,
"offset": 10002772,
"input_type": "log",
"ASWriteThroughput": 6474903.2,
"RecordLifetime_Min": 0,
"@timestamp": "2017-06-22T08:38:45.282Z",
"ASReadSize": 1593.5393305750315,
"BusyL1MergerThreadNum": 6,
"ASReadLatency_Max": 58476,
"RecordIngestLifetime": 3891.6131361066236,
"RecordFlyTime": 0.31896203554119545,
"RecordLifetime": 0.3245405896607431,
"CategoryString": "StaticMonitor",
"L2ParserProcessTime": 302.5,
"ToL2MergerQueueTime": 10.807692307692308,
"fields": {
"tag": "audience_ingest_service"
}
},
"fields": {
"timestamp": [
1498120722000
],
"@timestamp": [
1498120725282
]
},
"sort": [
1498120725282
]
}
I'm very confused with why some of the fields are sometimes out of the "_source" bracket which is not searchable when I create a visualize. Here're the index pattern and you can see that they're not searchable:
can you help me? What can I do to make these fields searchable when I create a visualize?