Fields found in discover not found when creating a visualize

Elastic Stack Kibana
1

Hi, I've encountered a problem using kibana 5.2.2
I have a index and many fields in it, and I can find them in the discover page. But it is strange that some of them are "sometimes" out of the "_source" block and not searchable and I can't find them when I create a visualize. Here're 2 forms I've found in discover page:
"_index": "fw-au-ingest-service-log-2017.06",
"_type": "log",
"_id": "AVzO9EiAC7mbjsXAV3CA",
"_score": null,
"_source": {
"NewInsertedASUserQps": 71.8,
"L1MergerProcessTime": 85303.41514726507,
"ToL1MergerQueueTime": 97981.33995163241,
"ASWriteQps": 4184.7,
"KafkaReadPayloadSize": 112.78925986113163,
"KafkaReadPayloadThroughput": 458082.3,
"type": "log",
"L2MergerProcessTime": 74669.47422680413,
"path": "/export/audience_ingest_service/workdir_2/audience_monitor_20170622.LOG",
"ASWriteSize": 1708.9726384209143,
"ToL2ParserQueueTime": 94.98969072164948,
"host": "auingest02.fwmrm.net",
"LABEL": "2",
"ASReadLatency": 7477.176254619144,
"ASWriteLatency": 6169.744598986713,
"KafkaReadPayloadQps": 4061.4,
"ASReadThroughput": 7018301.1,
"L1ParserProcessTime": 713.0639661426844,
"offset": 10058623,
"input_type": "log",
"ASWriteConflictSize": 50132,
"BusyL1MergerThreadNum": 21,
"RecordIngestLifetime": 190056.67592010446,
"CategoryString": "StaticMonitor",
"fields": {
"tag": "audience_ingest_service"
},
"ASWriteConflictThroughput": 5013.2,
"ASWriteFailCount": 1420,
"ToL1ParserQueueTime": 5602.192212817412,
"KafkaReadSize": 181.17508740828285,
"ASReadQps": 4194.7,
"@version": "1",
"KafkaReadQps": 4061.4,
"KafkaReadThroughput": 735824.5,
"BusyL1ParserThreadNum": 5,
"timestamp": "2017/06/22 08:38:50",
"ASWriteLatency_Max": 223428,
"RecordLifetime_Max": 381,
"ASWriteThroughput": 7151537.8,
"ASWriteConflictQps": 0.1,
"RecordLifetime_Min": 0,
"@timestamp": "2017-06-22T08:38:53.183Z",
"ASReadSize": 1673.1354089684603,
"ASReadLatency_Max": 333924,
"RecordFlyTime": 296.6113556125163,
"RecordLifetime": 296.82959326788216,
"L2ParserProcessTime": 14585.350515463917,
"ToL2MergerQueueTime": 444.1443298969072
},
"fields": {
"@timestamp": [
1498120733183
],
"timestamp": [
1498120730000
]
},
"sort": [
1498120733183
]
}

or

"_index": "fw-au-ingest-service-log-2017.06",
  "_type": "log",
  "_id": "AVzO9C28C7mbjsXAV1WG",
  "_score": null,
  "_source": {
    "NewInsertedASUserQps": 65.3,
    "L1MergerProcessTime": 3189.897819063005,
    "ToL1MergerQueueTime": 506.83509188206784,
    "ASWriteQps": 4001.9,
    "KafkaReadPayloadSize": 112.34488590468497,
    "ASWriteFailCount": 364,
    "KafkaReadPayloadThroughput": 445065.5,
    "ToL1ParserQueueTime": 112.45315024232633,
    "KafkaReadSize": 181.11619042810986,
    "type": "log",
    "L2MergerProcessTime": 1382.9807692307693,
    "path": "/export/audience_ingest_service/workdir_1/audience_monitor_20170622.LOG",
    "ASWriteSize": 1617.957270296609,
    "ASReadQps": 4036.3,
    "ToL2ParserQueueTime": 12.865384615384615,
    "@version": "1",
    "host": "auingest03.fwmrm.net",
    "KafkaReadQps": 3961.6,
    "KafkaReadThroughput": 717509.9,
    "LABEL": "1",
    "ASReadLatency": 566.4623293610484,
    "ASWriteLatency": 685.3377645618332,
    "KafkaReadPayloadQps": 3961.6,
    "timestamp": "2017/06/22 08:38:42",
    "ASWriteLatency_Max": 19908,
    "ASReadThroughput": 6432002.8,
    "L1ParserProcessTime": 52.298944870759286,
    "RecordLifetime_Max": 1,
    "offset": 10002772,
    "input_type": "log",
    "ASWriteThroughput": 6474903.2,
    "RecordLifetime_Min": 0,
    "@timestamp": "2017-06-22T08:38:45.282Z",
    "ASReadSize": 1593.5393305750315,
    "BusyL1MergerThreadNum": 6,
    "ASReadLatency_Max": 58476,
    "RecordIngestLifetime": 3891.6131361066236,
    "RecordFlyTime": 0.31896203554119545,
    "RecordLifetime": 0.3245405896607431,
    "CategoryString": "StaticMonitor",
    "L2ParserProcessTime": 302.5,
    "ToL2MergerQueueTime": 10.807692307692308,
    "fields": {
      "tag": "audience_ingest_service"
    }
  },
  "fields": {
    "timestamp": [
      1498120722000
    ],
    "@timestamp": [
      1498120725282
    ]
  },
  "sort": [
    1498120725282
  ]
}

I'm very confused with why some of the fields are sometimes out of the "_source" bracket which is not searchable when I create a visualize. Here're the index pattern and you can see that they're not searchable:

image.png1898×1135 152 KB

can you help me? What can I do to make these fields searchable when I create a visualize?

2

Searchable fields based on the index mapping.

Can you provide the mapping for that index? Example: http://localhost:9200/fw-au-ingest-service-log-*/_mapping.

3

Thank you @tsmalley I've cleared the history data and it worked

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.