Hi,
I am new to ELK. I have filebeat 7.8.1, Logstash 7.8.0 (in docker), Elasticsearch 7.8.0.
I have log like this in my .log file:
{"@timestamp":"2021-02-03T09:43:44.439+07:00","@version":"1","message":"channelActive TCPServer.tcpgw-server1.server-json","logger_name":"id.co.hanoman.tcpgate.ChannelHandler","thread_name":"nioEventLoopGroup-7-2","level":"INFO","level_value":20000,"HOSTNAME":"tcpgw-server1","logType":"channelActive","appId":"TCPServer.tcpgw-server1.server-json","connId":"51306|172.16.110.49|8002","data":{"CorrelationID":"00001f420000c86a118981ec04ac106e3100000000000000","AppID":"TCPServer.tcpgw-server1.server-json","IP":"172.16.110.49"}}
but, when it reached to elasticsearch and I open in Kibana dashboard, it becomes like this:
//
{
"_index": "bjb-qc-tcp-000004",
"_type": "_doc",
"_id": "6YnGZXcBGXTf36qvTa3W",
"_version": 1,
"_score": null,
"_source": {
"HOSTNAME": "tcpgw-server1",
"@version": "1",
"thread_name": "nioEventLoopGroup-7-2",
"input": {
"type": "log"
},
"host": {
"name": "bjbmwjavaappqc"
},
"level_value": 20000,
"tags": [
"_jsonparsefailure",
"beats_input_codec_json_applied"
],
"appId": "TCPServer.tcpgw-server1.server-json",
"@timestamp": "2021-02-03T02:43:44.439Z",
"level": "INFO",
"connId": "51306|172.16.110.49|8002",
"data": {
"AppID": "TCPServer.tcpgw-server1.server-json",
"IP": "172.16.110.49",
"CorrelationID": "00001f420000c86a118981ec04ac106e3100000000000000"
},
"ecs": {
"version": "1.5.0"
},
"agent": {
"type": "filebeat",
"hostname": "bjbmwjavaappqc",
"name": "bjbmwjavaappqc",
"id": "2ef4b0fd-24df-4dc0-b5eb-0711f0c5c9e3",
"version": "7.8.1",
"ephemeral_id": "0c205b58-7767-45a9-b335-8daa009121f8"
},
"logger_name": "id.co.hanoman.tcpgate.ChannelHandler",
"logType": "channelActive",
"msg": "channelActive TCPServer.tcpgw-server1.server-json",
"log": {
"file": {
"path": "/opt/hanoman/qc-java/log/tcpgw-server1.log"
},
"offset": 10611
}
},
"fields": {
"response.expiresAt": [],
"path.ITA_I_I.$.updatedAt": [],
"data.JMSTimestamp": [],
"dtree.Message/RC.0000.$.createdAt": [],
"paths.Codex.ISO8583.Message/0.0800.$.updatedAt": [],
"paths.$.ITA_E_I.$.createdAt": [],
"paths.$.AB_E.$.createdAt": [],
"dtree.$.ITA_E_E.$.createdAt": [],
"response.createdAt": [],
"paths.$.updatedAt": [],
"path.ITA_I_E.$.updatedAt": [],
"response.lastUpdate": [],
"response.value.ERROR-MAP.gss-sdb.10.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.Message/RC.0099.$.createdAt": [],
"paths.$.ITA_E_E.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.Message/RC.0099.$.createdAt": [],
"response.value.ERROR-MAP.gss-clps.KSM0125.createdAt": [],
"dtree.$.GSS_PADDING_DEE.$.createdAt": [],
"paths.Message/RC.0099.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.Message/RC.0000.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.Message/RC.0099.$.updatedAt": [],
"data.JMSExpiration": [],
"paths.Message/RC.0000.$.createdAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.Message/RC.0000.$.updatedAt": [],
"dtree.$.ITA_E_I.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300090.Message/18.6010.$.createdAt": [],
"data.Logger.Time": [],
"dtree.$.AAH_I.$.createdAt": [],
"path.createdAt": [],
"sub.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0800.$.updatedAt": [],
"data.JMS.Expiration": [],
"response.value.ERROR-MAP.gss-clps.KSM5417.updatedAt": [],
"path.MAH_E.$.updatedAt": [],
"response.value.ERROR-MAP.gss-clps.KSM0149.updatedAt": [],
"response.value.ERROR-MAP.gss-sdb.20.createdAt": [],
"path.AB_O.$.createdAt": [],
"dtree.Message/RC.0099.$.createdAt": [],
"dtree.$.TO_GSS.$.createdAt": [],
"path.MAH_I.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.Message/RC.0000.$.createdAt": [],
"response.value.ERROR-MAP.gss-clps.KSM0140.createdAt": [],
"paths.$.AB_O.$.createdAt": [],
"dtree.$.updatedAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.$.updatedAt": [],
"data.ACCESS.Token.time": [],
"paths.$.TO_GSS.$.createdAt": [],
"response.value.ERROR-MAP.gss-clps.KSM2010.updatedAt": [],
"data.ACCESS.apiSecret.createdAt": [],
"paths.$.AB_I.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.$.updatedAt": [],
"path.ITA_E_I.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.$.updatedAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.$.updatedAt": [],
"path.ITA_E_E.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.Message/RC.0099.$.updatedAt": [],
"paths.$.ITA_I_I.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.$.createdAt": [],
"dtree.$.ITA_I_E.$.updatedAt": [],
"dtree.$.ITA_I_I.$.updatedAt": [],
"data.Instansi.validations.createdAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.300090.Message/18.6010.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.Message/RC.0099.$.createdAt": [],
"path.AAH_I.$.updatedAt": [],
"path.AB_I.$.createdAt": [],
"response.value.ERROR-MAP.gss-sdb.20.updatedAt": [],
"path.AAH_E.$.updatedAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0800.$.createdAt": [],
"dtree.$.MAH_E.$.createdAt": [],
"dtree.$.FROM_GSS.$.updatedAt": [],
"data.ACCESS.apiSecret.updatedAt": [],
"dtree.$.MAH_I.$.createdAt": [],
"path.AB_E.$.createdAt": [],
"paths.$.COPY_REQUEST_DATA.$.updatedAt": [],
"paths.Message/RC.0099.Message/RCMSG/0/MESSAGE_ID.KSM2010.RequestData/REK_SOURCE.O.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.Message/RC.0099.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300090.Message/18.6010.$.updatedAt": [],
"Time": [],
"dtree.$.AAH_E.$.createdAt": [],
"dtree.$.AB_O.$.updatedAt": [],
"paths.$.AB_O.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.$.updatedAt": [],
"path.updatedAt": [],
"response.updatedAt": [],
"dtree.$.COPY_REQUEST_DATA.$.createdAt": [],
"dtree.$.ITA_E_E.$.updatedAt": [],
"paths.$.ITA_I_E.$.updatedAt": [],
"dtree.$.AB_I.$.updatedAt": [],
"path.TO_GSS.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.Message/RC.0099.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.Message/RC.0099.Message/RCMSG/0/MESSAGE_ID.KSM2010.RequestData/REK_SOURCE.O.$.updatedAt": [],
"dtree.$.ITA_E_I.$.updatedAt": [],
"paths.Message/RC.0000.$.updatedAt": [],
"dtree.$.AB_E.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.Message/RC.0000.$.createdAt": [],
"paths.$.ITA_E_E.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.Message/RC.0099.$.updatedAt": [],
"response.value.ERROR-MAP.gss-clps.KSM0125.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.Message/RC.0099.$.updatedAt": [],
"paths.$.ITA_E_I.$.updatedAt": [],
"response.value.ERROR-MAP.gss-sdb.10.updatedAt": [],
"paths.$.createdAt": [],
"paths.$.AB_E.$.updatedAt": [],
"paths.$.AB_I.$.updatedAt": [],
"response.value.ERROR-MAP.gss-clps.KSM5417.createdAt": [],
"data.Time": [],
"response.value.ERROR-MAP.gss-clps.KSM0149.createdAt": [],
"paths.Codex.ISO8583.Message/0.0800.$.createdAt": [],
"dtree.Message/RC.0000.$.updatedAt": [],
"paths.$.TO_GSS.$.updatedAt": [],
"response.value.ERROR-MAP.gss-clps.KSM0140.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.$.updatedAt": [],
"response.value.ERROR-MAP.gss-clps.*.updatedAt": [],
"dtree.$.MAH_I.$.updatedAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.$.createdAt": [],
"dtree.$.TO_GSS.$.updatedAt": [],
"paths.$.MAH_I.$.createdAt": [],
"dtree.$.MAH_E.$.updatedAt": [],
"dtree.$.createdAt": [],
"paths.$.AAH_E.$.createdAt": [],
"dtree.$.GSS_PADDING_DEE.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.Message/RC.0000.$.updatedAt": [],
"paths.$.AAH_I.$.createdAt": [],
"dtree.$.AAH_E.$.updatedAt": [],
"paths.$.MAH_E.$.createdAt": [],
"path.AB_O.$.updatedAt": [],
"path.ITA_I_E.$.createdAt": [],
"data.JMS.Timestamp": [],
"response.value.ERROR-MAP.gss-clps.KSM1547.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.$.updatedAt": [],
"path.ITA_I_I.$.createdAt": [],
"path.AB_E.$.updatedAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.$.updatedAt": [],
"response.value.ERROR-MAP.gss-sdb.*.updatedAt": [],
"@timestamp": [
"2021-02-03T02:43:44.439Z"
],
"path.AB_I.$.updatedAt": [],
"data.ACCESS.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.Message/RC.0000.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.Message/RC.0099.$.createdAt": [],
"sub.$.updatedAt": [],
"dtree.$.AAH_I.$.updatedAt": [],
"paths.Message/RC.0099.$.updatedAt": [],
"dtree.$.FROM_GSS.$.createdAt": [],
"dtree.Message/RC.0099.Message/RCMSG/0/MESSAGE_ID.KSM2010.RequestData/REK_SOURCE.O.$.updatedAt": [],
"path.COPY_REQUEST_DATA.$.createdAt": [],
"paths.$.COPY_REQUEST_DATA.$.createdAt": [],
"dtree.$.AB_E.$.createdAt": [],
"jms.Expiration": [],
"dtree.$.ITA_I_I.$.createdAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.010099.Message/18.6010.$.createdAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.$.createdAt": [],
"path.ITA_E_E.$.updatedAt": [],
"jms.Timestamp": [],
"dtree.$.AB_I.$.createdAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.300090.Message/18.6010.$.createdAt": [],
"dtree.Message/RC.0099.Message/RCMSG/0/MESSAGE_ID.KSM2010.RequestData/REK_SOURCE.O.$.createdAt": [],
"dtree.$.ITA_I_E.$.createdAt": [],
"paths.$.AAH_I.$.updatedAt": [],
"data.ACCESS.updatedAt": [],
"response.value.ERROR-MAP.gss-clps.KSM2010.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.Message/RC.0000.$.updatedAt": [],
"dtree.Message/RC.0099.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.310099.Message/18.6010.Message/RC.0000.$.createdAt": [],
"path.TO_GSS.$.createdAt": [],
"paths.$.ITA_I_E.$.createdAt": [],
"paths.$.AAH_E.$.updatedAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.020099.Message/18.6010.$.createdAt": [],
"path.COPY_REQUEST_DATA.$.updatedAt": [],
"path.ITA_E_I.$.updatedAt": [],
"dtree.$.COPY_REQUEST_DATA.$.updatedAt": [],
"response.value.ERROR-MAP.gss-sdb.*.createdAt": [],
"path.AAH_E.$.createdAt": [],
"paths.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.$.createdAt": [],
"paths.Message/RC.0099.Message/RCMSG/0/MESSAGE_ID.KSM2010.RequestData/REK_SOURCE.O.$.createdAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.400099.Message/18.6010.Message/RC.0000.$.updatedAt": [],
"dtree.Codex.ISO8583.Message/0.0200.Message/3.300099.Message/18.6010.Message/RC.0000.$.updatedAt": []
},
"highlight": {
"data.CorrelationID": [
"@kibana-highlighted-field@00001f420000c86a118981ec04ac106e3100000000000000@/kibana-highlighted-field@"
]
},
"sort": [
1612320224439
]
}
Why Why I got so many entries *.createdAt and *.updatedAt inside "fields"?
how can I remove them?
Thank you.