Hi Team,
We are trying to do our Volume sizing exercise and found that a json doc of size 1KB is taking up about 17KB in the index. Our use case is metrics related and most of the fields do not need indexing for search. In our usecase, we will be getting a huge number of these docs on a daily basis. Please help us to reduce the size of each doc here.
ES Version : 7.4.1
our template :
{
“analytics.program_viewlog_v1” : {
“order” : 0,
“index_patterns” : [
“analytics.program.viewlog.v1-*”
],
“settings” : {
“index” : {
“number_of_shards” : “1",
“refresh_interval” : “5s”
}
},
“mappings” : {
“properties” : {
“finalReport” : {
“type” : “boolean”
},
“captionsEnabled” : {
“type” : “boolean”
},
“sessionEndTime” : {
“type” : “date”
},
“mayBeFinalReport” : {
“type” : “boolean”
},
“viewlogId” : {
“index” : false,
“type” : “keyword”
},
“loginSessionId” : {
“index” : false,
“type” : “keyword”
},
“bitrate” : {
“type” : “long”
},
“deliveryCorrelationId” : {
“index” : false,
“type” : “keyword”
},
“deliveryId” : {
“index” : false,
“type” : “keyword”
},
“playbackPosition” : {
“type” : “double”
},
“uniqueWatchTime” : {
“type” : “long”
},
“errorCount” : {
“type” : “short”
},
“totalBufferTime” : {
“type” : “long”
},
“deviceType” : {
“type” : “keyword”
},
“watchedSecondsArray” : {
“type” : “byte”
},
“audienceId” : {
“index” : false,
“type” : “keyword”
},
“totalPauseTime” : {
“type” : “long”
},
“domainId” : {
“index” : false,
“type” : “keyword”
},
“bufferTime” : {
“type” : “long”
},
“networkLocationId” : {
“index” : false,
“type” : “keyword”
},
“lastGenFinalPosition” : {
“type” : “double”
},
“playbackPercentage” : {
“type” : “double”
},
“playerState” : {
“index” : false,
“type” : “keyword”
},
“creationTime” : {
“type” : “date”
},
“totalErrorCount” : {
“type” : “short”
},
“contentId” : {
“index” : false,
“type” : “keyword”
},
“viewingSessionId” : {
“index” : false,
“type” : “keyword”
},
“browserType” : {
“type” : “keyword”
},
“gen” : {
“type” : “long”
},
“playbackRate” : {
“type” : “double”
},
“avgBitRate” : {
“type” : “long”
},
“osInfo” : {
“type” : “keyword”
},
“assetId” : {
“index” : false,
“type” : “keyword”
},
“playerType” : {
“index” : false,
“type” : “keyword”
},
“totalBufferCount” : {
“type” : “short”
},
“seq” : {
“type” : “long”
},
“live” : {
“type” : “boolean”
},
“channelId” : {
“index” : false,
“type” : “keyword”
},
“eventTimestamp” : {
“type” : “date”
},
“clientId” : {
“index” : false,
“type” : “keyword”
},
“sessionStartTime” : {
“type” : “date”
},
“userAgent” : {
“index” : false,
“type” : “keyword”
},
“userId” : {
“index” : false,
“type” : “keyword”
},
“embedLocation” : {
“index” : false,
“type” : “keyword”
},
“totalWatchTime” : {
“type” : “long”
},
“bufferCount” : {
“type” : “short”
},
“geoLocation” : {
“type” : “nested”
},
“clientIp” : {
“index” : false,
“type” : “keyword”
},
“bcastInstanceId” : {
“index” : false,
“type” : “keyword”
},
“programId” : {
“index” : false,
“type” : “keyword”
}
}
},
“aliases” : { }
}
}
sample json being indexed
POST analytics.program.viewlog.v1 - defaultdomain - 2017.01.02 / _doc{
“domainId”: “performancetest”,
“eventTimestamp”: 1546281030000,
“messageProducerId”: null,
“messageSequence”: null,
“viewlogId”: “144e1e1b - 25b6 - 487b - a568 - dbc4262dea62”,
“creationTime”: 1546281030000,
“programId”: “esc_program: 1 ",“bcastInstanceId”:“esc_program:1",
“userId”: “USERID_0 ",“viewingSessionId”:“188f380b_viewingSessionId_0_1",
“gen”: 1,
“seq”: 0,
“playerState”: “WAITING”,
“playbackPosition”: 0,
“bufferTime”: 12,
“bufferCount”: 1,
“loginSessionId”: “188f380b_loginSessionId_0_1”,
“clientIp”: null,
“bitrate”: 177,
“contentId”: “esc_content: 1”,
“assetId”: “esc_cnt_ast: 1”,
“playerType”: “html5”,
“deliveryId”: “dlvry_akamai: 201”,
“deliveryCorrelationId”: null,
“audienceId”: “esc_audience: 201 ",“clientId”:“esc_client:1",
“networkLocationId”: “esc_networklocation: 1 ",“userAgent”:“Mozilla/5.0 (iPad; CPU OS 12_1_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1",
“lastGenFinalPosition”: 0,
“embedLocation”: “VCC”,
“channelId”: “esc_prg_cntr_chn: 1”,
“deviceType”: null,
“browserType”: null,
“osInfo”: null,
“geoLocation”: null,
“playbackPercentage”: 0.0,
“playbackRate”: 1.0,
“totalWatchTime”: 0,
“uniqueWatchTime”: 0,
“totalPauseTime”: 0,
“totalBufferTime”: 491,
“totalBufferCount”: 0,
“totalErrorCount”: 0,
“avgBitRate”: 177,
“sessionStartTime”: 1546281000,
“sessionEndTime”: 1573642787855,
“mayBeFinalReport”: false,
“watchedSecondsArray”: ,
“contentDuration”: 1200000,
“live”: false,
“finalReport”: false,
“captionsEnabled”: false
}