I have about 90 indexes 50gb each. I have created rollup job to process information of these indices.
The max page size for the rollup job is 10000 indexing is taking forever, any ideas on how i could ehance the performance of the job?
json: -
{ "config": { "id": "rollup-clipstat-video-company", "index_pattern": "clipstat-2.0-sessions-*", "rollup_index": "rollup-clipstat-video-company", "cron": "0 0 0 * * ?", "groups": { "date_histogram": { "fixed_interval": "30m", "field": "trueInitTime", "delay": "1d", "time_zone": "UTC" }, "terms": { "fields": [ "user", "media.id", "media.title.keyword", "lastState.st" ] } }, "metrics": [ { "field": "lastState.tp", "metrics": [ "sum" ] }, { "field": "watchPercent", "metrics": [ "sum", "value_count" ] } ], "timeout": "20s", "page_size": 10000 }, "status": { "job_state": "indexing", "current_position": { "lastState.st.terms": null, "media.id.terms": null, "media.title.keyword.terms": null, "trueInitTime.date_histogram": 1596331800000, "user.terms": "7785" }, "upgraded_doc_id": true }, "stats": { "pages_processed": 40, "documents_processed": 578183, "rollups_indexed": 400000, "trigger_count": 1, "index_time_in_ms": 76508, "index_total": 40, "index_failures": 0, "search_time_in_ms": 22300862, "search_total": 40, "search_failures": 0, "processing_time_in_ms": 7887, "processing_total": 40 } }