I'm running a 7 node cluster on 6.6.0 basic license and am trying to create a rollup job. The rollup job that I create seems to only create as many documents as the page_size variable. So if I have my page_size set to 1100, it will trigger the first time and roll into 1100 documents and then all triggers that follow don't appear to do anything. I checked all 7 nodes' elastic logs and see nothing that looks like an error. Am I doing something wrong? This is my rollup job's json:
{
"config": {
"id": "MetricBeatRollup",
"index_pattern": "metricbeat-*",
"rollup_index": "rolled-metricbeat",
"cron": "0 5 0 * * ?",
"groups": {
"date_histogram": {
"interval": "60m",
"field": "@timestamp",
"delay": "1d",
"time_zone": "UTC"
},
"terms": {
"fields": [
"fields.env.keyword",
"fields.app_server",
"fields.type.keyword",
"fields.group",
"fields.machine_use",
"event.dataset.keyword",
"host.name.keyword",
"windows.perfmon.instance_name.keyword",
"system.diskio.name",
"metricset.name.keyword",
"measurement.keyword",
"perfmon_name"
]
}
},
"metrics": [
{
"field": "system.network.in.bytes",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.network.out.bytes",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.process.summary.running",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.memory.used.pct",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.cpu.total.norm.pct",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "value",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.diskio.read.bytes",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.diskio.write.bytes",
"metrics": [
"avg",
"value_count"
]
}
],
"timeout": "20s",
"page_size": 1000
},
"status": {
"job_state": "started",
"current_position": {
"@timestamp.date_histogram": 1551150000000,
"event.dataset.keyword.terms": null,
"fields.app_server.terms": "ptc_app_server",
"fields.env.keyword.terms": null,
"fields.group.terms": null,
"fields.machine_use.terms": null,
"fields.type.keyword.terms": null,
"host.name.keyword.terms": null,
"measurement.keyword.terms": null,
"metricset.name.keyword.terms": null,
"perfmon_name.terms": null,
"system.diskio.name.terms": null,
"windows.perfmon.instance_name.keyword.terms": null
},
"upgraded_doc_id": true
},
"stats": {
"pages_processed": 1,
"documents_processed": 3374279,
"rollups_indexed": 1000,
"trigger_count": 6,
"index_time_in_ms": 2544,
"index_total": 1,
"index_failures": 0,
"search_time_in_ms": 166668,
"search_total": 7,
"search_failures": 6
}
}