Rollup Job rolls only to Page_Size

I'm running a 7 node cluster on 6.6.0 basic license and am trying to create a rollup job. The rollup job that I create seems to only create as many documents as the page_size variable. So if I have my page_size set to 1100, it will trigger the first time and roll into 1100 documents and then all triggers that follow don't appear to do anything. I checked all 7 nodes' elastic logs and see nothing that looks like an error. Am I doing something wrong? This is my rollup job's json:

{
"config": {
"id": "MetricBeatRollup",
"index_pattern": "metricbeat-*",
"rollup_index": "rolled-metricbeat",
"cron": "0 5 0 * * ?",
"groups": {
"date_histogram": {
"interval": "60m",
"field": "@timestamp",
"delay": "1d",
"time_zone": "UTC"
},
"terms": {
"fields": [
"fields.env.keyword",
"fields.app_server",
"fields.type.keyword",
"fields.group",
"fields.machine_use",
"event.dataset.keyword",
"host.name.keyword",
"windows.perfmon.instance_name.keyword",
"system.diskio.name",
"metricset.name.keyword",
"measurement.keyword",
"perfmon_name"
]
}
},
"metrics": [
{
"field": "system.network.in.bytes",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.network.out.bytes",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.process.summary.running",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.memory.used.pct",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.cpu.total.norm.pct",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "value",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.diskio.read.bytes",
"metrics": [
"avg",
"value_count"
]
},
{
"field": "system.diskio.write.bytes",
"metrics": [
"avg",
"value_count"
]
}
],
"timeout": "20s",
"page_size": 1000
},
"status": {
"job_state": "started",
"current_position": {
"@timestamp.date_histogram": 1551150000000,
"event.dataset.keyword.terms": null,
"fields.app_server.terms": "ptc_app_server",
"fields.env.keyword.terms": null,
"fields.group.terms": null,
"fields.machine_use.terms": null,
"fields.type.keyword.terms": null,
"host.name.keyword.terms": null,
"measurement.keyword.terms": null,
"metricset.name.keyword.terms": null,
"perfmon_name.terms": null,
"system.diskio.name.terms": null,
"windows.perfmon.instance_name.keyword.terms": null
},
"upgraded_doc_id": true
},
"stats": {
"pages_processed": 1,
"documents_processed": 3374279,
"rollups_indexed": 1000,
"trigger_count": 6,
"index_time_in_ms": 2544,
"index_total": 1,
"index_failures": 0,
"search_time_in_ms": 166668,
"search_total": 7,
"search_failures": 6
}
}

I've also had this cluster stable for months and since trying to get the rollup job working I've been having nodes crash, so suspecting it is still trying to run at each trigger.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.