Select latest docs for each transactions and apply filters on selections with pagination capability

[pramodbhade]

I have a situation where I need to load 100 records per page with the latest values and be able to paginate as well.
The selected latest records also get filtered for a set of filter values.

I'm using aggregation in the following orders
Composite agg (size=100) >> term agg (Ordered, size=1) >> Filter agg >> Term agg (multiple siblings)

Now the problem is that I need 100 records at once, but with filters the actual result goes somewhat from very less to 0 records.
How do I ensure to always get 100 records in a single search without having to re-iterate the search?

Below is the query I'm using right now:

GET /index_name/_search
{
  "size": 0,
  "aggs": {
    "alerts": {
      "composite": {
        "sources": [
          {
            "am-jobs": {
              "terms": {
                "field": "job_id.keyword"
              }
            }
          },
          {
            "am-host": {
              "terms": {
                "field": "app_host.keyword"
              }
            }
          }
        ],
        "size": 100
      },
      "aggs": {
        "record": {
          "terms": {
            "field": "last_update_time",
            "size": 1,
            "order": [
              {
                "last_status_changed": "desc"
              },
              {
                "last_update_time": "desc"
              }
            ]
          },
          "aggs": {
            "last_update_time": {
              "max": {
                "field": "last_update_time"
              }
            },
            "last_status_changed": {
              "max": {
                "field": "last_status_changed"
              }
            },
            "children": {

              "filter": {
                "bool": {
                  "must": [
                    {
                      "term": {
                        "automata_status.keyword": {
                          "value": "Running"
                        }
                      }
                    }
                  ]
                }
              },
              "aggs": {
                "session_number": {
                  "terms": {
                    "missing": "NA",
                    "field": "session_number.keyword",
                    "min_doc_count": 1
                  }
                },
                "priority": {
                  "terms": {
                    "missing": "NA",
                    "field": "automata_priority.keyword",
                    "min_doc_count": 1
                  }
                },
                "name": {
                  "terms": {
                    "missing": "NA",
                    "field": "automata_name.keyword",
                    "min_doc_count": 1
                  }
                },
                "start_time": {
                  "terms": {
                    "field": "automata_start_time",
                    "min_doc_count": 1
                  }
                },
                "end_time": {
                  "terms": {
                    "field": "automata_end_time",
                    "min_doc_count": 1
                  }
                },
                "last_status_changed": {
                  "terms": {
                    "field": "last_status_changed",
                    "min_doc_count": 1
                  }
                }
              }
            },
            "selector": {
                "bucket_selector": {
                  "buckets_path": {
                    "size_path": "children>name._bucket_count"
                  },
                  "script": "params.size_path > 0"
                }
              }
          }
        },
        "selector": {
          "bucket_selector": {
            "buckets_path": {
              "size_path": "record._bucket_count"
            },
            "script": "params.size_path > 0"
          }
        }
      }
    }
  },
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "last_status_changed": {
              "gte": "now-1d",
              "lte": "now"
            }
          }
        }
      ],
      "filter": [],
      "should": [],
      "must_not": [
        {
          "term": {
            "exception_state.keyword": {
              "value": "Remediation"
            }
          }
        },
        {
          "term": {
            "doc_type.keyword": {
              "value": "master_data"
            }
          }
        }
      ]
    }
  }
}