Moving average on derivative

Elastic Stack Elasticsearch
1

Elasticsearch version: 6.2

I'm trying to get the moving average of a derivative of the remaining bytes. However, I want to exclude positive results from the derivative. I have tried to add a bucket_selector, but it gives me a null_pointer_exception on "params.change". This is the query that includes positive derivatives:

POST user_profile/_search
{
  "size": 0,
  "query": {
    "match": {
      "user_key": "this_key"
    }
  },
  "aggs": {
    "history": {
      "nested": {
        "path": "history"
      },
      "aggs": {
        "last_week": {
          "filter": {
            "range": {
              "history.request_time": {
                "gte": "2018-04-13T06:30:39",
                "lte": "2018-04-20T06:30:39"
              }
            }
          },
          "aggs": {
            "bucket_by_day": {
              "date_histogram": {
                "field": "history.request_time",
                "interval": "day"
              },
              "aggs": {
                "max_remaining_bytes_of_day": {
                  "max": {
                    "field": "history.value.remainingBytes"
                  }
                },
                "balance_change": {
                  "derivative": {
                    "buckets_path": "max_remaining_bytes_of_day",
                    "gap_policy": "insert_zeros",
                    "unit": "hour"
                  }
                },
                "burndown_estimation": {
                  "moving_avg": {
                    "buckets_path": "balance_change.normalized_value"
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

This is what I tried to do:

POST user_profile/_search
{
  "size": 0,
  "query": {
    "match": {
      "user_key": "this_key"
    }
  },
  "aggs": {
    "history": {
      "nested": {
        "path": "history"
      },
      "aggs": {
        "last_week": {
          "filter": {
            "range": {
              "history.request_time": {
                "gte": "2018-04-13T06:30:39",
                "lte": "2018-04-20T06:30:39"
              }
            }
          },
          "aggs": {
            "bucket_by_day": {
              "date_histogram": {
                "field": "history.request_time",
                "interval": "day"
              },
              "aggs": {
                "max_remaining_bytes_of_day": {
                  "max": {
                    "field": "history.value.remainingBytes"
                  }
                },
                "balance_change": {
                  "derivative": {
                    "buckets_path": "max_remaining_bytes_of_day",
                    "gap_policy": "insert_zeros",
                    "unit": "hour"
                  }
                },
                "burndown_estimation": {
                  "moving_avg": {
                    "buckets_path": "balance_change.normalized_value"
                  }
                },
                "balance_change_filter": {
                  "bucket_selector": {
                    "buckets_path": {
                      "change": "balance_change"
                    },
                    "script": "params.change < 0"
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}
2

Solved by using bucket_script to remove positive entries and getting the moving average on the bucket_script aggregation

"max_data_of_bucket": {
  "max": {
    "field": "history.remainingBytes"
  }
},
"derivative_of_balance": {
  "derivative": {
    "buckets_path": "max_data_of_bucket",
    "unit": "hour"
  }
},
"burndown": {
  "bucket_script": {
    "buckets_path": {
      "balance_change": "derivative_of_balance.normalized_value"
    },
    "script": """
      if (params.balance_change > 0) {return null;}
      else {return params.balance_change;}
    """
  }
},
"moving_average_of_burndown": {
  "moving_avg": {
    "buckets_path": "burndown",
    "gap_policy": "skip",
    "window": 7,
    "model": "ewma",
    "settings": {
      "alpha": 0.4
    }
  }
}
3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.