Alert if any value in a specific field stops appearing

In my index, if any value has appeared in a specific field every day in the last 3 days, I expect to see this value at least once every day in the future. I want to receive an alert if any such value suddenly stops showing up (i.e., such a value does not appear in a full day 00:00am to 11:59pm). How do I approach this problem without having to manually enumerate all possible values for the field?

I'm not familiar with alarting but there could be elastic aggregation query to listup such terms.

Is this help you? Take care about timezone.

You might need Advanced Watcher to use this query to alert.

GET /kibana_sample_data_logs/_search
{
  "size":0,
  "runtime_mappings": {
    "days_elapsed": {
      "type": "long",
      "script": {
        "source": "Instant instant = Instant.ofEpochMilli(System.currentTimeMillis()); ZonedDateTime today = ZonedDateTime.ofInstant(instant, ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS); emit(ChronoUnit.DAYS.between(doc['timestamp'].value.truncatedTo(ChronoUnit.DAYS), today))"
      }
    }
  },
  "fields": [
    "days_elapsed", "now"
  ], 
  "query":{
    "range":{
      "timestamp": {
        "gte": "now-3d/d",
        "lte": "now/d",
        "time_zone": "+00:00"
      }
    }
  },
  "aggs": {
    "request":{
      "terms":{
        "field": "request.keyword"
      },
      "aggs": {
        "days_elapsed": {
          "terms": {
            "field": "days_elapsed",
            "include":["0","1","2","3"],
            "min_doc_count": 0,
            "order": {"_key":"asc"}
          }
        },
        "days_elapsed_selector":{
          "bucket_selector": {
            "buckets_path": {
              "3":"days_elapsed['3']>_count",
              "2":"days_elapsed['2']>_count",
              "1":"days_elapsed['1']>_count",
              "0":"days_elapsed['0']>_count"
            },
            "script": "params.3>0 && params.2>0 && params.1>0 && params.0==0"
          }
        }
      }
    }
  }
}
1 Like

@Tomo_M Your suggestion did help. I ended up with a solution like below. Leveraged quite a bit of painless scripting. For some reason, I wasn't able to take advantage of standard Java library features such as List.containsAll() method, so I ended up coding a few things from first principles.

{
  "trigger": {
    "schedule": {
      "interval": "60m"
    }
  },
  "input": {
    "chain": {
      "inputs": [
        {
          "rules1w": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "elastalert_status*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 0,
                  "query": {
                    "bool": {
                      "filter": [
                        {
                          "exists": {
                            "field": "rule_name"
                          }
                        },
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-7d/m",
                              "lt": "now/m"
                            }
                          }
                        }
                      ],
                      "must_not": {
                        "term": {
                          "rule_name": "flatline*"
                        }
                      }
                    }
                  },
                  "aggs": {
                    "RuleNames": {
                      "terms": {
                        "field": "rule_name",
                        "size": 10000
                      }
                    }
                  }
                }
              }
            }
          }
        },
        {
          "rules1d": {
            "search": {
              "request": {
                "search_type": "query_then_fetch",
                "indices": [
                  "elastalert_status*"
                ],
                "rest_total_hits_as_int": true,
                "body": {
                  "size": 0,
                  "query": {
                    "bool": {
                      "filter": [
                        {
                          "exists": {
                            "field": "rule_name"
                          }
                        },
                        {
                          "range": {
                            "@timestamp": {
                              "gte": "now-70m/m",
                              "lt": "now/m"
                            }
                          }
                        }
                      ],
                      "must_not": {
                        "term": {
                          "rule_name": "flatline*"
                        }
                      }
                    }
                  },
                  "aggs": {
                    "RuleNames": {
                      "terms": {
                        "field": "rule_name",
                        "size": 10000
                      }
                    }
                  }
                }
              }
            }
          }
        }
      ]
    }
  },
  "condition": {
    "always": {}
  },
  "actions": {
    "send_email": {
      "condition": {
        "script": {
          "source": "return ctx.payload.dropped.size() > 0",
          "lang": "painless"
        }
      },
      "transform": {
        "script": {
          "source": """
                int size = ctx.payload.rules1d.aggregations.RuleNames.buckets.size();
                String[] rule_names_1d = new String[size];
                for( int i = 0; i < size; i++ ) {
                  rule_names_1d[i] = ctx.payload.rules1d.aggregations.RuleNames.buckets[i].key;
                }
                
                size = ctx.payload.rules1w.aggregations.RuleNames.buckets.size();
                String[] rule_names_1w = new String[size];
                for( int i = 0; i < size; i++ ) {
                  rule_names_1w[i] = ctx.payload.rules1w.aggregations.RuleNames.buckets[i].key;
                }
                
                List dropped_rules = new ArrayList();
                for( int i = 0; i < rule_names_1w.length; i++ ){
                    String rule_i = rule_names_1w[i];
                    for( int j = 0; j < rule_names_1d.length; j++ ){
                        if( rule_i == rule_names_1d[j] ){
                            break;
                        } else if( j == rule_names_1d.length ){
                            dropped_rules.add( rule_i );
                        }
                    }
                }
                
                return[
                    'dropped': dropped_rules.stream().collect(Collectors.joining(", "))
                ]
                
                """,
          "lang": "painless"
        }
      },
      "email": {
        "profile": "standard",
        "to": [
          "emailid@domain.com"          
        ],
        "subject": "Elastalert Rule Stopped Running (Test)",
        "body": {
          "html": "Dropped Rules: {{ctx.payload.dropped}}"
        }
      }
    }
  }
}
1 Like

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.