Create an index with aggregations

[BrunoSdeMenezes]

Hi,
I need to create a new index from a previous index that exists but only with aggregations and some metrics. For example: I have a field called "status_contrato" which is a string, and need to bring the percentile of "status_contrato: APROVADO" and the percentile of "status_contrato: PENDENTE".

I know I can make this kind of metric on Kibana, but I need an index only with these parameters, without input all the data from the previous index.

My script until now:

input{
   elasticsearch{
    hosts => "localhost"
    index => "teste"

   }
}

filter {
    mutate{
      remove_field => ["@timestamp","@version","host","path", "column14", "column15", "column16","id_contrato","id_tarefa","message","status_tarefa","data_confirmacao","diretoria","nome_gestor","nome_parte_passiva","prazo_tarefa","tarefa_pendente","area_ou_user_resp"]

    }


    date{
      match => [ "data_registro", "dd/MM/yyyy" ]
      target => "data_registro"
      timezone => "America/Sao_Paulo"
    }

    if [status_contrato] != ""{
      //Make here the calculations
    }

    mutate{
      remove_field =>["status_contrato", "data_registro"]
    }

}

output{

  elasticsearch{
    hosts => "localhost:9200"
    index => "calculos"
    manage_template => false
  }
  stdout{}

}

I've tried using the aggregation filter, metric filter, ruby filter, but nothing seems to work.

Can someone help me out?

The mapping of the index I'm using to import is this one:

{
  "mapping": {
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "@version": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "area_ou_user_resp": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "column14": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "column15": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "column16": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "data_confirmacao": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "data_registro": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "diretoria": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "host": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "id_contrato": {
        "type": "long"
      },
      "id_tarefa": {
        "type": "long"
      },
      "message": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "nome_gestor": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "nome_parte_passiva": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "path": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "prazo_tarefa": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "status_contrato": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "tarefa_pendente": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      }
    }
  }
}

[Badger]

If you are fetching documents from elasticsearch then I would think you could have elasticsearch do the aggregation for you.

[BrunoSdeMenezes]

Yes,, I could do this... But these statics I wil be using on a Markdown to build a Ticker on a Dashboard.

So I need an automated solution to capture the docs from the previous index, make the calculations and build a new index doc to capture on Markdown.

Anyone have any other ideas how to do that?

[BrunoSdeMenezes]

Ok, I've got how to do it... Now I'm getting an index only with the metrics and calculations needed. This is the script:

input{
   elasticsearch{
    hosts => "localhost"
    index => "teste"
   }
}

filter {

      metrics{
        meter => [ "%{status_contrato}", "events"]
        add_tag => "metric"
        flush_interval => 25
      }

      mutate{
        remove_field =>["message","status_contrato", "[PENDENTE][rate_5m]", "[PENDENTE][rate_1m]", "[PENDENTE][rate_15m]", "[APROVADO][rate_1m]", "[APROVADO][rate_5m]", "[APROVADO][rate_15m]", "[events][rate_15m]", "[events][rate_1m]", "[events][rate_5m]"]
      }

      mutate{
        convert =>{"[APROVADO][count]" => "float"}
        convert =>{"[events][count]" => "float"}
      }

      if "metric" in [tags]{
        ruby{
          code => " aprovados = event.get(%{[APROVADO][count]})/event.get(%{[events][count]})*100;
                    event.set('porcentagem_aprovado', aprovados);

                    pendentes = event.get(%{[PENDENTE][count]})/event.get(%{[events][count]})*100;
                    event.set('porcentagem_pendente', pendentes);
                    "
          add_tag => "metric"
        }
      }

}

output{

  stdout{
    codec => rubydebug
  }

  if "metric" in [tags]{

      elasticsearch{
        hosts => "localhost:9200"
        index => "calculos"
      }
  }


}

But now I must resolve how to map correctly to show the name of field and value on markdown.