Getting a no_viable_alt_exception only when trying to filter using a scripted field

TL/DR: Big script works fine for calculating on documents but throws exception when filtering

I've built a quite involved scripted field to deal with a count in a list of strings, it also dealt with parsing dates in those strings. Here it is:

int getMonth(String sigla){
  if(sigla == null){
    return 0;
  } else if(sigla.equals("JAN")){
    return 1;
  } else if(sigla.equals("FEB")){
    return 2;
  } else if(sigla.equals("MAR")){
    return 3;
  } else if(sigla.equals("APR")){
    return 4;
  } else if(sigla.equals("MAY")){
    return 5;
  } else if(sigla.equals("JUN")){
    return 6;
  } else if(sigla.equals("JUL")){
    return 7;
  } else if(sigla.equals("AUG")){
    return 8;
  } else if(sigla.equals("SEP")){
    return 9;
  } else if(sigla.equals("OCT")){
    return 10;
  } else if(sigla.equals("NOV")){
    return 11;
  } else if(sigla.equals("DEC")){
    return 12;
  }
  return 0;
}

String cleanString(String dirty){
  if(dirty == null)
    return '';
  return dirty.toUpperCase()
    .trim()
    .replace('Á','A')
    .replace('Ã','A')
    .replace('Â','A')
    .replace('É','E')
    .replace('Ê','E')
    .replace('Í','I')
    .replace('Î','I')
    .replace('Ó','O')
    .replace('Ô','O')
    .replace('Õ','O')
    .replace('Ú','U')
    .replace('Û','U')
    .replace('Ç','C')
    .replace('/','')
    .replace('.','');
}

LocalDate getDateFromOracleString(String origin){
  int dia, mes, ano;
  if(origin.length() < 9)
    return null;
  mes = getMonth(origin.substring(3,6));
  try {
    dia = Integer.parseInt(origin.substring(0,2));
    ano = Integer.parseInt('20'+origin.substring(7,9));
  } catch(NumberFormatException e){
    return null;
  }
  if(mes == 0 || ano < 0 || dia < 1 || dia > 31)
    return null;
  return LocalDate.of(ano,mes,dia);
}

boolean contLiberadas = false;

Pattern separadorTarefa = /\ \|\ /;
Pattern partesTarefa = /;/;

def tarefasString = doc['lista_tarefas.keyword'].value;
if(tarefasString == null)
  return 0;
if(tarefasString.empty || tarefasString.equals(''))
  return 0;

def listaTarefas = new LinkedList();
String[] tarefas = separadorTarefa.split(tarefasString);

int i;
String[] tarefaEmPartes;
for (i=0 ; i<tarefas.length ; i++){
  tarefaEmPartes = partesTarefa.split(tarefas[i]);
  listaTarefas.add(tarefaEmPartes);
}

def timeZone = ZoneId.of(ZoneId.SHORT_IDS.get('BET'));

LocalDate dataIniMesFechado = Instant.ofEpochMilli(new Date().getTime()).atZone(timeZone).toLocalDate().minusMonths(1).withDayOfMonth(1);
LocalDate dataFimMesFechado = Instant.ofEpochMilli(new Date().getTime()).atZone(timeZone).toLocalDate().withDayOfMonth(1).minusDays(1);
LocalDate dataRegistro;
LocalDate dataConfirmacao;
String tipoTarefa;
String ultTipoTarefa = null;
int statusTarefa;
boolean confirmada;

ListIterator l = listaTarefas.listIterator(listaTarefas.indexOf(listaTarefas.peekFirst()));
def cont = 0;

while(l.hasNext()){
  tarefaEmPartes = l.next();

  if(tarefaEmPartes == null)
    throw new Exception('Formato de tarefa inválido - Tarefa nula na lista: ' + tarefasString);
  if(tarefaEmPartes.length < 5)
    throw new Exception('Formato de tarefa inválido - Tarefa: ' + tarefaEmPartes[0] + ' de tamanho inválido na lista: ' + tarefasString);

try{
    statusTarefa = Integer.parseInt(tarefaEmPartes[5]);
  } catch(NumberFormatException e){
    if(tarefaEmPartes[5] == null)
      throw new Exception('Formato de tarefa inválido - Status NULO na tarefa: ' + tarefaEmPartes[0] + ':' + tarefaEmPartes[4] + ' na lista: ' + tarefasString);
    throw new Exception('Formato de tarefa inválido - Falha em interpretar o status: '+tarefaEmPartes[5] + 'na tarefa: ' + tarefaEmPartes + ' na lista: '+tarefasString);
  }

  if(statusTarefa != 3){
    dataRegistro = getDateFromOracleString(tarefaEmPartes[0]);
    if(dataRegistro == null)
      throw new Exception('Formato de tarefa inválido - Falha em interpretar a data de registro: '+tarefaEmPartes[0] + 'na tarefa: ' + tarefaEmPartes + ' na lista: '+tarefasString);

    if(!tarefaEmPartes[2].equals('N/A')){
      confirmada = true;
      dataConfirmacao = getDateFromOracleString(tarefaEmPartes[2]);
      if(dataConfirmacao == null)
        throw new Exception('Formato de tarefa inválido - Falha em interpretar a data de confirmação: '+tarefaEmPartes[2] + 'na tarefa: ' + tarefaEmPartes + ' na lista: '+tarefasString);
    } else {
      confirmada = false;
    }

    tipoTarefa = cleanString(tarefaEmPartes[4]);
    if(tipoTarefa == '')
      throw new Exception('Formato de tarefa inválido - Tipo da tarefa vazio na tarefa: '+tarefaEmPartes[0] + 'na lista: ' + tarefaEmPartes + ' na lista: '+tarefasString);

    if(contLiberadas) {
      if(dataRegistro.isAfter(dataIniMesFechado.minusDays(1)) && dataRegistro.isBefore(dataFimMesFechado.plusDays(1))) {
        if(tipoTarefa.contains('CCC')){
          if(!ultTipoTarefa.contains('CCC') && !ultTipoTarefa.contains('BBB')){
            cont++;
          }
        } else if (tipoTarefa.contains('BBB')){
          cont++;
        } else if (tipoTarefa.contains('AAA')){
          cont++;
        }
      }
    }
    else{
      if(tipoTarefa.equals('XXX') || tipoTarefa.equals('YYY')) {
        if(confirmada == true){
          if(dataConfirmacao.isAfter(dataIniMesFechado.minusDays(1)) && dataConfirmacao.isBefore(dataFimMesFechado.plusDays(1))) {
              cont++;
          }
        }
      }
    }
    ultTipoTarefa = tipoTarefa;
  }
}
return cont;

The code works fine (and yes, I know it's wonky) to calculate what it needs to calculate, but whenever I try to filter using it it throws this exception:

"type":"illegal_argument_exception","reason":"invalid sequence of tokens near ['('].","caused_by":{"type":"no_viable_alt_exception","reason":null

While pointing to this:

"type":"illegal_argument_exception","reason":"invalid sequence of tokens near ['('].","caused_by":{"type":"no_viable_alt_exception","reason":null}}}}}],"caused_by":{"type":"script_exception","reason":"compile error","script_stack":["... ou inválida
int getMonth(String sigla){
 if(sigl ..."," ^---- HERE"],"script":"boolean gt(Supplier s, def v) {return s.get() > v}gt(()

I've really looked into it and I can't find what's at fault. If anyone feels like taking a look at that spaghetti and giving me a hand I'd be very thankful.

@ebalmeida,

If your data isn't sensitive, and if it doesn't contain personal user data, would you be willing to send me the index mapping and sample data from that index? I can try to reproduce it on my end that way.

If not, could you send the index mapping, and I can fabricate dummy data to work with.

Hi Larry,

The data itself is, unfortunately, quite sensitive. I can, however, send the mapping, no problem.

{
  "mapping": {
    "data": {
      "properties": {
        "data_registrado_contrato": {
          "type": "date"
        },
        "data_vigencia_fim": {
          "type": "date"
        },
        "data_vigencia_inicio": {
          "type": "date"
        },
        "departamento": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "departamento_ec": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "dois_ou_mais_shoppings": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "empresa": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "id": {
          "type": "float"
        },
        "id_contrato": {
          "type": "float"
        },
        "instrumentos": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "lista_tarefas": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword"
            }
          }
        },
        "loja_ancora": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "nome_outras_partes": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "nome_parte_ativa": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "nome_parte_passiva": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "origem_da_solicitacao": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "qtde_de_instrumentos": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "solicitante_responsavel": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "status_contrato": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "tipo_do_contrato": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "valor_contrato_total": {
          "type": "float"
        }
      }
    }
  }
}

As for lista_tarefas, which the script directly references, it comes in this format:
DD-MMM-YY hh.mm.ss.aaaZZZ AM/PM; DD-MMM-YY hh.mm.ss.aaaZZZ AM/PM; DD-MMM-YY hh.mm.ss.aaaZZZ AM/PM; "Task Id Number"; "Task Type"; "Status code" | Next task | Next task...

For example:

27-MAY-19 07.27.16.441000 PM;03-JUN-19 07.27.16.397000 PM;27-MAY-19 07.35.39.056000 PM;197773;Task Type A;2 | 27-MAY-19 07.35.39.109000 PM;27-MAY-19 07.35.39.070000 PM;27-MAY-19 07.35.52.245000 PM;197815;Task Type B;2 | 27-MAY-19 07.35.52.339000 PM;08-JUL-19 07.35.52.263000 PM;N/A;197816;Task Type C;1

Thank you so very much for looking into it.

Thanks for the mapping. I have good news and bad news. The good news is that I was able to reproduce the error you're seeing.

The bad news is that it's related to how Kibana has to wrap the painless script in order to execute it in the context of a filter. It places your script into a lambda in order to run it: https://github.com/elastic/kibana/blob/01c862762263541a006f539f5a26ffdd4881b103/packages/kbn-es-query/src/filters/phrase.js#L81:L84

Based on that, I'm thinking that you can't declare inline functions such as getMonth, cleanString, etc within a lamdba. I'm not sure where to go from here, as I'm sure you don't want to inline that logic every place you're using it in the script. I'll see if any Painless experts are around who might be able to help a bit more.

Yeah, a lambda wouldn't (afaik) be able to use the inline functions that painless allows. =/
I heard talk that the way filters treat scripted fields would change in the future (specifically to the fact that filters store the script so that when you change the scripts filters don't update). Maybe a future implementation could think of a different way to treat it and use the full functionality of scripted fields.

Well, I managed to get a different error, which is progress, I guess.
I tweaked the code to get rid of the inline functions and see if it would work. Here's the full stack trace with the new code.

Request to Elasticsearch failed: {"error":{"root_cause":[{"type":"script_exception","reason":"compile error","script_stack":[],"script":"boolean compare(Supplier s, def v) {return s.get() == v;}compare(() -> { 
boolean contLiberadas = false;
Pattern separadorTarefa = /\\ \\|\\ /;
Pattern partesTarefa = /;/;
def tarefasString = doc['lista_tarefas.keyword'].value;
if(tarefasString == null)
 return 0;
if(tarefasString.empty || tarefasString.equals(''))
 return 0;
def listaTarefas = new LinkedList();
String[] tarefas = separadorTarefa.split(tarefasString);
int i;
String[] tarefaEmPartes;
for (i=0 ; i<tarefas.length ; i++){
 tarefaEmPartes = partesTarefa.split(tarefas[i]);
 listaTarefas.add(tarefaEmPartes);
}
def timeZone = ZoneId.of(ZoneId.SHORT_IDS.get('BET'));
LocalDate dataIniMesFechado = Instant.ofEpochMilli(new Date().getTime()).atZone(timeZone).toLocalDate().minusMonths(1).withDayOfMonth(1);
LocalDate dataFimMesFechado = Instant.ofEpochMilli(new Date().getTime()).atZone(timeZone).toLocalDate().withDayOfMonth(1).minusDays(1);
LocalDate dataRegistro;
LocalDate dataConfirmacao;
String tipoTarefa;
String ultTipoTarefa = null;
int statusTarefa;
boolean confirmada;
DateTimeFormatter formato = DateTimeFormatter.ofPattern('dd-MMM-yy');
ListIterator l = listaTarefas.listIterator(listaTarefas.indexOf(listaTarefas.peekFirst()));
def cont = 0;
while(l.hasNext()){
 tarefaEmPartes = l.next();
 if(tarefaEmPartes == null)
 throw new Exception('Formato de tarefa inválido - Tarefa nula na lista: ' + tarefasString);
 if(tarefaEmPartes.length < 5)
 throw new Exception('Formato de tarefa inválido - Tarefa: ' + tarefaEmPartes[0] + ' de tamanho inválido na lista: ' + tarefasString);
try{
 statusTarefa = Integer.parseInt(tarefaEmPartes[5]);
 } catch(NumberFormatException e){
 if(tarefaEmPartes[5] == null)
 throw new Exception('Formato de tarefa inválido - Status NULO na tarefa: ' + tarefaEmPartes[0] + ':' + tarefaEmPartes[4] + ' na lista: ' + tarefasString);
 throw new Exception('Formato de tarefa inválido - Falha em interpretar o status: '+tarefaEmPartes[5] + 'na tarefa: ' + tarefaEmPartes + ' na lista: '+tarefasString);
 }
 if(statusTarefa != 3){
 dataRegistro = LocalDate.parse(tarefaEmPartes[0].substring(0,3)+tarefaEmPartes[0].substring(3,4).toUpperCase()+tarefaEmPartes[0].substring(4,6).toLowerCase()+tarefaEmPartes[0].substring(6,9),formato);
 if(dataRegistro == null)
 throw new Exception('Formato de tarefa inválido - Falha em interpretar a data de registro: '+tarefaEmPartes[0] + 'na tarefa: ' + tarefaEmPartes + ' na lista: '+tarefasString);
 if(!tarefaEmPartes[2].equals('N/A')){
 confirmada = true;
 dataConfirmacao = LocalDate.parse(tarefaEmPartes[2].substring(0,3)+tarefaEmPartes[2].substring(3,4).toUpperCase()+tarefaEmPartes[2].substring(4,6).toLowerCase()+tarefaEmPartes[2].substring(6,9),formato);
 if(dataConfirmacao == null)
 throw new Exception('Formato de tarefa inválido - Falha em interpretar a data de confirmação: '+tarefaEmPartes[2] + 'na tarefa: ' + tarefaEmPartes + ' na lista: '+tarefasString);
 } else {
 confirmada = false;
 }
 tipoTarefa = tarefaEmPartes[4].toUpperCase()
 .trim()
 .replace('Á','A')
 .replace('Ã','A')
 .replace('Â','A')
 .replace('É','E')
 .replace('Ê','E')
 .replace('Í','I')
 .replace('Î','I')
 .replace('Ó','O')
 .replace('Ô','O')
 .replace('Õ','O')
 .replace('Ú','U')
 .replace('Û','U')
 .replace('Ç','C')
 .replace('/','')
 .replace('.','');
 if(tipoTarefa == '')
 throw new Exception('Formato de tarefa inválido - Tipo da tarefa vazio na tarefa: '+tarefaEmPartes[0] + 'na lista: ' + tarefaEmPartes + ' na lista: '+tarefasString);
 if(contLiberadas) {
 if(dataRegistro.isAfter(dataIniMesFechado.minusDays(1)) && dataRegistro.isBefore(dataFimMesFechado.plusDays(1))) {
 if(tipoTarefa.contains('CCC')){
 if(!ultTipoTarefa.contains('CCC') && !ultTipoTarefa.contains('BBB')){
 cont++;
 }
 } else if (tipoTarefa.contains('BBB')){
 cont++;
 } else if (tipoTarefa.contains('AAA')){
 cont++;
 }
 }
 }
 else{
 if(tipoTarefa.equals('XXX') || tipoTarefa.equals('YYY')) {
 if(confirmada == true){
 if(dataConfirmacao.isAfter(dataIniMesFechado.minusDays(1)) && dataConfirmacao.isBefore(dataFimMesFechado.plusDays(1))) {
 cont++;
 }
 }
 }
 }
 ultTipoTarefa = tipoTarefa;
 }
}
return cont;
 }, params.value);","lang":"painless"}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":"sonae_090_contratos","node":"fRwyxY9eT6CLhfp-woSrKw","reason":{"type":"query_shard_exception","reason":"failed to create query: {
 \"bool\" : {
 \"must\" : [
 {
 \"range\" : {
 \"data_registrado_contrato\" : {
 \"from\" : 1401371972168,
 \"to\" : 1559138372169,
 \"include_lower\" : true,
 \"include_upper\" : true,
 \"format\" : \"epoch_millis\",
 \"boost\" : 1.0
 }
 }
 },
 {
 \"range\" : {
 \"data_registrado_contrato\" : {
 \"from\" : \"now-1M/M\",
 \"to\" : \"now/M\",
 \"include_lower\" : true,
 \"include_upper\" : false,
 \"boost\" : 1.0
 }
 }
 },
 {
 \"script\" : {
 \"script\" : {
 \"source\" : \"boolean compare(Supplier s, def v) {return s.get() == v;}compare(() -> {
boolean contLiberadas = false;\
...
return cont;\
 }, params.value);\",
 \"lang\" : \"painless\",
 \"params\" : {
 \"value\" : 1
 }
 },
 \"boost\" : 1.0
 }
 }
 ],
 \"filter\" : [
 {
 \"match_all\" : {
 \"boost\" : 1.0
 }
 }
 ],
 \"adjust_pure_negative\" : true,
 \"boost\" : 1.0
 }
}","index_uuid":"ory3m1FES1mywtPkcbYw0Q","index":"sonae_090_contratos","caused_by":{"type":"script_exception","reason":"compile error","script_stack":[],"script":"boolean compare(Supplier s, def v) {return s.get() == v;}compare(() -> { 
boolean
...
return cont;
 }, params.value);","lang":"painless","caused_by":{"type":"null_pointer_exception","reason":null}}}}],"caused_by":{"type":"script_exception","reason":"compile error","script_stack":[],"script":"boolean compare(Supplier s, def v) {return s.get() == v;}compare(() -> { 
boolean
...
return cont;
 }, params.value);","lang":"painless","caused_by":{"type":"null_pointer_exception","reason":null}}},"status":400}

It still calculates fine, just not when in a filter.

I created another post here to reference null pointers in filters using scripted fields.