Change in fuzzy + highlighting behavior between ES6 (6.7.2) and ES7 (7.9.0)

[mdunn (Mike Dunn)]

We are currently in the process of upgrading from ES6 (6.7.2) to ES7 (7.9.0) and we have noticed a change in behavior with regards to highlighting when fuzzy search is turned on. We are using the unified highlighter as we found it had much better performance than the fvh when used in conjunction with fuzzy searching (still not great performance).

These are some of the differences we've noticed between 6 and 7 where we were getting highlights in 6 but no longer getting them in 7 with the same query.

1. If a term is the same length as the prefix length and any "fuzziness" parameter is sent (including "0") the term will not be highlighted.
2. This one I'm less sure about the exact specifics but if we provide "AUTO:5,12" for a word that is stemmed to a term shorter than 5, it will not highlight the term.

This seems like something that could be due to the change in major versions from 7 -> 8 of Lucene but I haven't found any documentation from either elastic or lucene that we should expect a change in behavior. Is this a bug or do we need to find a work around?

Example create index (with mapping type omitted in 7):

{
  "settings": {
    "analysis": {
      "filter": {
        "light_stemmer": {
          "type": "stemmer",
          "language": "light_english"
        }
      },
      "analyzer": {
        "my_custom_analyzer": {
          "type": "custom", 
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "light_stemmer"
          ]
        }
      }
    }
  },
  "mappings": {
    "_doc": {
      "properties": {
        "field1": { 
          "type": "text",
          "analyzer": "my_custom_analyzer",
          "index_options": "offsets"
        }
      }
    }
  }
}

Example Document:

{
  "field1": "view"
}

Example query to repro #1

{
  "query": {
    "match": {
      "field1": {
        "query": "view",
        "operator": "OR",
        "fuzziness": "0",
        "prefix_length": 4,
        "max_expansions": 50,
        "fuzzy_transpositions": true,
        "lenient": false,
        "zero_terms_query": "NONE",
        "boost": 1.0
      }
    }
  },
  "highlight": {
    "pre_tags": [
      "<OPEN>"
    ],
    "post_tags": [
      "<CLOSE>"
    ],
    "number_of_fragments": 0,
    "type": "unified",
    "order": "score",
    "boundary_scanner_locale": "en",
    "fields": {
      "field1": {}
    }
  }
}

Example query to repro #2:

{
  "query": {
    "match": {
      "field1": {
        "query": "viewing",
        "operator": "OR",
        "fuzziness": "AUTO:5,12",
        "prefix_length": 3,
        "max_expansions": 50,
        "fuzzy_transpositions": true,
        "lenient": false,
        "zero_terms_query": "NONE",
        "boost": 1.0
      }
    }
  },
  "highlight": {
    "pre_tags": [
      "<OPEN>"
    ],
    "post_tags": [
      "<CLOSE>"
    ],
    "number_of_fragments": 0,
    "type": "unified",
    "order": "score",
    "boundary_scanner_locale": "en",
    "fields": {
      "field1": {}
    }
  }
}