Scrolling through entire result set is taking too much time

Elastic Stack Elasticsearch
1

I am using JAVA API to search and then scroll through search result of abt
800K. It takes abt 15 min to scroll through that 800K result set.
My search type is SCAN and using page size as 10,000 and scroll time as 6
min.
I am using Elasticsearch version 0.90.0.
any help will be appreciated.

Here is the code sample

   try {
        SearchRequestBuilder srb = getClient()
                .prepareSearch()
                .setSearchType(SearchType.SCAN)
                .setScroll(new TimeValue(600000))
                .setIndices("exp_mgr_min_test_v2")
                .setQuery(QueryBuilders.matchPhraseQuery(

"EXP_EXPENSE_STATE", "CA"))
.setSize(10000)
.setExtraSource(
XContentFactory.jsonBuilder().startObject().
startObject("facets").startObject("facet1")
.startObject("script").field(
"init_script", "init").field("map_script", "map_DC")
.field("reduce_script", "reduce_DC").
startObject("params").startArray("facet")
.endArray().startArray("fields").value(
"EXP_HCP_ID").value("S_EXP_EXPENSE_DATE")
.endArray().field("field", "DATA_ID")
.field("value_field",
"S_EXP_RECIPIENT_AMOUNT").endObject().endObject().endObject()
.endObject().endObject());
srb.addScriptField("RECIPIENT_ID",
"_source.EVENT_EXPENSE_RECIPIENT_DATA.EXP_HCP_ID");
srb.addScriptField("EXPENSE_DATE",
"_source.EVENT_EXPENSE_RECIPIENT_DATA.S_EXP_EXPENSE_DATE");

        intime = new Timestamp(System.currentTimeMillis());
        searchResponse = srb.execute().actionGet();

    } catch (Exception e) {

    }

    ScriptFacet facet = searchResponse.getFacets().facet("facet1");
    Map<String, Double> fr =  (Map<String, Double>)((Map<String, Object

) facet.facet()).get("total");

    try {
        while (true) {
            for (SearchHit hit : searchResponse.getHits()) {

                String dateFormatted = (hit.field("EXPENSE_DATE").

getValue() == null ? "NULL" : dfFormat
.format(dfParse.parse(hit.field("EXPENSE_DATE").
getValue().toString())));

                String key = "+"
                        + (hit.field("RECIPIENT_ID").getValue() == null

? "NULL" : hit.field("RECIPIENT_ID")
.getValue().toString().toLowerCase()) +
"+" + dateFormatted;
if (fr.get(key) != null) {
StringBuilder result = new StringBuilder();
result.append(DELIM);
result.append(hit.field("RECIPIENT_ID").getValue());
result.append(DELIM);
result.append(hit.field("EXPENSE_DATE").getValue());
result.append("\n");
}
}
searchResponse = client.prepareSearchScroll(searchResponse.
getScrollId())
.setScroll(new TimeValue(600000)).execute().
actionGet();

            if (searchResponse.getHits().hits().length == 0) {
                break;
            }
}
        System.out.println((new Timestamp(System.currentTimeMillis()).

getTime() - intime.getTime())/1000);
} catch (Exception e) {
throw new RuntimeException(e);
}

}

--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
For more options, visit https://groups.google.com/groups/opt_out.