I have several indexes spread across 3 shards that hold my data. I want to make a SearchAfter where I fetch 50 rows ordered by start time. When I run it I am returned 50 hits but half of them are duplicates as the data is spread to several indices. I thought it was to the query then detach approach but even after I switch to DfsQueryThenFetch I still return duplicates. How do I filter out duplicates (preferably by guid field)? Code below
//reduced for brevity
List<resultObjects> result = new List<resultObjects>();
DateRangeQuery dq= new DateRangeQuery()
{
Name = "StartTime",
Field = "StartTime",
GreaterThanOrEqualTo = from,
LessThanOrEqualTo = to
};
TermQuery tq1= new TermQuery()
{
Name = "Location",
Field = "Location"
Value = "sds"
};
BoolQuery boolQuery = new BoolQuery();
boolQuery.Filter = new List<QueryContainer>() { dq, tq };
boolQuery.Must = new List<QueryContainer>()
{
new MatchQuery()
{
Name = "user",
Field ="user",
Query= "test"
}
};
var searchRequest = new SearchRequest("data_index_*");
searchRequest.SearchType = SearchType.DfsQueryThenFetch;
searchRequest.From = 0;
searchRequest.Size = 50;
searchRequest.Query = boolQuery;
searchRequest.Sort = new List<ISort>
{
new FieldSort { Field=Infer.Field<resultObjects>
(p=>p.StartTime),Order=SortOrder.Ascending}
};
if (searchAfterTime < to)
searchRequest.SearchAfter = new List<object> { searchAfterTime };