HI,
Under high concurrency conditions on a Spark EMR cluster, elasticsearch is returning 0 hits for a query that would succeed otherwise. The query we are using involves geospatial joins across pre-indexed geoshapes.
We retry the requests when encountering this condition and the retries sometimes work, sometimes not.
I am not understanding why ES would return 0 results vs an http response that can be used to retry the requests. Also why do the results different for the same query?
we are running ES 5.5 via the AWS Elastic Service with this configuation:
Cluster size:
4 i3.2xlarge.elasticsearch worker nodes
2 shards/index
Index size: ~50-60GB primary
Here is a sample query that runs into this issue:
{"query":{
"bool" : {
"must" : [
{
"constant_score" : {
"filter" : {
"bool" : {
"should" : [
{
"geo_shape" : {
"start_loc" : {
"indexed_shape" : {
"id" : "3f712a7c-7d5f-4f16-87c6-3a673ef0e52d",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"end_loc" : {
"indexed_shape" : {
"id" : "3f712a7c-7d5f-4f16-87c6-3a673ef0e52d",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"start_loc" : {
"indexed_shape" : {
"id" : "770b8e13-6360-43e1-848f-806707370b89",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"end_loc" : {
"indexed_shape" : {
"id" : "770b8e13-6360-43e1-848f-806707370b89",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"start_loc" : {
"indexed_shape" : {
"id" : "94fe54a0-389d-4a1f-8382-6a1d5b22705f",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"end_loc" : {
"indexed_shape" : {
"id" : "94fe54a0-389d-4a1f-8382-6a1d5b22705f",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"start_loc" : {
"indexed_shape" : {
"id" : "e4baf91f-7258-4709-97d0-ef15afb0a27a",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
},
{
"geo_shape" : {
"end_loc" : {
"indexed_shape" : {
"id" : "e4baf91f-7258-4709-97d0-ef15afb0a27a",
"type" : "zone",
"index" : "zones_v1",
"path" : "geometry"
},
"relation" : "intersects"
},
"ignore_unmapped" : false,
"boost" : 1.0
}
}
],
"disable_coord" : false,
"adjust_pure_negative" : true,
"boost" : 1.0,
"_name" : "D"
}
},
"boost" : 1.0
}
},
{
"constant_score" : {
"filter" : {
"bool" : {
"must" : [
{
"range" : {
"startts" : {
"from" : "2018-04-01T04:00:00.000Z",
"to" : "2018-04-01T08:00:00.000Z",
"include_lower" : true,
"include_upper" : false,
"boost" : 1.0
}
}
},
{
"range" : {
"endts" : {
"from" : null,
"to" : "2018-04-03T00:00:00.000Z",
"include_lower" : true,
"include_upper" : false,
"boost" : 1.0
}
}
}
],
"disable_coord" : false,
"adjust_pure_negative" : true,
"boost" : 1.0
}
},
"boost" : 1.0
}
}
],
"disable_coord" : false,
"adjust_pure_negative" : true,
"boost" : 1.0
}
},"sort": [{"trip_id": {"order": "asc"}}]}
Thanks
Trang