Whenever we run into this scenario where a sniff happened and elastic was unable to connect, we then see this timeout with a massive negative number.
I'm not even sure where to start with this. Any help would be appreciated.
We're in the process of upgrading elastic, but this is currently happening with 5.6.
Connection Settings
var elasticPool = new Elasticsearch.Net.SniffingConnectionPool(nodes);
elasticSettings = new ConnectionSettings(elasticPool);
elasticSettings.SniffOnStartup(true)
.SniffOnConnectionFault(true)
.SniffLifeSpan(TimeSpan.FromMinutes(30));
.ConnectionLimit(160)
.DefaultIndex("jobs")
.DisableDirectStreaming(true)
.EnableHttpCompression(true)
.MaximumRetries(1)
.RequestTimeout(new TimeSpan(0, 0, 5))
.ThrowExceptions(true);
Response
Unsuccessful low level call on POST: /index/message%2CmessageChild/_search?routing=ID_GOES_HERE
# Audit trail of this API call :
- [1]BadResponse: Node: http: //1.1.1.1:9200/ Took: 00:00:05.0158085
- [2]SniffOnFail: Took: 00: 00: 00.0060049
- [3]SniffSuccess: Node: http: //1.1.1.1:9200/ Took: 00:00:00.0060049
- [4]MaxTimeoutReached: Took: -738460.19: 12: 17.8729319
# OriginalException: System.Net.WebException: The operation has timed out
at System.Net.HttpWebRequest.GetResponse()
at Elasticsearch.Net.HttpWebRequestConnection.Request[TReturn](RequestData requestData)
# Request: {
"aggs": {
"Type": {
"terms": {
"field": "type",
"order": [{
"_count": "desc"
}
],
"size": 100
}
},
"MessageType": {
"terms": {
"field": "messageType",
"order": [{
"_count": "desc"
}
],
"size": 100
}
},
"CreatedBy": {
"terms": {
"field": "createdBy",
"order": [{
"_count": "desc"
}
],
"size": 200
}
}
},
"from": 0,
"query": {
"bool": {
"must": [{
"terms": {
"locationId": ["ID_GOES_HERE"]
}
}, {
"terms": {
"roleId": ["1111111", "11", "111", "1111", "0000001", "0000011", "0000101", "0000111", "0001001", "0001011", "0001101", "0001111", "0010001", "0010011", "0010101", "0010111", "0011001", "0011011", "0011101", "0011111", "0100001", "0100011", "0100101", "0100111", "0101001", "0101011", "0101101", "0101111", "0110001", "0110011", "0110101", "0110111", "0111001", "0111011", "0111101", "0111111"]
}
}, {
"terms": {
"jobId": ["ID_GOES_HERE"]
}
}, {
"bool": {
"minimum_should_match": 1,
"should": [{
"match": {
"_all": {
"max_expansions": 1000,
"prefix_length": 2,
"query": "free",
"type": "phrase_prefix"
}
}
}, {
"has_child": {
"query": {
"match": {
"_all": {
"max_expansions": 1000,
"prefix_length": 2,
"query": "free",
"type": "phrase_prefix"
}
}
},
"type": "messageChild"
}
}
]
}
}, {
"bool": {
"must_not": [{
"exists": {
"field": "parentMessageId"
}
}
]
}
}
]
}
},
"size": 20,
"sort": [{
"createdDate": {
"order": "desc"
}
}
]
}