Overview:
Amazon Elasticsearch Service
Version: 6.7
Number of Nodes: 3
When throwing an exception from a painless script we are getting inconsistent error messages back.
In some cases we get an extra "exception: " prepended to the message.
We have been seeing this sporadically during our test runs. We don't see it consistently happening on a single node. The script is the same on all three nodes.
This is causing some of our tests to fail as we are looking for the specific message and when we get the extra "exception: " we fail.
We have a temporary work around on our side but wanting to know if anyone else is experiencing this issue and/or could help us diagnose what is happening.
We have spoken to Amazon and they have requested we post on ES forum as it seems to be a usability issue.
Expected (See full stack trace below):
'lang': 'painless', 'caused_by': {'type': 'exception', 'reason': 'Make sure that you provide at least one object!'}
Not Expected (See full stack trace below):
'lang': 'painless', 'caused_by': {'type': 'exception', 'reason': 'exception: Make sure that you provide at least one object!'}
Script Sample:
// To check at least one object active for the outing after update
if (params.object_id_to_add.empty && !ctx._source.objectId.empty && ctx._source.objectId.every(t -> params.object_id_to_remove.contains(t))) {
throw new Exception("Make sure that you provide at least one object!");
}
------Begin Expected----------------
[ERROR] 2020-04-16T09:13:20.507Z got an exception during serialization Traceback (most recent call last): File "/var/task/lambda_common/elastic/xray_connections.py", line 27, in perform_request headers=headers File "/var/task/elasticsearch/connection/http_requests.py", line 90, in perform_request self._raise_error(response.status_code, raw_data) F
[ERROR] 2020-04-16T09:13:20.507Z got an exception during serialization
Traceback (most recent call last):
File "/var/task/lambda_common/elastic/xray_connections.py", line 27, in perform_request
headers=headers
File "/var/task/elasticsearch/connection/http_requests.py", line 90, in perform_request
self._raise_error(response.status_code, raw_data)
File "/var/task/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: RequestError(400, 'illegal_argument_exception', '[********][x.x.x.x:9300][indices:data/write/update[s]]')
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/var/task/aws_xray_sdk/core/models/entity.py", line 254, in serialize
return jsonpickle.encode(self, unpicklable=False)
File "/var/task/jsonpickle/pickler.py", line 75, in encode
return backend.encode(context.flatten(value, reset=reset))
File "/var/task/jsonpickle/pickler.py", line 181, in flatten
return self._flatten(obj)
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 264, in _ref_obj_instance
return self._flatten_obj_instance(obj)
File "/var/task/jsonpickle/pickler.py", line 405, in _flatten_obj_instance
return self._getstate(state, data)
File "/var/task/jsonpickle/pickler.py", line 559, in _getstate
state = self._flatten_obj(obj)
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 463, in _flatten_dict_obj
flatten(k, v, data)
File "/var/task/jsonpickle/pickler.py", line 538, in _flatten_key_value_pair
data[k] = self._flatten(v)
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 463, in _flatten_dict_obj
flatten(k, v, data)
File "/var/task/jsonpickle/pickler.py", line 538, in _flatten_key_value_pair
data[k] = self._flatten(v)
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 209, in _list_recurse
return [self._flatten(v) for v in obj]
File "/var/task/jsonpickle/pickler.py", line 209, in <listcomp>
return [self._flatten(v) for v in obj]
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 264, in _ref_obj_instance
return self._flatten_obj_instance(obj)
File "/var/task/jsonpickle/pickler.py", line 398, in _flatten_obj_instance
state = obj.__getstate__()
File "/var/task/aws_xray_sdk/core/models/throwable.py", line 73, in __getstate__
if not self.stack:
AttributeError: 'Throwable' object has no attribute 'stack'
[ERROR] 2020-04-16T09:13:20.508Z RequestError(400, 'illegal_argument_exception', {'error': {'root_cause': [{'type': 'remote_transport_exception', 'reason': '[********][x.x.x.x:9300][indices:data/write/update[s]]'}], 'type': 'illegal_argument_exception', 'reason': 'failed to execute script', 'caused_by': {'type': 'script_exception', 'reason': 'runtime error', 'script_stack': ['throw new Exception("Make sure that you provide at least one object!");\n}\n\n', ' ^---- HERE'], 'script': 'update_outing_events', 'lang': 'painless', 'caused_by': {'type': 'exception', 'reason': 'Make sure that you provide at least one object!'}}}, 'status': 400})
Traceback (most recent call last):
File "/var/task/objectcommon/outing_event_dao/elasticsearch_dao/elasticsearch_outing_event_dao.py", line 148, in update_outing_event
object_id_to_add=object_id_to_add)
File "/var/task/lambda_common/elastic/elastic_client.py", line 284, in update_by_script
body=json.dumps(body), index=index_name)
File "/var/task/elasticsearch/client/utils.py", line 76, in _wrapped
return func(*args, params=params, **kwargs)
File "/var/task/elasticsearch/client/__init__.py", line 547, in update
doc_type, id, '_update'), params=params, body=body)
File "/var/task/elasticsearch/transport.py", line 318, in perform_request
status, headers_response, data = connection.perform_request(method, url, params, body, headers=headers, ignore=ignore, timeout=timeout)
File "/var/task/lambda_common/elastic/xray_connections.py", line 27, in perform_request
headers=headers
File "/var/task/elasticsearch/connection/http_requests.py", line 90, in perform_request
self._raise_error(response.status_code, raw_data)
File "/var/task/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: RequestError(400, 'illegal_argument_exception', '[********][x.x.x.x:9300][indices:data/write/update[s]]')
------End Expected----------------
------Begin Not Expected----------------
[ERROR] 2020-04-16T08:55:52.843Z b2b92fe7-df43-4613-bc4d-3189f7401928 got an exception during serialization
Traceback (most recent call last):
File "/var/task/lambda_common/elastic/xray_connections.py", line 27, in perform_request
headers=headers
File "/var/task/elasticsearch/connection/http_requests.py", line 90, in perform_request
self._raise_error(response.status_code, raw_data)
File "/var/task/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: RequestError(400, 'illegal_argument_exception', '[*********][x.x.x.x:9300][indices:data/write/update[s]]')
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/var/task/aws_xray_sdk/core/models/entity.py", line 254, in serialize
return jsonpickle.encode(self, unpicklable=False)
File "/var/task/jsonpickle/pickler.py", line 75, in encode
return backend.encode(context.flatten(value, reset=reset))
File "/var/task/jsonpickle/pickler.py", line 181, in flatten
return self._flatten(obj)
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 264, in _ref_obj_instance
return self._flatten_obj_instance(obj)
File "/var/task/jsonpickle/pickler.py", line 405, in _flatten_obj_instance
return self._getstate(state, data)
File "/var/task/jsonpickle/pickler.py", line 559, in _getstate
state = self._flatten_obj(obj)
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 463, in _flatten_dict_obj
flatten(k, v, data)
File "/var/task/jsonpickle/pickler.py", line 538, in _flatten_key_value_pair
data[k] = self._flatten(v)
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 463, in _flatten_dict_obj
flatten(k, v, data)
File "/var/task/jsonpickle/pickler.py", line 538, in _flatten_key_value_pair
data[k] = self._flatten(v)
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 209, in _list_recurse
return [self._flatten(v) for v in obj]
File "/var/task/jsonpickle/pickler.py", line 209, in <listcomp>
return [self._flatten(v) for v in obj]
File "/var/task/jsonpickle/pickler.py", line 185, in _flatten
return self._pop(self._flatten_obj(obj))
File "/var/task/jsonpickle/pickler.py", line 206, in _flatten_obj
return flatten_func(obj)
File "/var/task/jsonpickle/pickler.py", line 264, in _ref_obj_instance
return self._flatten_obj_instance(obj)
File "/var/task/jsonpickle/pickler.py", line 398, in _flatten_obj_instance
state = obj.__getstate__()
File "/var/task/aws_xray_sdk/core/models/throwable.py", line 73, in __getstate__
if not self.stack:
AttributeError: 'Throwable' object has no attribute 'stack'
[ERROR] 2020-04-16T08:55:52.844Z RequestError(400, 'illegal_argument_exception', {'error': {'root_cause': [{'type': 'remote_transport_exception', 'reason': '[*********][x.x.x.x:9300][indices:data/write/update[s]]'}], 'type': 'illegal_argument_exception', 'reason': 'failed to execute script', 'caused_by': {'type': 'script_exception', 'reason': 'runtime error', 'script_stack': ['throw new Exception("Make sure that you provide at least one object!");\n}\n\n', ' ^---- HERE'], 'script': 'update_outing_events', 'lang': 'painless', 'caused_by': {'type': 'exception', 'reason': 'exception: Make sure that you provide at least one object!'}}}, 'status': 400})
Traceback (most recent call last):
File "/var/task/objectcommon/outing_event_dao/elasticsearch_dao/elasticsearch_outing_event_dao.py", line 148, in update_outing_event
object_id_to_add=object_id_to_add)
File "/var/task/lambda_common/elastic/elastic_client.py", line 284, in update_by_script
body=json.dumps(body), index=index_name)
File "/var/task/elasticsearch/client/utils.py", line 76, in _wrapped
return func(*args, params=params, **kwargs)
File "/var/task/elasticsearch/client/__init__.py", line 547, in update
doc_type, id, '_update'), params=params, body=body)
File "/var/task/elasticsearch/transport.py", line 318, in perform_request
status, headers_response, data = connection.perform_request(method, url, params, body, headers=headers, ignore=ignore, timeout=timeout)
File "/var/task/lambda_common/elastic/xray_connections.py", line 27, in perform_request
headers=headers
File "/var/task/elasticsearch/connection/http_requests.py", line 90, in perform_request
self._raise_error(response.status_code, raw_data)
File "/var/task/elasticsearch/connection/base.py", line 125, in _raise_error
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
elasticsearch.exceptions.RequestError: RequestError(400, 'illegal_argument_exception', '[*********][x.x.x.x:9300][indices:data/write/update[s]]')
------End Not Expected----------------