We are currently upgrading our elastic stack from using the elasticsearch-rest-high-level-client 6.2.4 to elasticsearch-java client 8.19.0 and upgrading the server to match. While upgrading we have run into a problem that we did not experience with the high level rest client.
Truncated chunk (expected size: 222,493; actual size: 213,649)
We only get this error when we turn on xpack.security.http.ssl.enabled inelasticsearch.yml and enable it in our A10 thunder. If we turn off SSL to port 9200 all indexing works.
From my intesive research our bulks are set to defaults everywhere with a 1 second flush. Each bulk is under 2mb where the max is 5mb. Our setup almost exactly matches the examples found at https://github.com/elastic/elasticsearch-java/tree/v8.19.1/examples
We also turned on logging to our A10 and don’t see any dropped chunks or incomplete chunks from it.
I have also used https://www.elastic.co/docs/troubleshoot/elasticsearch and flipped as many knobs and switchs as I could.
I’m aware of two deviations from best practices:
- Document IDs are strings.
- Only one thread is performing the bulk indexing.
liberty | [8/14/25 15:17:47:909 EDT] 0000009d SystemOut O 15:17:47.909 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][284][0][0]: 8175 bytes readliberty | [8/14/25 15:17:47:923 EDT] 0000009d SystemOut O 15:17:47.922 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.wire - http-outgoing-1 << "ary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0171821, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568794,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0173309, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568795,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0156570, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568796,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0174508, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568797,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0179295, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568798,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0180697, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568799,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0176498, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568800,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0182953, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568801,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187111, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568802,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568803,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0193795, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568804,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0184412, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568805,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0186421, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568806,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0186407, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568807,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0184473, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568808,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0186521, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568809,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187767, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568810,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189126, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568811,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189694, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568812,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0188482, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568813,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189689, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568814,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0188500, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568815,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0186412, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568816,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187787, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568817,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187789, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568818,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187745, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568819,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187779, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568820,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187790, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568821,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187769, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568822,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187771, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568823,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0191894, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568824,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187775, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568825,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0190936, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568826,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0190865, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568827,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0191745, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568828,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0192808, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568829,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0192785, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568830,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187796, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"tota"liberty | [8/14/25 15:17:47:929 EDT] 0000009d SystemOut O 15:17:47.929 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][284][0][0]: 0 bytes readliberty | [8/14/25 15:17:47:932 EDT] 0000009d SystemOut O 15:17:47.932 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.InternalIODispatch - http-outgoing-1 [ACTIVE] [chunk-coded; completed: false]liberty | [8/14/25 15:17:47:935 EDT] 0000009d SystemOut O 15:17:47.935 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.InternalIODispatch - http-outgoing-1 [ACTIVE] Input readyliberty | [8/14/25 15:17:47:935 EDT] 0000009d SystemOut O 15:17:47.935 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.MainClientExec - [exchange: 6] Consume contentliberty | [8/14/25 15:17:47:941 EDT] 0000009d SystemOut O 15:17:47.941 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][0][3884][0]: 8192 bytes readliberty | [8/14/25 15:17:47:947 EDT] 0000009d SystemOut O 15:17:47.946 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.wire - http-outgoing-1 << "l":2,"successful":2,"failed":0},"_seq_no":568831,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0187777, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568832,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195687, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568833,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0197210, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568834,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189271, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568835,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195682, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568836,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195679, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568837,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195673, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568838,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189285, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568839,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189282, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568840,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0189299, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568841,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0191687, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568842,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0180513, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568843,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195385, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568844,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195410, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568845,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0194822, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568846,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195407, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568847,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195398, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568848,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195428, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568849,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195394, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568850,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195372, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568851,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195384, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568852,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195092, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568853,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0190566, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568854,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0196104, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568855,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0197446, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568856,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0196127, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568857,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0190569, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568858,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0196128, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568859,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195149, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568860,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195111, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568861,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0190552, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568862,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0190571, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568863,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0197297, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568864,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195102, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568865,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0191455, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568866,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195067, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568867,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_METERSETTING, 0195128, G-RES_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568868,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 20065_feede"liberty | [8/14/25 15:17:47:959 EDT] 0000009d SystemOut O 15:17:47.959 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][0][0][0]: 3884 bytes readliberty | [8/14/25 15:17:47:962 EDT] 0000009d SystemOut O 15:17:47.961 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.wire - http-outgoing-1 << "r_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568869,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 59320, 4\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568870,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 59321, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568871,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 59744, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568872,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 59745, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568873,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 8809, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568874,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 8810, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568875,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 23558, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568876,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 23559, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568877,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 23560, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568878,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 23557, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568879,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 17729, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568880,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 17727, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568881,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 20066, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568882,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 17728, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568883,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 23563, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568884,"_primary_term":1,"status":200}},{"index":{"_index":"map-feature","_id":"G_VALVE_CONTROL, Dist. Valve, 23562, 2\-, Open_feeder_ga_2165","_version":2,"result":"updated","_shards":{"total":2,"successful":2,"failed":0},"_seq_no":568885,"_primary_term":1,"status":200}}]}"liberty | [8/14/25 15:17:47:969 EDT] 0000009d SystemOut O 15:17:47.969 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][0][0][0]: 0 bytes readliberty | [8/14/25 15:17:47:971 EDT] 0000009d SystemOut O 15:17:47.971 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][0][0][0]: 0 bytes readliberty | [8/14/25 15:17:47:973 EDT] 0000009d SystemOut O 15:17:47.973 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.InternalIODispatch - http-outgoing-1 [ACTIVE] [chunk-coded; completed: false]liberty | [8/14/25 15:17:49:279 EDT] 0000009d SystemOut O 15:17:49.279 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.InternalIODispatch - http-outgoing-1 [ACTIVE] Input readyliberty | [8/14/25 15:17:49:280 EDT] 0000009d SystemOut O 15:17:49.280 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.MainClientExec - [exchange: 6] Consume contentliberty | [8/14/25 15:17:49:281 EDT] 0000009d SystemOut O 15:17:49.281 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][EOF][0][0][0]: -1 bytes readliberty | [8/14/25 15:17:49:282 EDT] 0000009d SystemOut O 15:17:49.282 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.client.InternalIODispatch - http-outgoing-1 [ACTIVE] Exceptionliberty | org.apache.http.MalformedChunkCodingException: CRLF expected at end of chunkliberty | at org.apache.http.impl.nio.codecs.ChunkDecoder.readChunkHead(ChunkDecoder.java:112)liberty | at org.apache.http.impl.nio.codecs.ChunkDecoder.read(ChunkDecoder.java:205)liberty | at org.apache.http.nio.util.SimpleInputBuffer.consumeContent(SimpleInputBuffer.java:66)liberty | at org.elasticsearch.client.HeapBufferedAsyncResponseConsumer.onContentReceived(HeapBufferedAsyncResponseConsumer.java:97)liberty | at org.apache.http.nio.protocol.AbstractAsyncResponseConsumer.consumeContent(AbstractAsyncResponseConsumer.java:147)liberty | at co.elastic.clients.transport.rest_client.SafeResponseConsumer.consumeContent(SafeResponseConsumer.java:78)liberty | at org.apache.http.impl.nio.client.MainClientExec.consumeContent(MainClientExec.java:329)liberty | at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.consumeContent(DefaultClientExchangeHandlerImpl.java:158)liberty | at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:336)liberty | at org.apache.http.impl.nio.client.InternalRequestExecutor.inputReady(InternalRequestExecutor.java:83)liberty | at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265)liberty | at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:87)liberty | at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:40)liberty | at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:121)liberty | at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)liberty | at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)liberty | at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)liberty | at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)liberty | at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)liberty | at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)liberty | at java.lang.Thread.run(Thread.java:825)liberty | [8/14/25 15:17:49:283 EDT] 0000009d SystemOut O 15:17:49.283 [elasticsearch-rest-client-0-thread-3] DEBUG org.apache.http.impl.nio.conn.ManagedNHttpClientConnectionImpl - http-outgoing-1 172.24.0.2:45778<->10.55.56.4:9200[ACTIVE][r:r][ACTIVE][r][NOT_HANDSHAKING][EOF][0][0][0]: Shutdownliberty | [8/14/25 15:17:49:285 EDT] 0000009d SystemOut O 15:17:49.284 [elasticsearch-rest-client-0-thread-3] DEBUG org.elasticsearch.client.RestClient - request [POST https://esearchdev.noa.org:9200/_bulk] failedliberty | org.apache.http.MalformedChunkCodingException: CRLF expected at end of chunkliberty | at org.apache.http.impl.nio.codecs.ChunkDecoder.readChunkHead(ChunkDecoder.java:112)liberty | at org.apache.http.impl.nio.codecs.ChunkDecoder.read(ChunkDecoder.java:205)liberty | at org.apache.http.nio.util.SimpleInputBuffer.consumeContent(SimpleInputBuffer.java:66)liberty | at org.elasticsearch.client.HeapBufferedAsyncResponseConsumer.onContentReceived(HeapBufferedAsyncResponseConsumer.java:97)liberty | at org.apache.http.nio.protocol.AbstractAsyncResponseConsumer.consumeContent(AbstractAsyncResponseConsumer.java:147)liberty | at co.elastic.clients.transport.rest_client.SafeResponseConsumer.consumeContent(SafeResponseConsumer.java:78)liberty | at org.apache.http.impl.nio.client.MainClientExec.consumeContent(MainClientExec.java:329)liberty | at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.consumeContent(DefaultClientExchangeHandlerImpl.java:158)liberty | at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:336)liberty | at org.apache.http.impl.nio.client.InternalRequestExecutor.inputReady(InternalRequestExecutor.java:83)liberty | at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265)liberty | at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:87)liberty | at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:40)liberty | at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:121)liberty | at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162)liberty | at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337)liberty | at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315)liberty | at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276)liberty | at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)liberty | at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)liberty | at java.lang.Thread.run(Thread.java:825)