Elasticsearch 6.5 does not work at all after upgrade from 6.4

Hi there,
I just upgraded my ES 6.4 to 6.5
Unfortunately nothing works at all. The cluster is green, but whatever query i run, i get

[parent] Data too large, data for [<http_request>] would be [12157808315/11.3gb], which is larger than the limit of [12025908428/11.1gb], usages [request=0/0b, fielddata=0/0b, in_flight_requests=0/0b, accounting=12157808315/11.3gb]

In the case this was just a query to create a index PUT test, but it fails with all queries a i tried. Even the most simple ones like get document count for example. And it is always the CircuitBreakingException

The ES is running in a docker container as the previous 6.4

/opt/jdk-11.0.1/bin/java -Xmx16g -Des.cgroups.hierarchy.override=/ -Des.path.home=/usr/share/elasticsearch -Des.path.conf=/usr/share/elasticsearch/config -Des.distribution.flavor=default -Des.distribution.type=tar -cp /usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch -Ediscovery.type=single-node

I don't see any exceptions during startup.
Nothing special in my config as well

node.name: es2-1
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
path.data: /data/elasticsearch
path.logs: /var/log/elasticsearch   
thread_pool.write.queue_size: 1000

Any ideas ?

Can you get the full output of the cluster stats API? Do you have monitoring installed so you can show heap usage prior to the upgrade?

I do not have any monitoring, as this is this is my migration server. I am reindexing from 2.4 to 6.x (6.4 until this morning)

The cluster is completely idle at the moment.
Here is the cluster stats

{
  "_nodes": {
    "total": 1,
    "successful": 1,
    "failed": 0
  },
  "cluster_name": "myES2",
  "cluster_uuid": "Fu-nazLFR4yZ1ovV9a1ShQ",
  "timestamp": 1542275073464,
  "status": "green",
  "indices": {
    "count": 28,
    "shards": {
      "total": 308,
      "primaries": 308,
      "replication": 0,
      "index": {
        "shards": {
          "min": 11,
          "max": 11,
          "avg": 11
        },
        "primaries": {
          "min": 11,
          "max": 11,
          "avg": 11
        },
        "replication": {
          "min": 0,
          "max": 0,
          "avg": 0
        }
      }
    },
    "docs": {
      "count": 4937586776,
      "deleted": 14760452
    },
    "store": {
      "size": "1.4tb",
      "size_in_bytes": 1575408095716
    },
    "fielddata": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "total_count": 0,
      "hit_count": 0,
      "miss_count": 0,
      "cache_size": 0,
      "cache_count": 0,
      "evictions": 0
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 7075,
      "memory": "11.3gb",
      "memory_in_bytes": 12157808315,
      "terms_memory": "10.9gb",
      "terms_memory_in_bytes": 11712020252,
      "stored_fields_memory": "284.1mb",
      "stored_fields_memory_in_bytes": 297969848,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "0b",
      "norms_memory_in_bytes": 0,
      "points_memory": "122.1mb",
      "points_memory_in_bytes": 128050115,
      "doc_values_memory": "18.8mb",
      "doc_values_memory_in_bytes": 19768100,
      "index_writer_memory": "0b",
      "index_writer_memory_in_bytes": 0,
      "version_map_memory": "0b",
      "version_map_memory_in_bytes": 0,
      "fixed_bit_set": "0b",
      "fixed_bit_set_memory_in_bytes": 0,
      "max_unsafe_auto_id_timestamp": -1,
      "file_sizes": {}
    }
  },
  "nodes": {
    "count": {
      "total": 1,
      "data": 1,
      "coordinating_only": 0,
      "master": 1,
      "ingest": 1
    },
    "versions": [
      "6.5.0"
    ],
    "os": {
      "available_processors": 48,
      "allocated_processors": 48,
      "names": [
        {
          "name": "Linux",
          "count": 1
        }
      ],
      "mem": {
        "total": "125.7gb",
        "total_in_bytes": 135056891904,
        "free": "2.6gb",
        "free_in_bytes": 2887745536,
        "used": "123gb",
        "used_in_bytes": 132169146368,
        "free_percent": 2,
        "used_percent": 98
      }
    },
    "process": {
      "cpu": {
        "percent": 0
      },
      "open_file_descriptors": {
        "min": 4391,
        "max": 4391,
        "avg": 4391
      }
    },
    "jvm": {
      "max_uptime": "36.2m",
      "max_uptime_in_millis": 2173346,
      "versions": [
        {
          "version": "11.0.1",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "11.0.1+13",
          "vm_vendor": "Oracle Corporation",
          "count": 1
        }
      ],
      "mem": {
        "heap_used": "14gb",
        "heap_used_in_bytes": 15105924688,
        "heap_max": "16gb",
        "heap_max_in_bytes": 17179869184
      },
      "threads": 34
    },
    "fs": {
      "total": "8.5tb",
      "total_in_bytes": 9423821557760,
      "free": "7.1tb",
      "free_in_bytes": 7835668680704,
      "available": "6.6tb",
      "available_in_bytes": 7360662077440
    },
    "plugins": [
      {
        "name": "ingest-user-agent",
        "version": "6.5.0",
        "elasticsearch_version": "6.5.0",
        "java_version": "1.8",
        "description": "Ingest processor that extracts information from a user agent",
        "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
        "extended_plugins": [],
        "has_native_controller": false
      },
      {
        "name": "ingest-geoip",
        "version": "6.5.0",
        "elasticsearch_version": "6.5.0",
        "java_version": "1.8",
        "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
        "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
        "extended_plugins": [],
        "has_native_controller": false
      }
    ],
    "network_types": {
      "transport_types": {
        "security4": 1
      },
      "http_types": {
        "security4": 1
      }
    }
  }
}

Increasing Xmx to 24g helps, but i wonder why 16g is not enough for an idle cluster doing nothing.

16g was enough for 6.4. I am not feeling good increasing the memory so high, considering the we shouldn't set Xmx to more than 32g

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.