Elasticsearch 6.5 does not work at all after upgrade from 6.4


(Georgi Ivanov) #1

Hi there,
I just upgraded my ES 6.4 to 6.5
Unfortunately nothing works at all. The cluster is green, but whatever query i run, i get

[parent] Data too large, data for [<http_request>] would be [12157808315/11.3gb], which is larger than the limit of [12025908428/11.1gb], usages [request=0/0b, fielddata=0/0b, in_flight_requests=0/0b, accounting=12157808315/11.3gb]

In the case this was just a query to create a index PUT test, but it fails with all queries a i tried. Even the most simple ones like get document count for example. And it is always the CircuitBreakingException

The ES is running in a docker container as the previous 6.4

/opt/jdk-11.0.1/bin/java -Xmx16g -Des.cgroups.hierarchy.override=/ -Des.path.home=/usr/share/elasticsearch -Des.path.conf=/usr/share/elasticsearch/config -Des.distribution.flavor=default -Des.distribution.type=tar -cp /usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch -Ediscovery.type=single-node

I don't see any exceptions during startup.
Nothing special in my config as well

node.name: es2-1
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
path.data: /data/elasticsearch
path.logs: /var/log/elasticsearch   
thread_pool.write.queue_size: 1000

Any ideas ?


(Christian Dahlqvist) #2

Can you get the full output of the cluster stats API? Do you have monitoring installed so you can show heap usage prior to the upgrade?


(Georgi Ivanov) #3

I do not have any monitoring, as this is this is my migration server. I am reindexing from 2.4 to 6.x (6.4 until this morning)

The cluster is completely idle at the moment.
Here is the cluster stats

{
  "_nodes": {
    "total": 1,
    "successful": 1,
    "failed": 0
  },
  "cluster_name": "myES2",
  "cluster_uuid": "Fu-nazLFR4yZ1ovV9a1ShQ",
  "timestamp": 1542275073464,
  "status": "green",
  "indices": {
    "count": 28,
    "shards": {
      "total": 308,
      "primaries": 308,
      "replication": 0,
      "index": {
        "shards": {
          "min": 11,
          "max": 11,
          "avg": 11
        },
        "primaries": {
          "min": 11,
          "max": 11,
          "avg": 11
        },
        "replication": {
          "min": 0,
          "max": 0,
          "avg": 0
        }
      }
    },
    "docs": {
      "count": 4937586776,
      "deleted": 14760452
    },
    "store": {
      "size": "1.4tb",
      "size_in_bytes": 1575408095716
    },
    "fielddata": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "0b",
      "memory_size_in_bytes": 0,
      "total_count": 0,
      "hit_count": 0,
      "miss_count": 0,
      "cache_size": 0,
      "cache_count": 0,
      "evictions": 0
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 7075,
      "memory": "11.3gb",
      "memory_in_bytes": 12157808315,
      "terms_memory": "10.9gb",
      "terms_memory_in_bytes": 11712020252,
      "stored_fields_memory": "284.1mb",
      "stored_fields_memory_in_bytes": 297969848,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "0b",
      "norms_memory_in_bytes": 0,
      "points_memory": "122.1mb",
      "points_memory_in_bytes": 128050115,
      "doc_values_memory": "18.8mb",
      "doc_values_memory_in_bytes": 19768100,
      "index_writer_memory": "0b",
      "index_writer_memory_in_bytes": 0,
      "version_map_memory": "0b",
      "version_map_memory_in_bytes": 0,
      "fixed_bit_set": "0b",
      "fixed_bit_set_memory_in_bytes": 0,
      "max_unsafe_auto_id_timestamp": -1,
      "file_sizes": {}
    }
  },
  "nodes": {
    "count": {
      "total": 1,
      "data": 1,
      "coordinating_only": 0,
      "master": 1,
      "ingest": 1
    },
    "versions": [
      "6.5.0"
    ],
    "os": {
      "available_processors": 48,
      "allocated_processors": 48,
      "names": [
        {
          "name": "Linux",
          "count": 1
        }
      ],
      "mem": {
        "total": "125.7gb",
        "total_in_bytes": 135056891904,
        "free": "2.6gb",
        "free_in_bytes": 2887745536,
        "used": "123gb",
        "used_in_bytes": 132169146368,
        "free_percent": 2,
        "used_percent": 98
      }
    },
    "process": {
      "cpu": {
        "percent": 0
      },
      "open_file_descriptors": {
        "min": 4391,
        "max": 4391,
        "avg": 4391
      }
    },
    "jvm": {
      "max_uptime": "36.2m",
      "max_uptime_in_millis": 2173346,
      "versions": [
        {
          "version": "11.0.1",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "vm_version": "11.0.1+13",
          "vm_vendor": "Oracle Corporation",
          "count": 1
        }
      ],
      "mem": {
        "heap_used": "14gb",
        "heap_used_in_bytes": 15105924688,
        "heap_max": "16gb",
        "heap_max_in_bytes": 17179869184
      },
      "threads": 34
    },
    "fs": {
      "total": "8.5tb",
      "total_in_bytes": 9423821557760,
      "free": "7.1tb",
      "free_in_bytes": 7835668680704,
      "available": "6.6tb",
      "available_in_bytes": 7360662077440
    },
    "plugins": [
      {
        "name": "ingest-user-agent",
        "version": "6.5.0",
        "elasticsearch_version": "6.5.0",
        "java_version": "1.8",
        "description": "Ingest processor that extracts information from a user agent",
        "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
        "extended_plugins": [],
        "has_native_controller": false
      },
      {
        "name": "ingest-geoip",
        "version": "6.5.0",
        "elasticsearch_version": "6.5.0",
        "java_version": "1.8",
        "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
        "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
        "extended_plugins": [],
        "has_native_controller": false
      }
    ],
    "network_types": {
      "transport_types": {
        "security4": 1
      },
      "http_types": {
        "security4": 1
      }
    }
  }
}

(Georgi Ivanov) #4

Increasing Xmx to 24g helps, but i wonder why 16g is not enough for an idle cluster doing nothing.

16g was enough for 6.4. I am not feeling good increasing the memory so high, considering the we shouldn't set Xmx to more than 32g


(system) #5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.