[BulkExport ES5.5.0]When one node down, the index rate of the cluster almost drop to 0

Elastic Stack Elasticsearch
1

When one node down for some reasons from ES5.5.0 cluster, Then the index rate of the cluster almost drop to 0. After 10 minutes, the index rate resumed. And this situation happened every time when node down, any one knows why ?

In the whole process, the shards ever on the downed node move to the other nodes all the time.

2

Could you show us the full output of the cluster stats API?

3

The result of GET /_cluster/stats?human&pretty as follows:

{
  "_nodes": {
    "total": 65,
    "successful": 65,
    "failed": 0
  },
  "cluster_name": "ps-st",
  "timestamp": 1534856637597,
  "status": "green",
  "indices": {
    "count": 134,
    "shards": {
      "total": 692,
      "primaries": 257,
      "replication": 1.6926070038910506,
      "index": {
        "shards": {
          "min": 2,
          "max": 360,
          "avg": 5.164179104477612
        },
        "primaries": {
          "min": 1,
          "max": 120,
          "avg": 1.9179104477611941
        },
        "replication": {
          "min": 1,
          "max": 59,
          "avg": 1.4402985074626866
        }
      }
    },
    "docs": {
      "count": 46684623468,
      "deleted": 14342948115
    },
    "store": {
      "size": "42.8tb",
      "size_in_bytes": 47101092713976,
      "throttle_time": "0s",
      "throttle_time_in_millis": 0
    },
    "fielddata": {
      "memory_size": "131.8kb",
      "memory_size_in_bytes": 135000,
      "evictions": 0
    },
    "query_cache": {
      "memory_size": "27.9gb",
      "memory_size_in_bytes": 30019415731,
      "total_count": 561904130720,
      "hit_count": 117705825276,
      "miss_count": 444198305444,
      "cache_size": 24493924,
      "cache_count": 58224175013,
      "evictions": 58199681089
    },
    "completion": {
      "size": "0b",
      "size_in_bytes": 0
    },
    "segments": {
      "count": 20810,
      "memory": "133.2gb",
      "memory_in_bytes": 143043390891,
      "terms_memory": "103.3gb",
      "terms_memory_in_bytes": 110992524605,
      "stored_fields_memory": "19.6gb",
      "stored_fields_memory_in_bytes": 21120619816,
      "term_vectors_memory": "0b",
      "term_vectors_memory_in_bytes": 0,
      "norms_memory": "979.6kb",
      "norms_memory_in_bytes": 1003136,
      "points_memory": "10.1gb",
      "points_memory_in_bytes": 10891779878,
      "doc_values_memory": "35.7mb",
      "doc_values_memory_in_bytes": 37463456,
      "index_writer_memory": "1.7gb",
      "index_writer_memory_in_bytes": 1894044993,
      "version_map_memory": "199.1mb",
      "version_map_memory_in_bytes": 208846953,
      "fixed_bit_set": "203.7kb",
      "fixed_bit_set_memory_in_bytes": 208608,
      "max_unsafe_auto_id_timestamp": 1534836479197,
      "file_sizes": {}
    }
  },
  "nodes": {
    "count": {
      "total": 65,
      "data": 60,
      "coordinating_only": 0,
      "master": 5,
      "ingest": 65
    },
    "versions": [
      "5.5.0"
    ],
    "os": {
      "available_processors": 2000,
      "allocated_processors": 2000,
      "names": [
        {
          "name": "Linux",
          "count": 65
        }
      ],
      "mem": {
        "total": "3.8tb",
        "total_in_bytes": 4210922209280,
        "free": "48gb",
        "free_in_bytes": 51643318272,
        "used": "3.7tb",
        "used_in_bytes": 4159278891008,
        "free_percent": 1,
        "used_percent": 99
      }
    },
    "process": {
      "cpu": {
        "percent": 312
      },
      "open_file_descriptors": {
        "min": 1895,
        "max": 2648,
        "avg": 2565
      }
    },
    "jvm": {
      "max_uptime": "109.9d",
      "max_uptime_in_millis": 9500560616,
      "versions": [
        {
          "version": "1.8.0_102",
          "vm_name": "Java HotSpot(TM) 64-Bit Server VM",
          "vm_version": "25.102-b14",
          "vm_vendor": "Oracle Corporation",
          "count": 65
        }
      ],
      "mem": {
        "heap_used": "524.7gb",
        "heap_used_in_bytes": 563403078696,
        "heap_max": "1.8tb",
        "heap_max_in_bytes": 2018634629120
      },
      "threads": 18532
    },
    "fs": {
      "total": "178.1tb",
      "total_in_bytes": 195862200320000,
      "free": "134.9tb",
      "free_in_bytes": 148380249014272,
      "available": "134.9tb",
      "available_in_bytes": 148380249128960,
      "spins": "true"
    },
    "plugins": [
      {
        "name": "x-pack",
        "version": "5.5.0",
        "description": "Elasticsearch Expanded Pack Plugin",
        "classname": "org.elasticsearch.xpack.XPackPlugin",
        "has_native_controller": true
      },
      {
        "name": "analysis-ik",
        "version": "5.5.0",
        "description": "IK Analyzer for Elasticsearch",
        "classname": "org.elasticsearch.plugin.analysis.ik.AnalysisIkPlugin",
        "has_native_controller": false
      }
    ],
    "network_types": {
      "transport_types": {
        "security4": 65
      },
      "http_types": {
        "security4": 65
      }
    }
  }
}
4

Anything relevant in the Elasticsearch logs?

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.