Killing pending tasks

Elastic Stack Elasticsearch
1

Hi everyone,

We work on an Elasticsearch cluster which seems locked.

We found out that we have critical tasks we launched a few days ago :

curl -X GET 10.129.28.194:9200/_cluster/pending_tasks?pretty
{
  "tasks" : [
    {
      "insert_order" : 942304,
      "priority" : "URGENT",
      "source" : "elected-as-master ([1] nodes joined)",
      "executing" : true,
      "time_in_queue_millis" : 310,
      "time_in_queue" : "310ms"
    },
    {
      "insert_order" : 942306,
      "priority" : "URGENT",
      "source" : "shard-started StartedShardEntry{shardId [[sro-a601-index][0]], allocationId [qeDAgdScRUGz7oGMSMtkRA], primary term [1], message [master {elasticsearch7-mj-1-xjzc4}{sutreixDR8OPUQhM99FB_w}{tMoFRgqpSkqDeTSfDnzaTg}{10.129.28.194}{10.129.28.194:9300}{cdhilmrstw}{ml.machine_memory=1073741824, xpack.installed=true, transform.node=true, ml.max_open_jobs=20} marked shard as initializing, but shard state is [POST_RECOVERY], mark shard as started]}",
      "executing" : false,
      "time_in_queue_millis" : 9,
      "time_in_queue" : "9ms"
    },
    {
      "insert_order" : 942305,
      "priority" : "HIGH",
      "source" : "cluster_reroute(post-join reroute)",
      "executing" : false,
      "time_in_queue_millis" : 309,
      "time_in_queue" : "309ms"
    },
    {
      "insert_order" : 942307,
      "priority" : "NORMAL",
      "source" : "update snapshot after shards started [false] or node configuration changed [true]",
      "executing" : false,
      "time_in_queue_millis" : 8,
      "time_in_queue" : "8ms"
    }
  ]
}

Is there any action we can consider to kill those tasks ?

Thanks !

Lucien

2

No, you'd need to restart the cluster.

Why does it seem locked?
What is the output from the _cluster/stats?pretty&human API?

3

Hello,

Thnak you for your feedback.

Here is the output from the _cluster/stats?pretty&human API :

curl -X GET "10.129.28.194:9200/_cluster/stats?pretty&human"
{
  "_nodes" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "zLRNxXcFRZCzkD3lr-xf8A",
  "timestamp" : 1635407465761,
  "indices" : {
    "count" : 0,
    "shards" : { },
    "docs" : {
      "count" : 0,
      "deleted" : 0
    },
    "store" : {
      "size" : "0b",
      "size_in_bytes" : 0,
      "reserved" : "0b",
      "reserved_in_bytes" : 0
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "total_count" : 0,
      "hit_count" : 0,
      "miss_count" : 0,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 0,
      "memory" : "0b",
      "memory_in_bytes" : 0,
      "terms_memory" : "0b",
      "terms_memory_in_bytes" : 0,
      "stored_fields_memory" : "0b",
      "stored_fields_memory_in_bytes" : 0,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "0b",
      "norms_memory_in_bytes" : 0,
      "points_memory" : "0b",
      "points_memory_in_bytes" : 0,
      "doc_values_memory" : "0b",
      "doc_values_memory_in_bytes" : 0,
      "index_writer_memory" : "0b",
      "index_writer_memory_in_bytes" : 0,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "0b",
      "fixed_bit_set_memory_in_bytes" : 0,
      "max_unsafe_auto_id_timestamp" : -9223372036854775808,
      "file_sizes" : { }
    },
    "mappings" : {
      "field_types" : [ ]
    },
    "analysis" : {
      "char_filter_types" : [ ],
      "tokenizer_types" : [ ],
      "filter_types" : [ ],
      "analyzer_types" : [ ],
      "built_in_char_filters" : [ ],
      "built_in_tokenizers" : [ ],
      "built_in_filters" : [ ],
      "built_in_analyzers" : [ ]
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "coordinating_only" : 0,
      "data" : 1,
      "data_cold" : 1,
      "data_content" : 1,
      "data_hot" : 1,
      "data_warm" : 1,
      "ingest" : 1,
      "master" : 1,
      "ml" : 1,
      "remote_cluster_client" : 1,
      "transform" : 1,
      "voting_only" : 0
    },
    "versions" : [
      "7.10.0"
    ],
    "os" : {
      "available_processors" : 2,
      "allocated_processors" : 2,
      "names" : [
        {
          "name" : "Linux",
          "count" : 1
        }
      ],
      "pretty_names" : [
        {
          "pretty_name" : "Red Hat Enterprise Linux Server 7.7 (Maipo)",
          "count" : 1
        }
      ],
      "mem" : {
        "total" : "1gb",
        "total_in_bytes" : 1073741824,
        "free" : "127.3mb",
        "free_in_bytes" : 133574656,
        "used" : "896.6mb",
        "used_in_bytes" : 940167168,
        "free_percent" : 12,
        "used_percent" : 88
      }
    },
    "process" : {
      "cpu" : {
        "percent" : 1
      },
      "open_file_descriptors" : {
        "min" : 259,
        "max" : 259,
        "avg" : 259
      }
    },
    "jvm" : {
      "max_uptime" : "7.7d",
      "max_uptime_in_millis" : 671008167,
      "versions" : [
        {
          "version" : "15.0.1",
          "vm_name" : "OpenJDK 64-Bit Server VM",
          "vm_version" : "15.0.1+9",
          "vm_vendor" : "AdoptOpenJDK",
          "bundled_jdk" : true,
          "using_bundled_jdk" : true,
          "count" : 1
        }
      ],
      "mem" : {
        "heap_used" : "131.4mb",
        "heap_used_in_bytes" : 137840568,
        "heap_max" : "512mb",
        "heap_max_in_bytes" : 536870912
      },
      "threads" : 51
    },
    "fs" : {
      "total" : "1.9gb",
      "total_in_bytes" : 2136997888,
      "free" : "1.9gb",
      "free_in_bytes" : 2045497344,
      "available" : "1.9gb",
      "available_in_bytes" : 2045497344
    },
    "plugins" : [ ],
    "network_types" : {
      "transport_types" : {
        "security4" : 1
      },
      "http_types" : {
        "security4" : 1
      }
    },
    "discovery_types" : {
      "single-node" : 1
    },
    "packaging_types" : [
      {
        "flavor" : "default",
        "type" : "rpm",
        "count" : 1
      }
    ],
    "ingest" : {
      "number_of_pipelines" : 0,
      "processor_stats" : { }
    }
  }
}
4

That cluster has no data in it?

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.