We’ve seen this before, and has cleared up after the master nodes (self?) restarted ES:
$ curl http://es.newsscribe.angwin:9209/
{
"name" : "es02.newsscribe.angwin",
"cluster_name" : "mc_es_cluster",
"cluster_uuid" : "t-tHTIhgT-iGQnt9_I9IPw",
"version" : {
"number" : "8.17.3",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "a091390de485bd4b127884f7e565c0cad59b10d2",
"build_date" : "2025-02-28T10:07:26.089129809Z",
"build_snapshot" : false,
"lucene_version" : "9.12.0",
"minimum_wire_compatibility_version" : "7.17.0",
"minimum_index_compatibility_version" : "7.0.0"
},
"tagline" : "You Know, for Search"
}
Pending tasks:
$ curl -s 'http://es.newsscribe.angwin:9209/_cluster/pending_tasks' | jq .
{
"tasks": [
{
"insert_order": 74242,
"priority": "URGENT",
"source": "update_tsdb_data_stream_end_times",
"executing": false,
"time_in_queue_millis": 1365296472,
"time_in_queue": "15.8d"
},
{
"insert_order": 74244,
"priority": "URGENT",
"source": "delete-index [[.ds-ilm-history-7-2025.08.10-000017/B_at6sRGSv2uRGk9PmiDmQ]]",
"executing": false,
"time_in_queue_millis": 1267496649,
"time_in_queue": "14.6d"
},
{
"insert_order": 74254,
"priority": "URGENT",
"source": "delete-index [[.ds-ilm-history-7-2025.08.21-000019/R1s6hzBNTxWgXReAJLSbCg]]",
"executing": false,
"time_in_queue_millis": 564296645,
"time_in_queue": "6.5d"
},
{
"insert_order": 74258,
"priority": "URGENT",
"source": "delete-index [[.ds-.slm-history-7-2025.08.15-000009/b7ogdRy_TUmZwZ-4ZrPK0w]]",
"executing": false,
"time_in_queue_millis": 330596798,
"time_in_queue": "3.8d"
},
{
"insert_order": 74240,
"priority": "NORMAL",
"source": "ilm-set-step-info {policy [.deprecation-indexing-ilm-policy], index [.ds-.logs-deprecation.elasticsearch-default-2025.07.22-000007], currentStep [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"check-rollover-ready\"}]}",
"executing": false,
"time_in_queue_millis": 1365416719,
"time_in_queue": "15.8d"
},
{
"insert_order": 74241,
"priority": "NORMAL",
"source": "ilm-set-step-info {policy [mediacloud-lifecycle-policy], index [mc_search-000011], currentStep [{\"phase\":\"hot\",\"action\":\"rollover\",\"name\":\"check-rollover-ready\"}]}",
"executing": false,
"time_in_queue_millis": 1365416719,
"time_in_queue": "15.8d"
},
{
"insert_order": 74243,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 1286885065,
"time_in_queue": "14.8d"
},
{
"insert_order": 74245,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 1200485214,
"time_in_queue": "13.8d"
},
{
"insert_order": 74246,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 1114085062,
"time_in_queue": "12.8d"
},
{
"insert_order": 74247,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 1027685036,
"time_in_queue": "11.8d"
},
{
"insert_order": 74248,
"priority": "NORMAL",
"source": "rollover_index source [.ds-.slm-history-7-2025.11.15-000024] to target [.ds-.slm-history-7-2025.11.22-000025]",
"executing": false,
"time_in_queue_millis": 1025996618,
"time_in_queue": "11.8d"
},
{
"insert_order": 74249,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 941285053,
"time_in_queue": "10.8d"
},
{
"insert_order": 74250,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 854885074,
"time_in_queue": "9.8d"
},
{
"insert_order": 74251,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 768485120,
"time_in_queue": "8.8d"
},
{
"insert_order": 74252,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 682085051,
"time_in_queue": "7.8d"
},
{
"insert_order": 74253,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 595685215,
"time_in_queue": "6.8d"
},
{
"insert_order": 74255,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 509285139,
"time_in_queue": "5.8d"
},
{
"insert_order": 74256,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 422885071,
"time_in_queue": "4.8d"
},
{
"insert_order": 74257,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 336485086,
"time_in_queue": "3.8d"
},
{
"insert_order": 74259,
"priority": "NORMAL",
"source": "create_snapshot [snapshot-2025.12.01-6okdeqjbslwmcutnijomva]",
"executing": false,
"time_in_queue_millis": 250085114,
"time_in_queue": "2.8d"
},
{
"insert_order": 74260,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 250085114,
"time_in_queue": "2.8d"
},
{
"insert_order": 74261,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 163685150,
"time_in_queue": "1.8d"
},
{
"insert_order": 74262,
"priority": "NORMAL",
"source": "update_slm_stats",
"executing": false,
"time_in_queue_millis": 77285191,
"time_in_queue": "21.4h"
}
]
}
Detailed task info:
$ curl -s 'http://es.newsscribe.angwin:9209/_tasks?detailed' | jq .
{
"nodes": {
"7xEGzqWGStqIsK3ex3OjWQ": {
"name": "es04.newsscribe.angwin",
"transport_address": "10.1.1.24:9309",
"host": "10.1.1.24",
"ip": "10.1.1.24:9309",
"roles": [
"data"
],
"attributes": {
"transform.config_version": "10.0.0",
"ml.config_version": "12.0.0",
"xpack.installed": "true"
},
"tasks": {
"7xEGzqWGStqIsK3ex3OjWQ:534512250": {
"node": "7xEGzqWGStqIsK3ex3OjWQ",
"id": 534512250,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799331,
"running_time_in_nanos": 118273,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"ODP_A5m0SGaEs50OAyBtfQ": {
"name": "es02.newsscribe.angwin",
"transport_address": "10.1.1.22:9309",
"host": "10.1.1.22",
"ip": "10.1.1.22:9309",
"roles": [
"data",
"master"
],
"attributes": {
"transform.config_version": "10.0.0",
"ml.config_version": "12.0.0",
"xpack.installed": "true"
},
"tasks": {
"ODP_A5m0SGaEs50OAyBtfQ:544940073": {
"node": "ODP_A5m0SGaEs50OAyBtfQ",
"id": 544940073,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799308,
"running_time_in_nanos": 154393,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"HX_qaMLLSq2feX4DP0vXdA": {
"name": "es06.newsscribe.angwin",
"transport_address": "10.1.1.26:9309",
"host": "10.1.1.26",
"ip": "10.1.1.26:9309",
"roles": [
"data"
],
"attributes": {
"xpack.installed": "true",
"ml.config_version": "12.0.0",
"transform.config_version": "10.0.0"
},
"tasks": {
"HX_qaMLLSq2feX4DP0vXdA:553736807": {
"node": "HX_qaMLLSq2feX4DP0vXdA",
"id": 553736807,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799308,
"running_time_in_nanos": 117751,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"KnQw9Gu8Q9e07aQzULALZw": {
"name": "es07.newsscribe.angwin",
"transport_address": "10.1.1.27:9309",
"host": "10.1.1.27",
"ip": "10.1.1.27:9309",
"roles": [
"data"
],
"attributes": {
"transform.config_version": "10.0.0",
"ml.config_version": "12.0.0",
"xpack.installed": "true"
},
"tasks": {
"KnQw9Gu8Q9e07aQzULALZw:176": {
"node": "KnQw9Gu8Q9e07aQzULALZw",
"id": 176,
"type": "persistent",
"action": "health-node[c]",
"status": {
"state": "STARTED"
},
"description": "id=health-node",
"start_time_in_millis": 1761452589874,
"running_time_in_nanos": 3350209430942227,
"cancellable": true,
"cancelled": false,
"parent_task_id": "cluster:52",
"headers": {}
},
"KnQw9Gu8Q9e07aQzULALZw:573612856": {
"node": "KnQw9Gu8Q9e07aQzULALZw",
"id": 573612856,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799305,
"running_time_in_nanos": 145009,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"aYc_TLeFS7W_j7D4msLBBQ": {
"name": "es01.newsscribe.angwin",
"transport_address": "10.1.1.21:9309",
"host": "10.1.1.21",
"ip": "10.1.1.21:9309",
"roles": [
"data",
"master"
],
"attributes": {
"transform.config_version": "10.0.0",
"ml.config_version": "12.0.0",
"xpack.installed": "true"
},
"tasks": {
"aYc_TLeFS7W_j7D4msLBBQ:551453856": {
"node": "aYc_TLeFS7W_j7D4msLBBQ",
"id": 551453856,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799307,
"running_time_in_nanos": 140731,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"sPtOraRBRCGNCNBym3bVfA": {
"name": "es08.newsscribe.angwin",
"transport_address": "10.1.1.28:9309",
"host": "10.1.1.28",
"ip": "10.1.1.28:9309",
"roles": [
"data"
],
"attributes": {
"transform.config_version": "10.0.0",
"ml.config_version": "12.0.0",
"xpack.installed": "true"
},
"tasks": {
"sPtOraRBRCGNCNBym3bVfA:184": {
"node": "sPtOraRBRCGNCNBym3bVfA",
"id": 184,
"type": "persistent",
"action": "geoip-downloader[c]",
"status": {
"successful_downloads": 0,
"failed_downloads": 0,
"total_download_time": 0,
"databases_count": 0,
"skipped_updates": 0,
"expired_databases": 0
},
"description": "id=geoip-downloader",
"start_time_in_millis": 1761452589875,
"running_time_in_nanos": 3350209433981178,
"cancellable": true,
"cancelled": false,
"parent_task_id": "cluster:53",
"headers": {}
},
"sPtOraRBRCGNCNBym3bVfA:549691533": {
"node": "sPtOraRBRCGNCNBym3bVfA",
"id": 549691533,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799309,
"running_time_in_nanos": 263403,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"oq8arnWmQtqG9pDHnZhnAw": {
"name": "es03.newsscribe.angwin",
"transport_address": "10.1.1.23:9309",
"host": "10.1.1.23",
"ip": "10.1.1.23:9309",
"roles": [
"data",
"master"
],
"attributes": {
"transform.config_version": "10.0.0",
"ml.config_version": "12.0.0",
"xpack.installed": "true"
},
"tasks": {
"oq8arnWmQtqG9pDHnZhnAw:547863591": {
"node": "oq8arnWmQtqG9pDHnZhnAw",
"id": 547863591,
"type": "transport",
"action": "indices:admin/delete",
"description": "",
"start_time_in_millis": 1764472088421,
"running_time_in_nanos": 330710886396794,
"cancellable": false,
"headers": {}
},
"oq8arnWmQtqG9pDHnZhnAw:494893948": {
"node": "oq8arnWmQtqG9pDHnZhnAw",
"id": 494893948,
"type": "transport",
"action": "indices:admin/delete",
"description": "",
"start_time_in_millis": 1764238388422,
"running_time_in_nanos": 564410885435084,
"cancellable": false,
"headers": {}
},
"oq8arnWmQtqG9pDHnZhnAw:403969823": {
"node": "oq8arnWmQtqG9pDHnZhnAw",
"id": 403969823,
"type": "transport",
"action": "indices:admin/rollover",
"description": "",
"start_time_in_millis": 1763776688421,
"running_time_in_nanos": 1026110886528048,
"cancellable": true,
"cancelled": false,
"headers": {}
},
"oq8arnWmQtqG9pDHnZhnAw:563438641": {
"node": "oq8arnWmQtqG9pDHnZhnAw",
"id": 563438641,
"type": "transport",
"action": "cluster:admin/snapshot/create",
"description": "snapshot [mediacloud-elasticsearch-newscribe-snapshots:snapshot-2025.12.01-6okdeqjbslwmcutnijomva]",
"start_time_in_millis": 1764552600008,
"running_time_in_nanos": 250199299359869,
"cancellable": false,
"headers": {}
},
"oq8arnWmQtqG9pDHnZhnAw:378655036": {
"node": "oq8arnWmQtqG9pDHnZhnAw",
"id": 378655036,
"type": "transport",
"action": "indices:admin/delete",
"description": "",
"start_time_in_millis": 1763535188422,
"running_time_in_nanos": 1267610885509213,
"cancellable": false,
"headers": {}
},
"oq8arnWmQtqG9pDHnZhnAw:588214120": {
"node": "oq8arnWmQtqG9pDHnZhnAw",
"id": 588214120,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799308,
"running_time_in_nanos": 177422,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
},
"RZmA-8lOR7mCA2cxdb7xGA": {
"name": "es05.newsscribe.angwin",
"transport_address": "10.1.1.25:9309",
"host": "10.1.1.25",
"ip": "10.1.1.25:9309",
"roles": [
"data"
],
"attributes": {
"xpack.installed": "true",
"ml.config_version": "12.0.0",
"transform.config_version": "10.0.0"
},
"tasks": {
"RZmA-8lOR7mCA2cxdb7xGA:535659129": {
"node": "RZmA-8lOR7mCA2cxdb7xGA",
"id": 535659129,
"type": "transport",
"action": "cluster:monitor/tasks/lists",
"description": "",
"start_time_in_millis": 1764802799308,
"running_time_in_nanos": 470172,
"cancellable": true,
"cancelled": false,
"headers": {}
},
"RZmA-8lOR7mCA2cxdb7xGA:535659130": {
"node": "RZmA-8lOR7mCA2cxdb7xGA",
"id": 535659130,
"type": "transport",
"action": "cluster:monitor/tasks/lists[n]",
"description": "",
"start_time_in_millis": 1764802799308,
"running_time_in_nanos": 124577,
"cancellable": true,
"cancelled": false,
"parent_task_id": "RZmA-8lOR7mCA2cxdb7xGA:535659129",
"headers": {}
}
}
}
}
}