Failed to execute watch

I am facing the below errors in elastic search. Please note I am using docker-compose

**Elasticsearch logs**
cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sda1)]], net usable_space [8.6gb], net total_space [19.5gb], types [ext4]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:15:33,159Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "heap size [256mb], compressed ordinary object pointers [true]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:15:34,221Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "node name [482e9f3c4d6a], node ID [uTO0fZcASOWL1EyGJYBHow], cluster name [docker-cluster]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:01,372Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "[controller/167] [Main.cc@114] controller (64 bit): Version 7.9.3 (Build 6c27059cb8397a) Copyright (c) 2020 Elasticsearch BV" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:07,983Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "creating NettyAllocator with the following configs: [name=unpooled, factors={es.unsafe.use_unpooled_allocator=false, g1gc_enabled=true, g1gc_region_size=1mb, heap_size=256mb}]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:08,495Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "using discovery type [single-node] and seed hosts providers [settings]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:11,618Z", "level": "WARN", "component": "o.e.g.DanglingIndicesState", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:14,614Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "initialized" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:14,615Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "starting ..." }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:15,494Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "publish_address {172.20.0.2:9300}, bound_addresses {0.0.0.0:9300}" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:17,831Z", "level": "WARN", "component": "o.e.b.BootstrapChecks", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:17,847Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "cluster UUID [JJkI-5jXRAGNBgKswKO9Iw]" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:18,265Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "elected-as-master ([1] nodes joined)[{482e9f3c4d6a}{uTO0fZcASOWL1EyGJYBHow}{KtP1aoAeTK2-Iz2XARwLdA}{172.20.0.2}{172.20.0.2:9300}{dilmrt}{ml.machine_memory=4126208000, xpack.installed=true, transform.node=true, ml.max_open_jobs=20} elect leader, BECOME_MASTER_TASK, FINISH_ELECTION], term: 56, version: 1017, delta: master node changed {previous , current [{482e9f3c4d6a}{uTO0fZcASOWL1EyGJYBHow}{KtP1aoAeTK2-Iz2XARwLdA}{172.20.0.2}{172.20.0.2:9300}{dilmrt}{ml.machine_memory=4126208000, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}]}" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:19,034Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "master node changed {previous , current [{482e9f3c4d6a}{uTO0fZcASOWL1EyGJYBHow}{KtP1aoAeTK2-Iz2XARwLdA}{172.20.0.2}{172.20.0.2:9300}{dilmrt}{ml.machine_memory=4126208000, xpack.installed=true, transform.node=true, ml.max_open_jobs=20}]}, term: 56, version: 1017, reason: Publication{term=56, version=1017}" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:19,479Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "publish_address {172.20.0.2:9200}, bound_addresses {0.0.0.0:9200}", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:19,502Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "started", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:22,539Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "license [5e5eb245-4c2c-4f6d-a94c-067f986363ab] mode [trial] - valid", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:22,687Z", "level": "WARN", "component": "o.e.l.LicenseService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "License [will expire] on [Tuesday, December 15, 2020].\n# If you have a new license, please update it. Otherwise, please reach out to\n# your support contact.\n# \n# Commercial plugins operate with reduced functionality on license expiration:\n# - security\n# - Cluster health, cluster stats and indices stats operations are blocked\n# - All data operations (read and write) continue to work\n# - watcher\n# - PUT / GET watch APIs are disabled, DELETE watch API continues to work\n# - Watches execute and write to the history\n# - The actions of the watches don't execute\n# - monitoring\n# - The agent will stop collecting cluster and indices metrics\n# - The agent will stop automatically cleaning indices older than [xpack.monitoring.history.duration]\n# - graph\n# - Graph explore APIs are disabled\n# - ml\n# - Machine learning APIs are disabled\n# - logstash\n# - Logstash will continue to poll centrally-managed pipelines\n# - beats\n# - Beats will continue to poll centrally-managed configuration\n# - deprecation\n# - Deprecation APIs are disabled\n# - upgrade\n# - Upgrade API is disabled\n# - sql\n# - SQL support is disabled\n# - rollup\n# - Creating and Starting rollup jobs will no longer be allowed.\n# - Stopping/Deleting existing jobs, RollupCaps API and RollupSearch continue to function.\n# - transform\n# - Creating, starting, updating transforms will no longer be allowed.\n# - Stopping/Deleting existing transforms continue to function.\n# - analytics\n# - Aggregations provided by Analytics plugin are no longer usable.\n# - ccr\n# - Creating new follower indices will be blocked\n# - Configuring auto-follow patterns will be blocked\n# - Auto-follow patterns will no longer discover new leader indices\n# - The CCR monitoring endpoint will be blocked\n# - Existing follower indices will continue to replicate data", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:22,802Z", "level": "INFO", "component": "o.e.g.GatewayService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "recovered [22] indices into cluster_state", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:47,960Z", "level": "ERROR", "component": "o.e.x.w.i.s.ExecutableSearchInput", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute [search] input for watch [JJkI-5jXRAGNBgKswKO9Iw_elasticsearch_version_mismatch], reason [all shards failed]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:48,864Z", "level": "WARN", "component": "o.e.x.w.e.ExecutionService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute watch [JJkI-5jXRAGNBgKswKO9Iw_elasticsearch_version_mismatch]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:49,259Z", "level": "ERROR", "component": "o.e.x.w.i.s.ExecutableSearchInput", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute [search] input for watch [JJkI-5jXRAGNBgKswKO9Iw_elasticsearch_cluster_status], reason [all shards failed]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:49,265Z", "level": "WARN", "component": "o.e.x.w.e.ExecutionService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute watch [JJkI-5jXRAGNBgKswKO9Iw_elasticsearch_cluster_status]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:49,450Z", "level": "ERROR", "component": "o.e.x.w.i.s.ExecutableSearchInput", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute [search] input for watch [JJkI-5jXRAGNBgKswKO9Iw_xpack_license_expiration], reason [all shards failed]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:49,970Z", "level": "ERROR", "component": "o.e.x.w.i.s.ExecutableSearchInput", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute [search] input for watch [JJkI-5jXRAGNBgKswKO9Iw_logstash_version_mismatch], reason [all shards failed]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:50,133Z", "level": "WARN", "component": "o.e.x.w.e.ExecutionService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute watch [JJkI-5jXRAGNBgKswKO9Iw_logstash_version_mismatch]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:50,368Z", "level": "ERROR", "component": "o.e.x.w.i.s.ExecutableSearchInput", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute [search] input for watch [JJkI-5jXRAGNBgKswKO9Iw_elasticsearch_nodes], reason [all shards failed]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:16:50,417Z", "level": "WARN", "component": "o.e.x.w.e.ExecutionService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "failed to execute watch [JJkI-5jXRAGNBgKswKO9Iw_elasticsearch_nodes]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:17:05,630Z", "level": "INFO", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[.monitoring-es-7-2020.11.21][0]]]).", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }
elasticsearch_1 | {"type": "server", "timestamp": "2020-11-21T09:19:25,765Z", "level": "INFO", "component": "o.e.m.j.JvmGcMonitorService", "cluster.name": "docker-cluster", "node.name": "482e9f3c4d6a", "message": "[gc][190] overhead, spent [331ms] collecting in the last [1s]", "cluster.uuid": "JJkI-5jXRAGNBgKswKO9Iw", "node.id": "uTO0fZcASOWL1EyGJYBHow" }

Please format your code/logs/config using the </> button, or markdown style back ticks. It helps to make things easy to read which helps us help you :slight_smile:

What is the output from _cat/shards/.watches?v?

Now Elasticsearch is working without any errors.