Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.19.0.3:9200

Hello guys,

I am trying to setup an ELK stack with docker compose and TLS, and I get this message from kibana's logs.

Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.19.0.3:9200

Anything wrong with my yml?

version: '2.2'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es01
    restart: unless-stopped
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key  
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5    
    networks:
      - elastic
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es02
    restart: unless-stopped
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es03
    restart: unless-stopped
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass 
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es03/es03.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es03/es03.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data03:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic
  
  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.1
    restart: unless-stopped
    environment:
      - ELASTICSEARCH_HOSTS=["https://es01:9200","https://es02:9200","https://es03:9200"]
      - XPACK_SECURITY_ENABLED=true
      - ELASTICSEARCH_USERNAME=elastic
      - ELASTICSEARCH_PASSWORD=pass
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=$CERTS_DIR/ca/ca.crt
      - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=d3e5bd37d75a348b5e0736cbccf515b6  
    ports:
      - 5601:5601
    volumes: ['certs:$CERTS_DIR']    
    networks:
      - elastic  
    depends_on:
      - es01
      - es02
      - es03  

  wait_until_ready:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: /usr/bin/true
    depends_on: {"es01": {"condition": "service_healthy"}}

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local
  certs:
    driver: local      

networks:
  elastic:
    driver: bridge

PS. It used to work fine, but we wanted to use a smaller instance, so I moved it

Please share your Elasticsearch and Kibana logs.

Kibana logs

{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["info","plugins-service"],"pid":1219,"message":"Plugin \"metricsEntities\" is disabled."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["info","http","server","Preboot"],"pid":1219,"message":"http server running at http://0.0.0.0:5601"}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Support for setting server.host to \"0\" in kibana.yml is deprecated and will be removed in Kibana version 8.0.0. Instead use \"0.0.0.0\" to bind to all interfaces."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Setting [elasticsearch.username] to \"elastic\" is deprecated. You should use the \"kibana_system\" user instead."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"plugins.scanDirs is deprecated and is no longer used"}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Config key [monitoring.cluster_alerts.email_notifications.email_address] will be required for email notifications to work in 8.0.\""}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Setting [monitoring.username] to \"elastic\" is deprecated. You should use the \"kibana_system\" user instead."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"\"xpack.reporting.roles\" is deprecated. Granting reporting privilege through a \"reporting_user\" role will not be supported starting in 8.0. Please set \"xpack.reporting.roles.enabled\" to \"false\" and grant reporting privileges to users using Kibana application privileges **Management > Security > Roles**."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Session idle timeout (\"xpack.security.session.idleTimeout\") will be set to 1 hour by default in the next major version (8.0)."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Session lifespan (\"xpack.security.session.lifespan\") will be set to 30 days by default in the next major version (8.0)."}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["info","plugins-system","standard"],"pid":1219,"message":"Setting up [113] plugins: [translations,licensing,globalSearch,globalSearchProviders,banners,licenseApiGuard,code,usageCollection,xpackLegacy,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,securityOss,share,screenshotMode,telemetry,newsfeed,mapsEms,mapsLegacy,legacyExport,kibanaLegacy,embeddable,uiActionsEnhanced,fieldFormats,expressions,charts,esUiShared,bfetch,data,savedObjects,visualizations,visTypeXy,visTypeVislib,visTypeTimelion,features,visTypeTagcloud,visTypeTable,visTypePie,visTypeMetric,visTypeMarkdown,tileMap,regionMap,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,timelion,indexPatternFieldEditor,home,searchprofiler,painlessLab,grokdebugger,graph,visTypeVega,management,watcher,licenseManagement,indexPatternManagement,advancedSettings,discover,discoverEnhanced,dashboard,dashboardEnhanced,visualize,visTypeTimeseries,savedObjectsManagement,spaces,security,transform,savedObjectsTagging,lens,reporting,canvas,lists,ingestPipelines,fileUpload,maps,dataVisualizer,encryptedSavedObjects,dataEnhanced,dashboardMode,cloud,snapshotRestore,fleet,indexManagement,rollup,remoteClusters,crossClusterReplication,indexLifecycleManagement,eventLog,actions,alerting,triggersActionsUi,stackAlerts,ruleRegistry,osquery,ml,cases,timelines,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,console,apmOss,apm]"}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["info","plugins","taskManager"],"pid":1219,"message":"TaskManager is identified by the Kibana UUID: 80663143-fed4-4e76-a9c1-a75e998a7b92"}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["warning","plugins","security","config"],"pid":1219,"message":"Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["warning","plugins","security","config"],"pid":1219,"message":"Session cookies will be transmitted over insecure connections. This is not recommended."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["warning","plugins","reporting","config"],"pid":1219,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.reporting.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["warning","plugins","reporting","config"],"pid":1219,"message":"Chromium sandbox provides an additional layer of protection, but is not supported for Linux CentOS 8.4.2105\n OS. Automatically setting 'xpack.reporting.capture.browser.chromium.disableSandbox: true'."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing common resources shared between all indices"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.uptime.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.logs.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.metrics.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.apm.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","savedobjects-service"],"pid":1219,"message":"Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations..."}
{"type":"log","@timestamp":"2022-01-13T08:45:42+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:44+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:47+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:49+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:52+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:54+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:56+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:59+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:01+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:04+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:06+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:09+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:14+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:17+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:21+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:26+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:31+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:36+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}

Elasticsearch logs part 1

Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:45:39,860Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[9], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:45:39,876Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:39,877Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-12896459498932564283, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,340Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,347Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,364Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,368Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,370Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,370Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,371Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,372Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,372Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,373Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,374Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,374Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,379Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,386Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,387Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,391Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,391Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,393Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,393Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,394Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,395Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,396Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,401Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,403Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,404Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,415Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,416Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,625Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdf)]], net usable_space [529.9gb], net total_space [589.5gb], types [ext4]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,625Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,936Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "uncaught exception in thread [main]", 
"stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:171) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.main(Command.java:79) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81) ~[elasticsearch-7.15.1.jar:7.15.1]",
"Caused by: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:383) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:288) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",

Elasticsearch logs part 2

"at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167) ~[elasticsearch-7.15.1.jar:7.15.1]",
"... 6 more"] }
uncaught exception in thread [main]
java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]
	at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83)
	at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423)
	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320)
	at org.elasticsearch.node.Node.<init>(Node.java:383)
	at org.elasticsearch.node.Node.<init>(Node.java:288)
	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399)
	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167)
	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158)
	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75)
	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114)
	at org.elasticsearch.cli.Command.main(Command.java:79)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:46:10,747Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[6], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:46:10,815Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:46:10,815Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-17850755216468522263, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,791Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,791Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,792Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,792Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,793Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,798Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,910Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,924Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,925Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,925Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,926Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,926Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,927Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,927Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,928Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,937Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,937Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,938Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,939Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,939Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,940Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,940Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,952Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,965Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,965Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,966Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,967Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,972Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": 
"o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,472Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdf)]], net usable_space [529.9gb], net total_space [589.5gb], types [ext4]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,474Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
uncaught exception in thread [main]
java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]
	at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83)
	at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423)
	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320)
	at org.elasticsearch.node.Node.<init>(Node.java:383)
	at org.elasticsearch.node.Node.<init>(Node.java:288)
	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399)
	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167)
	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158)
	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75)
	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114)
	at org.elasticsearch.cli.Command.main(Command.java:79)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
{"type": "server", "timestamp": "2022-01-13T08:46:45,713Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "uncaught exception in thread [main]", 
"stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:171) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.main(Command.java:79) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81) ~[elasticsearch-7.15.1.jar:7.15.1]",
"Caused by: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:383) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:288) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167) ~[elasticsearch-7.15.1.jar:7.15.1]",
"... 6 more"] }
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:48:01,482Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[7], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:48:01,486Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:48:01,487Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-14287106734366476380, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }

That's your problem. You need to be running 7.16.2 or higher.

Thanks manage to fix it with your help!

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.