Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.19.0.3:9200

Spyros_Agriopoulos · January 12, 2022, 3:45pm

Hello guys,

I am trying to setup an ELK stack with docker compose and TLS, and I get this message from kibana's logs.

Anything wrong with my yml?

version: '2.2'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es01
    restart: unless-stopped
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key  
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5    
    networks:
      - elastic
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es02
    restart: unless-stopped
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es03
    restart: unless-stopped
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass 
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es03/es03.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es03/es03.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data03:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic
  
  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.1
    restart: unless-stopped
    environment:
      - ELASTICSEARCH_HOSTS=["https://es01:9200","https://es02:9200","https://es03:9200"]
      - XPACK_SECURITY_ENABLED=true
      - ELASTICSEARCH_USERNAME=elastic
      - ELASTICSEARCH_PASSWORD=pass
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=$CERTS_DIR/ca/ca.crt
      - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=d3e5bd37d75a348b5e0736cbccf515b6  
    ports:
      - 5601:5601
    volumes: ['certs:$CERTS_DIR']    
    networks:
      - elastic  
    depends_on:
      - es01
      - es02
      - es03  

  wait_until_ready:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: /usr/bin/true
    depends_on: {"es01": {"condition": "service_healthy"}}

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local
  certs:
    driver: local      

networks:
  elastic:
    driver: bridge

PS. It used to work fine, but we wanted to use a smaller instance, so I moved it

warkolm · January 12, 2022, 10:57pm

Please share your Elasticsearch and Kibana logs.

Spyros_Agriopoulos · January 13, 2022, 8:51am

Kibana logs

{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["info","plugins-service"],"pid":1219,"message":"Plugin \"metricsEntities\" is disabled."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["info","http","server","Preboot"],"pid":1219,"message":"http server running at http://0.0.0.0:5601"}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Support for setting server.host to \"0\" in kibana.yml is deprecated and will be removed in Kibana version 8.0.0. Instead use \"0.0.0.0\" to bind to all interfaces."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Setting [elasticsearch.username] to \"elastic\" is deprecated. You should use the \"kibana_system\" user instead."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"plugins.scanDirs is deprecated and is no longer used"}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Config key [monitoring.cluster_alerts.email_notifications.email_address] will be required for email notifications to work in 8.0.\""}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Setting [monitoring.username] to \"elastic\" is deprecated. You should use the \"kibana_system\" user instead."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"\"xpack.reporting.roles\" is deprecated. Granting reporting privilege through a \"reporting_user\" role will not be supported starting in 8.0. Please set \"xpack.reporting.roles.enabled\" to \"false\" and grant reporting privileges to users using Kibana application privileges **Management > Security > Roles**."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Session idle timeout (\"xpack.security.session.idleTimeout\") will be set to 1 hour by default in the next major version (8.0)."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Session lifespan (\"xpack.security.session.lifespan\") will be set to 30 days by default in the next major version (8.0)."}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["info","plugins-system","standard"],"pid":1219,"message":"Setting up [113] plugins: [translations,licensing,globalSearch,globalSearchProviders,banners,licenseApiGuard,code,usageCollection,xpackLegacy,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,securityOss,share,screenshotMode,telemetry,newsfeed,mapsEms,mapsLegacy,legacyExport,kibanaLegacy,embeddable,uiActionsEnhanced,fieldFormats,expressions,charts,esUiShared,bfetch,data,savedObjects,visualizations,visTypeXy,visTypeVislib,visTypeTimelion,features,visTypeTagcloud,visTypeTable,visTypePie,visTypeMetric,visTypeMarkdown,tileMap,regionMap,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,timelion,indexPatternFieldEditor,home,searchprofiler,painlessLab,grokdebugger,graph,visTypeVega,management,watcher,licenseManagement,indexPatternManagement,advancedSettings,discover,discoverEnhanced,dashboard,dashboardEnhanced,visualize,visTypeTimeseries,savedObjectsManagement,spaces,security,transform,savedObjectsTagging,lens,reporting,canvas,lists,ingestPipelines,fileUpload,maps,dataVisualizer,encryptedSavedObjects,dataEnhanced,dashboardMode,cloud,snapshotRestore,fleet,indexManagement,rollup,remoteClusters,crossClusterReplication,indexLifecycleManagement,eventLog,actions,alerting,triggersActionsUi,stackAlerts,ruleRegistry,osquery,ml,cases,timelines,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,console,apmOss,apm]"}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["info","plugins","taskManager"],"pid":1219,"message":"TaskManager is identified by the Kibana UUID: 80663143-fed4-4e76-a9c1-a75e998a7b92"}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["warning","plugins","security","config"],"pid":1219,"message":"Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["warning","plugins","security","config"],"pid":1219,"message":"Session cookies will be transmitted over insecure connections. This is not recommended."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["warning","plugins","reporting","config"],"pid":1219,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.reporting.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["warning","plugins","reporting","config"],"pid":1219,"message":"Chromium sandbox provides an additional layer of protection, but is not supported for Linux CentOS 8.4.2105\n OS. Automatically setting 'xpack.reporting.capture.browser.chromium.disableSandbox: true'."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing common resources shared between all indices"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.uptime.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.logs.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.metrics.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.apm.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","savedobjects-service"],"pid":1219,"message":"Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations..."}
{"type":"log","@timestamp":"2022-01-13T08:45:42+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:44+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:47+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:49+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:52+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:54+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:56+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:59+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:01+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:04+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:06+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:09+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:14+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:17+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:21+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:26+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:31+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:36+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}

Spyros_Agriopoulos · January 13, 2022, 8:52am

Elasticsearch logs part 1

Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:45:39,860Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[9], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:45:39,876Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:39,877Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-12896459498932564283, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,340Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,347Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,364Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,368Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,370Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,370Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,371Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,372Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,372Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,373Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,374Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,374Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,379Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,386Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,387Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,391Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,391Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,393Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,393Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,394Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,395Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,396Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,401Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,403Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,404Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,415Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,416Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,625Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdf)]], net usable_space [529.9gb], net total_space [589.5gb], types [ext4]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,625Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,936Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "uncaught exception in thread [main]", 
"stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:171) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.main(Command.java:79) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81) ~[elasticsearch-7.15.1.jar:7.15.1]",
"Caused by: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:383) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:288) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",

Spyros_Agriopoulos · January 13, 2022, 8:52am

Elasticsearch logs part 2

"at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167) ~[elasticsearch-7.15.1.jar:7.15.1]",
"... 6 more"] }
uncaught exception in thread [main]
java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]
	at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83)
	at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423)
	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320)
	at org.elasticsearch.node.Node.<init>(Node.java:383)
	at org.elasticsearch.node.Node.<init>(Node.java:288)
	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399)
	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167)
	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158)
	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75)
	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114)
	at org.elasticsearch.cli.Command.main(Command.java:79)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:46:10,747Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[6], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:46:10,815Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:46:10,815Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-17850755216468522263, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,791Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,791Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,792Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,792Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,793Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,798Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,910Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,924Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,925Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,925Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,926Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,926Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,927Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,927Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,928Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,937Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,937Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,938Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,939Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,939Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,940Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,940Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,952Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,965Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,965Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,966Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,967Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,972Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": 
"o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,472Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdf)]], net usable_space [529.9gb], net total_space [589.5gb], types [ext4]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,474Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
uncaught exception in thread [main]
java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]
	at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83)
	at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423)
	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320)
	at org.elasticsearch.node.Node.<init>(Node.java:383)
	at org.elasticsearch.node.Node.<init>(Node.java:288)
	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399)
	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167)
	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158)
	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75)
	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114)
	at org.elasticsearch.cli.Command.main(Command.java:79)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
{"type": "server", "timestamp": "2022-01-13T08:46:45,713Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "uncaught exception in thread [main]", 
"stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:171) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.main(Command.java:79) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81) ~[elasticsearch-7.15.1.jar:7.15.1]",
"Caused by: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:383) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:288) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167) ~[elasticsearch-7.15.1.jar:7.15.1]",
"... 6 more"] }
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:48:01,482Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[7], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:48:01,486Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:48:01,487Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-14287106734366476380, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }

warkolm · January 13, 2022, 8:57am

That's your problem. You need to be running 7.16.2 or higher.

Spyros_Agriopoulos · January 13, 2022, 1:22pm

Thanks manage to fix it with your help!

system · February 10, 2022, 1:22pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.

Topic		Replies	Views
Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 127.0.0.1:9200 Kibana elastic-stack-security , docker	1	7382	June 22, 2022
Unable to retrieve version information from Elasticsearch nodes. unable to get local issuer certificate Elasticsearch elastic-stack-security , docker	1	739	September 8, 2021
Unable to retrieve version information from Elasticsearch nodes Kibana docker	1	209	September 23, 2023
Kibana <---> Elasticsearch integration with SSL enable Kibana elastic-stack-security , docker	6	486	September 16, 2022
[ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. getaddrinfo ENOTFOUND poem Elasticsearch docker	2	1526	July 10, 2022

Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.19.0.3:9200

Related topics