Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.19.0.3:9200

Spyros_Agriopoulos · January 12, 2022, 3:45pm

Hello guys,

I am trying to setup an ELK stack with docker compose and TLS, and I get this message from kibana's logs.

Anything wrong with my yml?

version: '2.2'
services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es01
    restart: unless-stopped
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es01/es01.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate 
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es01/es01.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es01/es01.key  
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data01:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    ports:
      - 9200:9200
    healthcheck:
      test: curl --cacert $CERTS_DIR/ca/ca.crt -s https://localhost:9200 >/dev/null; if [[ $$? == 52 ]]; then echo 0; else echo 1; fi
      interval: 30s
      timeout: 10s
      retries: 5    
    networks:
      - elastic
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es02
    restart: unless-stopped
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es02/es02.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es02/es02.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es02/es02.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data02:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: es03
    restart: unless-stopped
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - ELASTIC_PASSWORD=pass 
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=$CERTS_DIR/es03/es03.key
      - xpack.security.http.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.http.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.certificate_authorities=$CERTS_DIR/ca/ca.crt
      - xpack.security.transport.ssl.certificate=$CERTS_DIR/es03/es03.crt
      - xpack.security.transport.ssl.key=$CERTS_DIR/es03/es03.key
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes: ['data03:/usr/share/elasticsearch/data', 'certs:$CERTS_DIR']
    networks:
      - elastic
  
  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.1
    restart: unless-stopped
    environment:
      - ELASTICSEARCH_HOSTS=["https://es01:9200","https://es02:9200","https://es03:9200"]
      - XPACK_SECURITY_ENABLED=true
      - ELASTICSEARCH_USERNAME=elastic
      - ELASTICSEARCH_PASSWORD=pass
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=$CERTS_DIR/ca/ca.crt
      - ELASTICSEARCH_SSL_VERIFICATIONMODE=certificate
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=d3e5bd37d75a348b5e0736cbccf515b6  
    ports:
      - 5601:5601
    volumes: ['certs:$CERTS_DIR']    
    networks:
      - elastic  
    depends_on:
      - es01
      - es02
      - es03  

  wait_until_ready:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    command: /usr/bin/true
    depends_on: {"es01": {"condition": "service_healthy"}}

volumes:
  data01:
    driver: local
  data02:
    driver: local
  data03:
    driver: local
  certs:
    driver: local      

networks:
  elastic:
    driver: bridge

PS. It used to work fine, but we wanted to use a smaller instance, so I moved it

warkolm · January 12, 2022, 10:57pm

Please share your Elasticsearch and Kibana logs.

Spyros_Agriopoulos · January 13, 2022, 8:51am

Kibana logs

{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["info","plugins-service"],"pid":1219,"message":"Plugin \"metricsEntities\" is disabled."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["info","http","server","Preboot"],"pid":1219,"message":"http server running at http://0.0.0.0:5601"}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Support for setting server.host to \"0\" in kibana.yml is deprecated and will be removed in Kibana version 8.0.0. Instead use \"0.0.0.0\" to bind to all interfaces."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Setting [elasticsearch.username] to \"elastic\" is deprecated. You should use the \"kibana_system\" user instead."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"plugins.scanDirs is deprecated and is no longer used"}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Config key [monitoring.cluster_alerts.email_notifications.email_address] will be required for email notifications to work in 8.0.\""}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Setting [monitoring.username] to \"elastic\" is deprecated. You should use the \"kibana_system\" user instead."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"\"xpack.reporting.roles\" is deprecated. Granting reporting privilege through a \"reporting_user\" role will not be supported starting in 8.0. Please set \"xpack.reporting.roles.enabled\" to \"false\" and grant reporting privileges to users using Kibana application privileges **Management > Security > Roles**."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Session idle timeout (\"xpack.security.session.idleTimeout\") will be set to 1 hour by default in the next major version (8.0)."}
{"type":"log","@timestamp":"2022-01-13T08:45:34+00:00","tags":["warning","config","deprecation"],"pid":1219,"message":"Session lifespan (\"xpack.security.session.lifespan\") will be set to 30 days by default in the next major version (8.0)."}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["info","plugins-system","standard"],"pid":1219,"message":"Setting up [113] plugins: [translations,licensing,globalSearch,globalSearchProviders,banners,licenseApiGuard,code,usageCollection,xpackLegacy,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,securityOss,share,screenshotMode,telemetry,newsfeed,mapsEms,mapsLegacy,legacyExport,kibanaLegacy,embeddable,uiActionsEnhanced,fieldFormats,expressions,charts,esUiShared,bfetch,data,savedObjects,visualizations,visTypeXy,visTypeVislib,visTypeTimelion,features,visTypeTagcloud,visTypeTable,visTypePie,visTypeMetric,visTypeMarkdown,tileMap,regionMap,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,timelion,indexPatternFieldEditor,home,searchprofiler,painlessLab,grokdebugger,graph,visTypeVega,management,watcher,licenseManagement,indexPatternManagement,advancedSettings,discover,discoverEnhanced,dashboard,dashboardEnhanced,visualize,visTypeTimeseries,savedObjectsManagement,spaces,security,transform,savedObjectsTagging,lens,reporting,canvas,lists,ingestPipelines,fileUpload,maps,dataVisualizer,encryptedSavedObjects,dataEnhanced,dashboardMode,cloud,snapshotRestore,fleet,indexManagement,rollup,remoteClusters,crossClusterReplication,indexLifecycleManagement,eventLog,actions,alerting,triggersActionsUi,stackAlerts,ruleRegistry,osquery,ml,cases,timelines,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,console,apmOss,apm]"}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["info","plugins","taskManager"],"pid":1219,"message":"TaskManager is identified by the Kibana UUID: 80663143-fed4-4e76-a9c1-a75e998a7b92"}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["warning","plugins","security","config"],"pid":1219,"message":"Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."}
{"type":"log","@timestamp":"2022-01-13T08:45:35+00:00","tags":["warning","plugins","security","config"],"pid":1219,"message":"Session cookies will be transmitted over insecure connections. This is not recommended."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["warning","plugins","reporting","config"],"pid":1219,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.reporting.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["warning","plugins","reporting","config"],"pid":1219,"message":"Chromium sandbox provides an additional layer of protection, but is not supported for Linux CentOS 8.4.2105\n OS. Automatically setting 'xpack.reporting.capture.browser.chromium.disableSandbox: true'."}
{"type":"log","@timestamp":"2022-01-13T08:45:36+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing common resources shared between all indices"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.uptime.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.logs.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.metrics.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","plugins","ruleRegistry"],"pid":1219,"message":"Write is disabled; not installing resources for index .alerts-observability.apm.alerts"}
{"type":"log","@timestamp":"2022-01-13T08:45:37+00:00","tags":["info","savedobjects-service"],"pid":1219,"message":"Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations..."}
{"type":"log","@timestamp":"2022-01-13T08:45:42+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:44+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:47+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:49+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:52+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:54+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:56+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:45:59+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:01+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:04+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:06+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:09+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:14+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:17+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:21+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:26+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.3:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:31+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.2:9200"}
{"type":"log","@timestamp":"2022-01-13T08:46:36+00:00","tags":["error","savedobjects-service"],"pid":1219,"message":"Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 172.24.0.4:9200"}

Spyros_Agriopoulos · January 13, 2022, 8:52am

Elasticsearch logs part 1

Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:45:39,860Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[9], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:45:39,876Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:39,877Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-12896459498932564283, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,340Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,347Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,355Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,356Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,357Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,358Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,364Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,368Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,369Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,370Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,370Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,371Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,372Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,372Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,373Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,374Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,374Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,379Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,380Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,383Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,386Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,387Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,391Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,391Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,392Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,393Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,393Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,394Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,395Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,396Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,401Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,403Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,404Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,415Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,416Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,625Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdf)]], net usable_space [529.9gb], net total_space [589.5gb], types [ext4]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,625Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
{"type": "server", "timestamp": "2022-01-13T08:45:51,936Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "uncaught exception in thread [main]", 
"stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:171) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.main(Command.java:79) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81) ~[elasticsearch-7.15.1.jar:7.15.1]",
"Caused by: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:383) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:288) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",

Spyros_Agriopoulos · January 13, 2022, 8:52am

Elasticsearch logs part 2

"at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167) ~[elasticsearch-7.15.1.jar:7.15.1]",
"... 6 more"] }
uncaught exception in thread [main]
java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]
	at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83)
	at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423)
	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320)
	at org.elasticsearch.node.Node.<init>(Node.java:383)
	at org.elasticsearch.node.Node.<init>(Node.java:288)
	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399)
	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167)
	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158)
	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75)
	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114)
	at org.elasticsearch.cli.Command.main(Command.java:79)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:46:10,747Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[6], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:46:10,815Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:46:10,815Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-17850755216468522263, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,791Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,791Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,792Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,792Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,793Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,798Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,910Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,911Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,924Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,925Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,925Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,926Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,926Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,927Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,927Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,928Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,937Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,937Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,938Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,939Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,939Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,940Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,940Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,949Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vector-tile]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,950Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,951Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,952Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,965Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,965Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,966Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,967Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2022-01-13T08:46:43,972Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,030Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,031Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,032Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,033Z", "level": "INFO", "component": 
"o.e.p.PluginsService", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,472Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/xvdf)]], net usable_space [529.9gb], net total_space [589.5gb], types [ext4]" }
{"type": "server", "timestamp": "2022-01-13T08:46:44,474Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "heap size [512mb], compressed ordinary object pointers [true]" }
uncaught exception in thread [main]
java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]
	at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83)
	at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423)
	at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320)
	at org.elasticsearch.node.Node.<init>(Node.java:383)
	at org.elasticsearch.node.Node.<init>(Node.java:288)
	at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219)
	at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399)
	at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167)
	at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158)
	at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75)
	at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114)
	at org.elasticsearch.cli.Command.main(Command.java:79)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123)
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81)
For complete error details, refer to the log at /usr/share/elasticsearch/logs/es-docker-cluster.log
{"type": "server", "timestamp": "2022-01-13T08:46:45,713Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "uncaught exception in thread [main]", 
"stacktrace": ["org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:171) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:158) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:75) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:114) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.cli.Command.main(Command.java:79) ~[elasticsearch-cli-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:123) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:81) ~[elasticsearch-7.15.1.jar:7.15.1]",
"Caused by: java.lang.IllegalStateException: cannot downgrade a node from version [7.16.2] to version [7.15.1]",
"at org.elasticsearch.env.NodeMetadata.upgradeToCurrentVersion(NodeMetadata.java:83) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.loadNodeMetadata(NodeEnvironment.java:423) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.env.NodeEnvironment.<init>(NodeEnvironment.java:320) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:383) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.node.Node.<init>(Node.java:288) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap$5.<init>(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:219) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:399) ~[elasticsearch-7.15.1.jar:7.15.1]",
"at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:167) ~[elasticsearch-7.15.1.jar:7.15.1]",
"... 6 more"] }
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Elasticsearch (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Elasticsearch
WARNING: System::setSecurityManager will be removed in a future release
WARNING: A terminally deprecated method in java.lang.System has been called
WARNING: System::setSecurityManager has been called by org.elasticsearch.bootstrap.Security (file:/usr/share/elasticsearch/lib/elasticsearch-7.15.1.jar)
WARNING: Please consider reporting this to the maintainers of org.elasticsearch.bootstrap.Security
WARNING: System::setSecurityManager will be removed in a future release
{"type": "server", "timestamp": "2022-01-13T08:48:01,482Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "version[7.15.1], pid[7], build[default/docker/83c34f456ae29d60e94d886e455e6a3409bba9ed/2021-10-07T21:56:19.031608185Z], OS[Linux/5.11.0-1025-aws/amd64], JVM[Eclipse Adoptium/OpenJDK 64-Bit Server VM/17/17+35]" }
{"type": "server", "timestamp": "2022-01-13T08:48:01,486Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2022-01-13T08:48:01,487Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-14287106734366476380, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }

warkolm · January 13, 2022, 8:57am

That's your problem. You need to be running 7.16.2 or higher.

Spyros_Agriopoulos · January 13, 2022, 1:22pm

Thanks manage to fix it with your help!

system · February 10, 2022, 1:22pm

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.