Kibana] Action failed with 'Request timed out' while CREATE_NEW_TARGET -> CREATE_NEW_TARGET in docker swarm environment

Elastic Stack Kibana
1

Hi Community, I am trying to deploy Elastic Cluster with 3 nodes on a docker swarm environment with Kibana.

All 4 services (3 elastic and 1 kibana) are running fine. Below image for reference.

image
image1338×184 50.7 KB

But Kibana is failing with error "kibana] Action failed with 'Request timed out' while CREATE_NEW_TARGET -> CREATE_NEW_TARGET in docker swarm environment"

After some time, kibana service failed.

docker-compose (elk.yml) file

version: '3.7'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.2
    volumes:
      - "/mnt/es01:/usr/share/elasticsearch/data"
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      node.name: "elasticsearch"
      node.roles: "master"
      cluster.name: "docker-cluster"
      network.host: "0.0.0.0"
      http.port: "9200"
      http.host: "0.0.0.0"
      discovery.seed_hosts: "elasticsearch,es02,es03"
      cluster.initial_master_nodes: "elasticsearch"
      ELASTIC_PASSWORD: "changeme"
      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "false"
      xpack.security.transport.ssl.enabled: "false"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - caddy
    deploy:
      placement:
        constraints: [node.role == manager]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.2
    depends_on:
      - elasticsearch
    volumes:
      - "/mnt/es02:/usr/share/elasticsearch/data"
    ports:
      - "9201:9200"
      - "9301:9300"
    environment:
      node.name: "es02"
      node.roles: "data"
      cluster.name: "docker-cluster"
      network.host: "0.0.0.0"
      http.port: "9200"
      http.host: "0.0.0.0"
      discovery.seed_hosts: "elasticsearch,es02,es03"
      cluster.initial_master_nodes: "es02"
      ELASTIC_PASSWORD: "changeme"
      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "false"
      xpack.security.transport.ssl.enabled: "false"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - caddy
    deploy:
      placement:
        constraints: [node.role == manager]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.2
    depends_on:
      - es02
    volumes:
      - "/mnt/es03:/usr/share/elasticsearch/data"
    ports:
      - "9202:9200"
      - "9302:9300"
    environment:
      node.name: "es03"
      node.roles: "data"
      cluster.name: "docker-cluster"
      network.host: "0.0.0.0"
      http.port: "9200"
      http.host: "0.0.0.0"
      discovery.seed_hosts: "elasticsearch,es02,es03"
      cluster.initial_master_nodes: "es03"
      ELASTIC_PASSWORD: "changeme"
      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
      bootstrap.memory_lock: "true"
      xpack.security.enabled: "false"
      xpack.security.transport.ssl.enabled: "false"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks:
      - caddy
    deploy:
      placement:
        constraints: [node.role == manager]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
  kibana:
    image: docker.elastic.co/kibana/kibana:8.2.2
    volumes:
      - "/mnt/config/kibana.yml:/usr/share/kibana/config/kibana.yml"
      - "/mnt/kibana:/usr/share/kibana/data"
    ports:
      - "5601:5601"
    networks:
      - caddy
    depends_on:
      - elasticsearch
    deploy:
      placement:
        constraints: [node.role == worker]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
volumes:
  es01:
    driver: "local"
  es02:
    driver: "local"
  es03:
    driver: "local"
  kibana:
    driver: "local"
  config:
    driver: "local"  
networks:
  caddy:
    external: true

Kibana Configuration

---
server.name: kibana
server.host: 0.0.0.0
elasticsearch.hosts: [ "http://elasticsearch:9200", "http://es02:9200", "http://es03:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: kibana_system
elasticsearch.password: changeme

kibana logs

elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.390+00:00][INFO ][plugins-service] Plugin "cloudSecurityPosture" is disabled.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.421+00:00][INFO ][plugins-service] Plugin "metricsEntities" is disabled.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.503+00:00][INFO ][http.server.Preboot] http server running at http://0.0.0.0:5601
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.543+00:00][INFO ][plugins-system.preboot] Setting up [1] plugins: [interactiveSetup]
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.576+00:00][WARN ][config.deprecation] The default mechanism for Reporting privileges will work differently in future versions, which will affect the behavior of this cluster. Set "xpack.reporting.roles.enabled" to "false" to adopt the future behavior before upgrading.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.775+00:00][INFO ][plugins-system.standard] Setting up [117] plugins: [translations,monitoringCollection,licensing,globalSearch,globalSearchProviders,features,mapsEms,licenseApiGuard,usageCollection,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,share,embeddable,uiActionsEnhanced,screenshotMode,screenshotting,banners,newsfeed,fieldFormats,expressions,eventAnnotation,dataViews,charts,esUiShared,bfetch,data,savedObjects,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,customIntegrations,home,searchprofiler,painlessLab,grokdebugger,management,watcher,advancedSettings,spaces,security,savedObjectsTagging,reporting,lists,fileUpload,ingestPipelines,encryptedSavedObjects,dataEnhanced,cloud,snapshotRestore,telemetry,licenseManagement,eventLog,actions,alerting,triggersActionsUi,transform,stackAlerts,ruleRegistry,savedObjectsManagement,console,controls,graph,fleet,indexManagement,remoteClusters,crossClusterReplication,indexLifecycleManagement,visualizations,canvas,visTypeXy,visTypeVislib,visTypeVega,visTypeTimeseries,rollup,visTypeTimelion,visTypeTagcloud,visTypeTable,visTypeMetric,visTypeHeatmap,visTypeMarkdown,dashboard,dashboardEnhanced,expressionXY,expressionTagcloud,expressionPartitionVis,visTypePie,expressionMetricVis,expressionHeatmap,expressionGauge,visTypeGauge,dataViewFieldEditor,sharedUX,discover,lens,osquery,maps,dataVisualizer,ml,cases,timelines,sessionView,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,apm,discoverEnhanced,dataViewManagement]
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.790+00:00][INFO ][plugins.taskManager] TaskManager is identified by the Kibana UUID: 4d187746-6f68-4518-8f1c-feb718b10f0e
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.919+00:00][WARN ][plugins.security.config] Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.920+00:00][WARN ][plugins.security.config] Session cookies will be transmitted over insecure connections. This is not recommended.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.945+00:00][WARN ][plugins.security.config] Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.946+00:00][WARN ][plugins.security.config] Session cookies will be transmitted over insecure connections. This is not recommended.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.963+00:00][WARN ][plugins.reporting.config] Generating a random key for xpack.reporting.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.reporting.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.964+00:00][WARN ][plugins.reporting.config] Found 'server.host: "0.0.0.0"' in Kibana configuration. Reporting is not able to use this as the Kibana server hostname. To enable PNG/PDF Reporting to work, 'xpack.reporting.kibanaServer.hostname: localhost' is automatically set in the configuration. You can prevent this message by adding 'xpack.reporting.kibanaServer.hostname: localhost' in kibana.yml.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.973+00:00][WARN ][plugins.encryptedSavedObjects] Saved objects encryption key is not set. This will severely limit Kibana functionality. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:58.990+00:00][WARN ][plugins.actions] APIs are disabled because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:59.007+00:00][WARN ][plugins.alerting] APIs are disabled because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:59.022+00:00][INFO ][plugins.ruleRegistry] Installing common resources shared between all indices
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:59.603+00:00][INFO ][plugins.screenshotting.config] Chromium sandbox provides an additional layer of protection, and is supported for Linux Ubuntu 20.04 OS. Automatically enabling Chromium sandbox.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:07:59.658+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 10.0.1.138:9200
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:08:00.334+00:00][INFO ][plugins.screenshotting.chromium] Browser executable: /usr/share/kibana/x-pack/plugins/screenshotting/chromium/headless_shell-linux_x64/headless_shell
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:08:02.156+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 10.0.1.131:9200
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:08:04.645+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. There are no living connections
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:19:37.312+00:00][INFO ][savedobjects-service] Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:19:37.313+00:00][INFO ][savedobjects-service] Starting saved objects migrations
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:07.337+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'Request timed out'. Retrying attempt 1 in 2 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:07.338+00:00][INFO ][savedobjects-service] [.kibana_task_manager] INIT -> INIT. took: 30012ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:09.351+00:00][INFO ][savedobjects-service] [.kibana_task_manager] INIT -> CREATE_NEW_TARGET. took: 2014ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:37.333+00:00][ERROR][savedobjects-service] [.kibana] Action failed with 'Request timed out'. Retrying attempt 1 in 2 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:37.334+00:00][INFO ][savedobjects-service] [.kibana] INIT -> INIT. took: 60009ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:39.343+00:00][INFO ][savedobjects-service] [.kibana] INIT -> CREATE_NEW_TARGET. took: 2010ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:39.353+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'Request timed out'. Retrying attempt 1 in 2 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:20:39.353+00:00][INFO ][savedobjects-service] [.kibana_task_manager] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 30002ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:09.346+00:00][ERROR][savedobjects-service] [.kibana] Action failed with 'Request timed out'. Retrying attempt 1 in 2 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:09.346+00:00][INFO ][savedobjects-service] [.kibana] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 30003ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:41.369+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'There are no living connections'. Retrying attempt 2 in 4 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:41.369+00:00][INFO ][savedobjects-service] [.kibana_task_manager] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 62016ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:42.205+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. There are no living connections
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:45.372+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'There are no living connections'. Retrying attempt 3 in 8 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:45.372+00:00][INFO ][savedobjects-service] [.kibana_task_manager] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 4003ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:53.375+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'There are no living connections'. Retrying attempt 4 in 16 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:21:53.375+00:00][INFO ][savedobjects-service] [.kibana_task_manager] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 8003ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:23:11.363+00:00][ERROR][savedobjects-service] [.kibana] Action failed with 'Request timed out'. Retrying attempt 2 in 4 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:23:11.363+00:00][INFO ][savedobjects-service] [.kibana] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 122017ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:24:09.389+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'Request timed out'. Retrying attempt 5 in 32 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:24:09.390+00:00][INFO ][savedobjects-service] [.kibana_task_manager] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 136014ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:25:15.384+00:00][ERROR][savedobjects-service] [.kibana] Action failed with 'Request timed out'. Retrying attempt 3 in 8 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:25:15.385+00:00][INFO ][savedobjects-service] [.kibana] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 124021ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:26:41.404+00:00][ERROR][savedobjects-service] [.kibana_task_manager] Action failed with 'Request timed out'. Retrying attempt 6 in 64 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:26:41.404+00:00][INFO ][savedobjects-service] [.kibana_task_manager] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 152015ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:27:23.405+00:00][ERROR][savedobjects-service] [.kibana] Action failed with 'Request timed out'. Retrying attempt 4 in 16 seconds.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:27:23.405+00:00][INFO ][savedobjects-service] [.kibana] CREATE_NEW_TARGET -> CREATE_NEW_TARGET. took: 128021ms.
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:27:59.023+00:00][ERROR][plugins.ruleRegistry] Error: Timeout: it took more than 1200000ms
elk_kibana.1.fdnk5mhmq8t8@swarmn    |     at Timeout._onTimeout (/usr/share/kibana/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.js:54:20)
elk_kibana.1.fdnk5mhmq8t8@swarmn    |     at listOnTimeout (node:internal/timers:559:17)
elk_kibana.1.fdnk5mhmq8t8@swarmn    |     at processTimers (node:internal/timers:502:7)
elk_kibana.1.fdnk5mhmq8t8@swarmn    | [2022-06-14T07:27:59.025+00:00][ERROR][plugins.ruleRegistry] Error: Failure installing common resources shared between all indices. Timeout: it took more than 1200000ms
elk_kibana.1.fdnk5mhmq8t8@swarmn    |     at ResourceInstaller.installWithTimeout (/usr/share/kibana/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.js:63:13)
elk_kibana.1.fdnk5mhmq8t8@swarmn    |     at ResourceInstaller.installCommonResources (/usr/share/kibana/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.js:77:5)

Elastic nodes (Elasticsearch, es02 and es03) have huge logs. I am getting body limit error while pasting them here. Is there a way to attach the logs?

2

Are you sure Elasticsearch is accessible at that IP?

3

Now sure how Elasticsearch cluster works. Today I tried single node Elasticsearch, kibana was able to communicate with Elasticsearch (even though Elasticsearch uses internal IP). I think how the docker networking works.

Also I am using Caddy as reverse proxy in our docker swarm cluster.

But [status] Kibana is now available (was degraded). Unable to access kibana in this case as well.

kibana logs

elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:05.678+00:00][INFO ][plugins-service] Plugin "cloudSecurityPosture" is disabled.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:05.726+00:00][INFO ][plugins-service] Plugin "metricsEntities" is disabled.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:05.825+00:00][INFO ][http.server.Preboot] http server running at http://0.0.0.0:5601
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:05.876+00:00][INFO ][plugins-system.preboot] Setting up [1] plugins: [interactiveSetup]
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:06.151+00:00][INFO ][plugins-system.standard] Setting up [117] plugins: [translations,monitoringCollection,licensing,globalSearch,globalSearchProviders,features,mapsEms,licenseApiGuard,usageCollection,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,share,embeddable,uiActionsEnhanced,screenshotMode,screenshotting,banners,newsfeed,fieldFormats,expressions,eventAnnotation,dataViews,charts,esUiShared,bfetch,data,savedObjects,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,customIntegrations,home,searchprofiler,painlessLab,grokdebugger,management,watcher,advancedSettings,spaces,security,savedObjectsTagging,reporting,lists,fileUpload,ingestPipelines,encryptedSavedObjects,dataEnhanced,cloud,snapshotRestore,telemetry,licenseManagement,eventLog,actions,alerting,triggersActionsUi,transform,stackAlerts,ruleRegistry,savedObjectsManagement,console,controls,graph,fleet,indexManagement,remoteClusters,crossClusterReplication,indexLifecycleManagement,visualizations,canvas,visTypeXy,visTypeVislib,visTypeVega,visTypeTimeseries,rollup,visTypeTimelion,visTypeTagcloud,visTypeTable,visTypeMetric,visTypeHeatmap,visTypeMarkdown,dashboard,dashboardEnhanced,expressionXY,expressionTagcloud,expressionPartitionVis,visTypePie,expressionMetricVis,expressionHeatmap,expressionGauge,visTypeGauge,dataViewFieldEditor,sharedUX,discover,lens,osquery,maps,dataVisualizer,ml,cases,timelines,sessionView,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,apm,discoverEnhanced,dataViewManagement]
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:06.166+00:00][INFO ][plugins.taskManager] TaskManager is identified by the Kibana UUID: 908d079b-eec5-4d19-9d42-41a54e61acaa
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:06.301+00:00][WARN ][plugins.security.config] Session cookies will be transmitted over insecure connections. This is not recommended.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:06.321+00:00][WARN ][plugins.security.config] Session cookies will be transmitted over insecure connections. This is not recommended.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:06.340+00:00][WARN ][plugins.reporting.config] Found 'server.host: "0.0.0.0"' in Kibana configuration. Reporting is not able to use this as the Kibana server hostname. To enable PNG/PDF Reporting to work, 'xpack.reporting.kibanaServer.hostname: localhost' is automatically set in the configuration. You can prevent this message by adding 'xpack.reporting.kibanaServer.hostname: localhost' in kibana.yml.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:06.406+00:00][INFO ][plugins.ruleRegistry] Installing common resources shared between all indices
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:07.029+00:00][INFO ][plugins.screenshotting.config] Chromium sandbox provides an additional layer of protection, and is supported for Linux Ubuntu 20.04 OS. Automatically enabling Chromium sandbox.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:07.155+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. connect ECONNREFUSED 10.0.1.217:9200
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:07.809+00:00][INFO ][plugins.screenshotting.chromium] Browser executable: /usr/share/kibana/x-pack/plugins/screenshotting/chromium/headless_shell-linux_x64/headless_shell
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:19.969+00:00][ERROR][elasticsearch-service] Unable to retrieve version information from Elasticsearch nodes. security_exception: [security_exception] Reason: unable to authenticate user [kibana_system] for REST request [/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip]
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.067+00:00][INFO ][savedobjects-service] Waiting until all Elasticsearch nodes are compatible with Kibana before starting saved objects migrations...
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.068+00:00][INFO ][savedobjects-service] Starting saved objects migrations
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.124+00:00][INFO ][savedobjects-service] [.kibana_task_manager] INIT -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT. took: 41ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.138+00:00][INFO ][savedobjects-service] [.kibana] INIT -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT. took: 57ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.430+00:00][INFO ][savedobjects-service] [.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ. took: 306ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.434+00:00][INFO ][savedobjects-service] [.kibana] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ. took: 296ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.556+00:00][INFO ][savedobjects-service] [.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT. took: 126ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.563+00:00][INFO ][savedobjects-service] [.kibana] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT. took: 129ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.571+00:00][INFO ][savedobjects-service] [.kibana] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> UPDATE_TARGET_MAPPINGS. took: 8ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.574+00:00][INFO ][savedobjects-service] [.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> UPDATE_TARGET_MAPPINGS. took: 18ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:24.946+00:00][INFO ][savedobjects-service] [.kibana_task_manager] UPDATE_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK. took: 372ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:25.052+00:00][INFO ][savedobjects-service] [.kibana] UPDATE_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK. took: 481ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:28.468+00:00][INFO ][savedobjects-service] [.kibana_task_manager] UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> DONE. took: 3522ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:28.469+00:00][INFO ][savedobjects-service] [.kibana_task_manager] Migration completed after 4386ms
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:28.470+00:00][INFO ][savedobjects-service] [.kibana] UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> DONE. took: 3418ms.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:28.470+00:00][INFO ][savedobjects-service] [.kibana] Migration completed after 4389ms
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:28.477+00:00][INFO ][plugins-system.preboot] Stopping all plugins.
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:28.480+00:00][INFO ][plugins-system.standard] Starting [117] plugins: [translations,monitoringCollection,licensing,globalSearch,globalSearchProviders,features,mapsEms,licenseApiGuard,usageCollection,taskManager,telemetryCollectionManager,telemetryCollectionXpack,kibanaUsageCollection,share,embeddable,uiActionsEnhanced,screenshotMode,screenshotting,banners,newsfeed,fieldFormats,expressions,eventAnnotation,dataViews,charts,esUiShared,bfetch,data,savedObjects,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,customIntegrations,home,searchprofiler,painlessLab,grokdebugger,management,watcher,advancedSettings,spaces,security,savedObjectsTagging,reporting,lists,fileUpload,ingestPipelines,encryptedSavedObjects,dataEnhanced,cloud,snapshotRestore,telemetry,licenseManagement,eventLog,actions,alerting,triggersActionsUi,transform,stackAlerts,ruleRegistry,savedObjectsManagement,console,controls,graph,fleet,indexManagement,remoteClusters,crossClusterReplication,indexLifecycleManagement,visualizations,canvas,visTypeXy,visTypeVislib,visTypeVega,visTypeTimeseries,rollup,visTypeTimelion,visTypeTagcloud,visTypeTable,visTypeMetric,visTypeHeatmap,visTypeMarkdown,dashboard,dashboardEnhanced,expressionXY,expressionTagcloud,expressionPartitionVis,visTypePie,expressionMetricVis,expressionHeatmap,expressionGauge,visTypeGauge,dataViewFieldEditor,sharedUX,discover,lens,osquery,maps,dataVisualizer,ml,cases,timelines,sessionView,securitySolution,observability,uptime,infra,upgradeAssistant,monitoring,logstash,enterpriseSearch,apm,discoverEnhanced,dataViewManagement]
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:29.405+00:00][INFO ][plugins.monitoring.monitoring] config sourced from: production cluster
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:30.930+00:00][INFO ][http.server.Kibana] http server running at http://0.0.0.0:5601
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.015+00:00][INFO ][status] Kibana is now degraded
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.237+00:00][INFO ][plugins.monitoring.monitoring.kibana-monitoring] Starting monitoring stats collection
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.238+00:00][INFO ][plugins.fleet] Beginning fleet setup
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.605+00:00][INFO ][plugins.ml] Task ML:saved-objects-sync-task: scheduled with interval 1h
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.622+00:00][INFO ][plugins.ruleRegistry] Installed common resources shared between all indices
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.622+00:00][INFO ][plugins.ruleRegistry] Installing resources for index .alerts-security.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.623+00:00][INFO ][plugins.ruleRegistry] Installing resources for index .preview.alerts-security.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.623+00:00][INFO ][plugins.ruleRegistry] Installing resources for index .alerts-observability.uptime.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.623+00:00][INFO ][plugins.ruleRegistry] Installing resources for index .alerts-observability.logs.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.623+00:00][INFO ][plugins.ruleRegistry] Installing resources for index .alerts-observability.metrics.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.624+00:00][INFO ][plugins.ruleRegistry] Installing resources for index .alerts-observability.apm.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.646+00:00][INFO ][plugins.ruleRegistry] Installed resources for index .alerts-observability.metrics.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.651+00:00][INFO ][plugins.ruleRegistry] Installed resources for index .alerts-observability.logs.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.652+00:00][INFO ][plugins.ruleRegistry] Installed resources for index .alerts-observability.apm.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.652+00:00][INFO ][plugins.ruleRegistry] Installed resources for index .alerts-observability.uptime.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:31.653+00:00][INFO ][plugins.ruleRegistry] Installed resources for index .alerts-security.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:32.006+00:00][INFO ][plugins.fleet] Fleet setup completed
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:32.026+00:00][INFO ][plugins.securitySolution] Dependent plugin setup complete - Starting ManifestTask
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:32.046+00:00][INFO ][plugins.ruleRegistry] Installed resources for index .preview.alerts-security.alerts
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:34.935+00:00][INFO ][status] Kibana is now available (was degraded)
elastic_kibana.1.8m9zoqbkva7s@swarmn    | [2022-06-15T17:25:35.532+00:00][INFO ][plugins.ml] Task ML:saved-objects-sync-task: No ML saved objects in need of synchronization

docker-compose

version: '3.7'

services:
  setup:
    image: tuneitme/setup
    init: true
    volumes:
      - /mnt/setup:/state
    environment:
      - ELASTIC_PASSWORD=changeme
      - KIBANA_SYSTEM_PASSWORD=changeme
    networks:
      - caddy
    deploy:
      placement:
        constraints: [node.role == worker]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    depends_on:
      - setup
    volumes:
      - /mnt/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - /mnt/elastic:/usr/share/elasticsearch/data
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - ELASTIC_PASSWORD=changeme
    networks:
      - caddy
    deploy:
      placement:
        constraints: [node.role == manager]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
  kibana:
    image: docker.elastic.co/kibana/kibana:8.2.0
    volumes:
      - /mnt/config/kibana.yml:/usr/share/kibana/config/kibana.yml
      - /mnt/kibana:/usr/share/kibana/data
    ports:
      - "5601:5601"
    environment:
      - KIBANA_SYSTEM_PASSWORD=changeme
    networks:
      - caddy
    depends_on:
      - elasticsearch
    deploy:
      placement:
        constraints: [node.role == worker]
      replicas: 1
      update_config:
        parallelism: 2
        delay: 10s
      restart_policy:
        condition: on-failure
volumes:
  setup:
    driver: "local"
  elastic:
    driver: "local"
  kibana:
    driver: "local"
networks:
  caddy:
    external: true

kibana configuration

---
server.name: kibana
server.host: 0.0.0.0
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: kibana_system
elasticsearch.password: changeme
xpack.encryptedSavedObjects.encryptionKey: encryptionkey
xpack.reporting.encryptionKey: encryptionkey
xpack.security.encryptionKey: encryptionkey
xpack.reporting.roles.enabled: false

Elasticsearch configuration

---
cluster.name: "docker-cluster"
network.host: 0.0.0.0
xpack.license.self_generated.type: trial
xpack.security.enabled: true
4

Can someone help with it?

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.