Logstash Monitoring

There are no index formed for while trying to monitor logstash using legacy settings, please help, below are the logs of logstash.

[2021-08-27T15:04:57,444][INFO ][logstash.runner          ] Log4j configuration path used is: /etc/logstash/log4j2.properties
[2021-08-27T15:04:57,456][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"7.13.4", "jruby.version"=>"jruby 9.2.16.0 (2.5.7) 2021-03-03 f82228dc32 OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +indy +jit [linux-x86_64]"}
[2021-08-27T15:04:57,462][DEBUG][logstash.modules.scaffold] Found module {:module_name=>"fb_apache", :directory=>"/usr/share/logstash/modules/fb_apache/configuration"}
[2021-08-27T15:04:57,463][DEBUG][logstash.plugins.registry] Adding plugin to the registry {:name=>"fb_apache", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x3cc1990 @directory="/usr/share/logstash/modules/fb_apache/configuration", @module_name="fb_apache", @kibana_version_parts=["6", "0", "0"]>}
[2021-08-27T15:04:57,465][DEBUG][logstash.modules.scaffold] Found module {:module_name=>"netflow", :directory=>"/usr/share/logstash/modules/netflow/configuration"}
[2021-08-27T15:04:57,465][DEBUG][logstash.plugins.registry] Adding plugin to the registry {:name=>"netflow", :type=>:modules, :class=>#<LogStash::Modules::Scaffold:0x6e609e8a @directory="/usr/share/logstash/modules/netflow/configuration", @module_name="netflow", @kibana_version_parts=["6", "0", "0"]>}
[2021-08-27T15:04:57,899][DEBUG][logstash.runner          ] -------- Logstash Settings (* means modified) ---------
[2021-08-27T15:04:57,900][DEBUG][logstash.runner          ] node.name: "0.0.0.0"
[2021-08-27T15:04:57,900][DEBUG][logstash.runner          ] *path.data: "/var/lib/logstash" (default: "/usr/share/logstash/data")
[2021-08-27T15:04:57,900][DEBUG][logstash.runner          ] modules.cli: <Java::OrgLogstashUtil::ModulesSettingArray:1 []>
[2021-08-27T15:04:57,901][DEBUG][logstash.runner          ] modules: []
[2021-08-27T15:04:57,901][DEBUG][logstash.runner          ] modules_list: []
[2021-08-27T15:04:57,901][DEBUG][logstash.runner          ] modules_variable_list: []
[2021-08-27T15:04:57,901][DEBUG][logstash.runner          ] modules_setup: false
[2021-08-27T15:04:57,901][DEBUG][logstash.runner          ] config.test_and_exit: false
[2021-08-27T15:04:57,902][DEBUG][logstash.runner          ] *config.reload.automatic: true (default: false)
[2021-08-27T15:04:57,902][DEBUG][logstash.runner          ] config.reload.interval: #<Java::OrgLogstashUtil::TimeValue:0x7f888518>
[2021-08-27T15:04:57,902][DEBUG][logstash.runner          ] config.support_escapes: false
[2021-08-27T15:04:57,902][DEBUG][logstash.runner          ] config.field_reference.parser: "STRICT"
[2021-08-27T15:04:57,902][DEBUG][logstash.runner          ] metric.collect: true
[2021-08-27T15:04:57,903][DEBUG][logstash.runner          ] pipeline.id: "main"
[2021-08-27T15:04:57,903][DEBUG][logstash.runner          ] pipeline.system: false
[2021-08-27T15:04:57,903][DEBUG][logstash.runner          ] pipeline.workers: 2
[2021-08-27T15:04:57,903][DEBUG][logstash.runner          ] pipeline.batch.size: 125
[2021-08-27T15:04:57,903][DEBUG][logstash.runner          ] pipeline.batch.delay: 50
[2021-08-27T15:04:57,904][DEBUG][logstash.runner          ] pipeline.unsafe_shutdown: false
[2021-08-27T15:04:57,904][DEBUG][logstash.runner          ] pipeline.java_execution: true
[2021-08-27T15:04:57,904][DEBUG][logstash.runner          ] pipeline.reloadable: true
[2021-08-27T15:04:57,904][DEBUG][logstash.runner          ] pipeline.plugin_classloaders: false
[2021-08-27T15:04:57,904][DEBUG][logstash.runner          ] pipeline.separate_logs: false
[2021-08-27T15:04:57,905][DEBUG][logstash.runner          ] pipeline.ordered: "auto"
[2021-08-27T15:04:57,905][DEBUG][logstash.runner          ] pipeline.ecs_compatibility: "disabled"
[2021-08-27T15:04:57,905][DEBUG][logstash.runner          ] path.plugins: []
[2021-08-27T15:04:57,905][DEBUG][logstash.runner          ] config.debug: false
[2021-08-27T15:04:57,905][DEBUG][logstash.runner          ] *log.level: "debug" (default: "info")
[2021-08-27T15:04:57,906][DEBUG][logstash.runner          ] version: false
[2021-08-27T15:04:57,906][DEBUG][logstash.runner          ] help: false
[2021-08-27T15:04:57,906][DEBUG][logstash.runner          ] log.format: "plain"
[2021-08-27T15:04:57,906][DEBUG][logstash.runner          ] http.enabled: true
[2021-08-27T15:04:57,906][DEBUG][logstash.runner          ] http.host: "127.0.0.1"
[2021-08-27T15:04:57,907][DEBUG][logstash.runner          ] http.port: 9600..9700
[2021-08-27T15:04:57,907][DEBUG][logstash.runner          ] http.environment: "production"
[2021-08-27T15:04:57,907][DEBUG][logstash.runner          ] queue.type: "memory"
[2021-08-27T15:04:57,907][DEBUG][logstash.runner          ] queue.drain: false
[2021-08-27T15:04:57,907][DEBUG][logstash.runner          ] queue.page_capacity: 67108864
[2021-08-27T15:04:57,907][DEBUG][logstash.runner          ] queue.max_bytes: 1073741824
[2021-08-27T15:04:57,908][DEBUG][logstash.runner          ] queue.max_events: 0
[2021-08-27T15:04:57,908][DEBUG][logstash.runner          ] queue.checkpoint.acks: 1024
[2021-08-27T15:04:57,908][DEBUG][logstash.runner          ] queue.checkpoint.writes: 1024
[2021-08-27T15:04:57,908][DEBUG][logstash.runner          ] queue.checkpoint.interval: 1000
[2021-08-27T15:04:57,909][DEBUG][logstash.runner          ] queue.checkpoint.retry: false
[2021-08-27T15:04:57,909][DEBUG][logstash.runner          ] dead_letter_queue.enable: false
[2021-08-27T15:04:57,909][DEBUG][logstash.runner          ] dead_letter_queue.max_bytes: 1073741824
[2021-08-27T15:04:57,909][DEBUG][logstash.runner          ] dead_letter_queue.flush_interval: 5000
[2021-08-27T15:04:57,909][DEBUG][logstash.runner          ] slowlog.threshold.warn: #<Java::OrgLogstashUtil::TimeValue:0x7ab9ef6c>
[2021-08-27T15:04:57,910][DEBUG][logstash.runner          ] slowlog.threshold.info: #<Java::OrgLogstashUtil::TimeValue:0x6ba1a7fd>
[2021-08-27T15:04:57,910][DEBUG][logstash.runner          ] slowlog.threshold.debug: #<Java::OrgLogstashUtil::TimeValue:0xf35f81d>
[2021-08-27T15:04:57,910][DEBUG][logstash.runner          ] slowlog.threshold.trace: #<Java::OrgLogstashUtil::TimeValue:0x15db982e>
[2021-08-27T15:04:57,910][DEBUG][logstash.runner          ] keystore.classname: "org.logstash.secret.store.backend.JavaKeyStore"
[2021-08-27T15:04:57,910][DEBUG][logstash.runner          ] *keystore.file: "/etc/logstash/logstash.keystore" (default: "/usr/share/logstash/config/logstash.keystore")
[2021-08-27T15:04:57,911][DEBUG][logstash.runner          ] *path.queue: "/var/lib/logstash/queue" (default: "/usr/share/logstash/data/queue")
[2021-08-27T15:04:57,911][DEBUG][logstash.runner          ] *path.dead_letter_queue: "/var/lib/logstash/dead_letter_queue" (default: "/usr/share/logstash/data/dead_letter_queue")
[2021-08-27T15:04:57,911][DEBUG][logstash.runner          ] *path.settings: "/etc/logstash" (default: "/usr/share/logstash/config")
[2021-08-27T15:04:57,911][DEBUG][logstash.runner          ] *path.logs: "/var/log/logstash" (default: "/usr/share/logstash/logs")
[2021-08-27T15:04:57,912][DEBUG][logstash.runner          ] xpack.management.enabled: false
[2021-08-27T15:04:57,912][DEBUG][logstash.runner          ] xpack.management.logstash.poll_interval: #<Java::OrgLogstashUtil::TimeValue:0x1af6ad68>
[2021-08-27T15:04:57,912][DEBUG][logstash.runner          ] xpack.management.pipeline.id: ["main"]
[2021-08-27T15:04:57,912][DEBUG][logstash.runner          ] xpack.management.elasticsearch.username: "logstash_system"
[2021-08-27T15:04:57,912][DEBUG][logstash.runner          ] xpack.management.elasticsearch.hosts: ["https://localhost:9200"]
[2021-08-27T15:04:57,913][DEBUG][logstash.runner          ] xpack.management.elasticsearch.ssl.verification_mode: "certificate"
[2021-08-27T15:04:57,913][DEBUG][logstash.runner          ] xpack.management.elasticsearch.sniffing: false
[2021-08-27T15:04:57,913][DEBUG][logstash.runner          ] *xpack.monitoring.enabled: true (default: false)
[2021-08-27T15:04:57,913][DEBUG][logstash.runner          ] *xpack.monitoring.elasticsearch.hosts: ["http://10.12.9.50:9200", "http://10.12.9.47:9200"] (default: ["http://localhost:9200"])
[2021-08-27T15:04:57,913][DEBUG][logstash.runner          ] *xpack.monitoring.collection.interval: #<Java::OrgLogstashUtil::TimeValue:0x6378d7a1> (default: #<Java::OrgLogstashUtil::TimeValue:0x5fe8810d>)
[2021-08-27T15:04:57,914][DEBUG][logstash.runner          ] xpack.monitoring.collection.timeout_interval: #<Java::OrgLogstashUtil::TimeValue:0x5a4736b8>
[2021-08-27T15:04:57,914][DEBUG][logstash.runner          ] xpack.monitoring.elasticsearch.username: "logstash_system"
[2021-08-27T15:04:57,914][DEBUG][logstash.runner          ] *xpack.monitoring.elasticsearch.password: "logstash123"
[2021-08-27T15:04:57,914][DEBUG][logstash.runner          ] xpack.monitoring.elasticsearch.ssl.verification_mode: "certificate"
[2021-08-27T15:04:57,914][DEBUG][logstash.runner          ] *xpack.monitoring.elasticsearch.sniffing: true (default: false)
[2021-08-27T15:04:57,915][DEBUG][logstash.runner          ] xpack.monitoring.collection.pipeline.details.enabled: true
[2021-08-27T15:04:57,916][DEBUG][logstash.runner          ] xpack.monitoring.collection.config.enabled: true
[2021-08-27T15:04:57,916][DEBUG][logstash.runner          ] monitoring.enabled: false
[2021-08-27T15:04:57,916][DEBUG][logstash.runner          ] monitoring.elasticsearch.hosts: ["http://localhost:9200"]
[2021-08-27T15:04:57,917][DEBUG][logstash.runner          ] monitoring.collection.interval: #<Java::OrgLogstashUtil::TimeValue:0x2de83c2c>
[2021-08-27T15:04:57,917][DEBUG][logstash.runner          ] monitoring.collection.timeout_interval: #<Java::OrgLogstashUtil::TimeValue:0x67e28dad>
[2021-08-27T15:04:57,917][DEBUG][logstash.runner          ] monitoring.elasticsearch.username: "logstash_system"
[2021-08-27T15:04:57,917][DEBUG][logstash.runner          ] monitoring.elasticsearch.ssl.verification_mode: "certificate"
[2021-08-27T15:04:57,919][DEBUG][logstash.runner          ] monitoring.elasticsearch.sniffing: false
[2021-08-27T15:04:57,919][DEBUG][logstash.runner          ] monitoring.collection.pipeline.details.enabled: true
[2021-08-27T15:04:57,919][DEBUG][logstash.runner          ] monitoring.collection.config.enabled: true
[2021-08-27T15:04:57,920][DEBUG][logstash.runner          ] node.uuid: ""
[2021-08-27T15:04:57,921][DEBUG][logstash.runner          ] --------------- Logstash Settings 

Now, the curl output is below:,

[root@Test-Logstash-01 logstash]# curl -u logstash_system:logstash123 -X GET "http://10.12.9.50:9200"
{
  "name" : "elk-coord-02",
  "cluster_name" : "bhuwan-elk-cluster",
  "cluster_uuid" : "xtfZXJbuS8G0ohlD0Zx6ww",
  "version" : {
    "number" : "7.14.0",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "dd5a0a2acaa2045ff9624f3729fc8a6f40835aa1",
    "build_date" : "2021-07-29T20:49:32.864135063Z",
    "build_snapshot" : false,
    "lucene_version" : "8.9.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}
[root@Test-Logstash-01 logstash]# curl -u logstash_system:logstash123 -X GET "http://10.12.9.47:9200"
{
  "name" : "elk-coord-01",
  "cluster_name" : "bhuwan-elk-cluster",
  "cluster_uuid" : "xtfZXJbuS8G0ohlD0Zx6ww",
  "version" : {
    "number" : "7.14.0",
    "build_flavor" : "default",
    "build_type" : "rpm",
    "build_hash" : "dd5a0a2acaa2045ff9624f3729fc8a6f40835aa1",
    "build_date" : "2021-07-29T20:49:32.864135063Z",
    "build_snapshot" : false,
    "lucene_version" : "8.9.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}
[root@Test-Logstash-01 logstash]# 

The output of:

POST .monitoring-logstash-*/_search
{
  "size": 0,
  "aggs": {
    "ids": {
      "terms": {
        "field": "logstash_stats.logstash.uuid",
        "size": 10
      }
    }
  }
}

is below:

{
  "took" : 632,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 256,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "ids" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "d8d24420-597b-431e-8de1-b0cc80c3ff37",
          "doc_count" : 196
        }
      ]
    }
  }
}

Please kindly suggest what am I missing or what is the issue