2 nodes in the cluster while it should be 1


(Dmitriy S) #1

Hi all,

I have an odd behavior with my Elasticsearch instance. I installed the latest 1/6 version on the clean server and configured it within Graylog. So I have only one virtual server in 1 cluster. But in the cluster state I see 2 nodes on the same IP address:

[root@hostname ~]# curl 'localhost:9200/_cat/nodes?v'
host               ip              heap.percent ram.percent load node.role master name
hostname.my.com 199.115.104.146            3          73 0.71 d         *      es_node_1
hostname.my.com 199.115.104.146           46          73 0.71 c         -      es_node_1


# curl -XGET 'http://127.0.0.1:9200/_cluster/health?pretty=true'
{
  "cluster_name" : "es-my-cluster",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 2,
  "number_of_data_nodes" : 1,
  "active_primary_shards" : 1,
  "active_shards" : 1,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0
}

# curl -XGET 'http://localhost:9200/_nodes?pretty=true'
{
  "cluster_name" : "es-my-cluster",
  "nodes" : {
    "Jc9vyUdrT4uh1Flyecy-Xg" : {
      "name" : "es_node_1",
      "transport_address" : "inet[/X.X.X.146:9350]",
      "host" : "hostname.my.com",
      "ip" : "X.X.X.146",
      "version" : "1.5.2",
      "build" : "62ff986",
      "attributes" : {
        "client" : "true",
        "data" : "false",
        "master" : "false"
      },
      "settings" : {
        "node" : {
          "name" : "es_node_1",
          "client" : "true",
          "data" : "false",
          "master" : "false"
        },
        "cluster" : {
          "name" : "es-my-cluster"
        },
        "path" : {
          "data" : "/var/lib/elasticsearch",
          "logs" : "/var/log/elasticsearch",
          "conf" : "/etc/elasticsearch"
        },
        "discovery" : {
          "zen" : {
            "ping" : {
              "unicasts" : {
                "hosts" : [ "127.0.0.1" ]
              },
              "multicast" : {
                "enabled" : "false"
              },
              "unicast" : {
                "hosts" : "127.0.0.1:9300"
              }
            }
          },
          "initial_state_timeout" : "3s"
        },
        "plugins" : {
          "mandatory" : "graylog2-monitor"
        },
        "name" : "es_node_1",
        "http" : {
          "enabled" : "false"
        },
        "action" : {
          "auto_create_index" : "false"
        },
        "client" : {
          "type" : "node"
        },
        "bootstrap" : {
          "mlockall" : "true"
        },
        "transport" : {
          "tcp" : {
            "port" : "9350"
          }
        }
      },
      "os" : {
        "refresh_interval_in_millis" : 1000,
        "available_processors" : 4,
        "cpu" : {
          "vendor" : "Intel",
          "model" : "Xeon",
          "mhz" : 2133,
          "total_cores" : 32,
          "total_sockets" : 1,
          "cores_per_socket" : 32,
          "cache_size_in_bytes" : 12288
        },
        "mem" : {
          "total_in_bytes" : 8589934592
        },
        "swap" : {
          "total_in_bytes" : 0
        }
      },
      "process" : {
        "refresh_interval_in_millis" : 1000,
        "id" : 587,
        "max_file_descriptors" : 4096,
        "mlockall" : false
      },
      "jvm" : {
        "pid" : 587,
        "version" : "1.8.0_45",
        "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
        "vm_version" : "25.45-b02",
        "vm_vendor" : "Oracle Corporation",
        "start_time_in_millis" : 1436699558179,
        "mem" : {
          "heap_init_in_bytes" : 1073741824,
          "heap_max_in_bytes" : 1020067840,
          "non_heap_init_in_bytes" : 2555904,
          "non_heap_max_in_bytes" : 0,
          "direct_max_in_bytes" : 1020067840
        },
        "gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
        "memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ]
      },
      "thread_pool" : {
        "percolate" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "1k"
        },
        "listener" : {
          "type" : "fixed",
          "min" : 2,
          "max" : 2,
          "queue_size" : -1
        },
        "index" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "200"
        },
        "refresh" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "suggest" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "1k"
        },
        "generic" : {
          "type" : "cached",
          "keep_alive" : "30s",
          "queue_size" : -1
        },
        "warmer" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "search" : {
          "type" : "fixed",
          "min" : 12,
          "max" : 12,
          "queue_size" : "1k"
        },
        "flush" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "optimize" : {
          "type" : "fixed",
          "min" : 1,
          "max" : 1,
          "queue_size" : -1
        },
        "management" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 5,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "get" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "1k"
        },
        "merge" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "bulk" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "50"
        },
        "snapshot" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        }
      },
      "network" : {
        "refresh_interval_in_millis" : 5000,
        "primary_interface" : {
          "address" : "X.X.X.146",
          "name" : "eth0",
          "mac_address" : "00:18:51:3E:C2:A5"
        }
      },
      "transport" : {
        "bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9350]",
        "publish_address" : "inet[/X.X.X.146:9350]",
        "profiles" : { }
      },
      "plugins" : [ {
        "name" : "graylog2-monitor",
        "version" : "0.1.0",
        "description" : "Monitors the current cluster state.",
        "jvm" : true,
        "site" : false
      } ]
    },
    "Ohqfp4iaQnSTAWxJD6Pg8g" : {
      "name" : "es_node_1",
      "transport_address" : "inet[/X.X.X.146:9300]",
      "host" : "hostname.my.com",
      "ip" : "X.X.X.146",
      "version" : "1.6.0",
      "build" : "cdd3ac4",
      "http_address" : "inet[/X.X.X.146:9200]",
      "settings" : {
        "pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
        "path" : {
          "conf" : "/etc/elasticsearch",
          "data" : "/var/lib/elasticsearch",
          "logs" : "/var/log/elasticsearch",
          "home" : "/usr/share/elasticsearch"
        },
        "cluster" : {
          "name" : "es-my-cluster"
        },
        "node" : {
          "name" : "es_node_1"
        },
        "discovery" : {
          "zen" : {
            "ping" : {
              "unicasts" : {
                "hosts" : [ "127.0.0.1" ]
              },
              "multicast" : {
                "enabled" : "false"
              }
            }
          }
        },
        "name" : "es_node_1",
        "client" : {
          "type" : "node"
        },
        "foreground" : "yes",
        "bootstrap" : {
          "mlockall" : "true"
        },
        "config.ignore_system_properties" : "true",
        "config" : "/etc/elasticsearch/elasticsearch.yml"
      },
      "os" : {
        "refresh_interval_in_millis" : 1000,
        "available_processors" : 4,
        "cpu" : {
          "vendor" : "Intel",
          "model" : "Xeon",
          "mhz" : 2133,
          "total_cores" : 32,
          "total_sockets" : 1,
          "cores_per_socket" : 32,
          "cache_size_in_bytes" : 12288
        },
        "mem" : {
          "total_in_bytes" : 8589934592
        },
        "swap" : {
          "total_in_bytes" : 0
        }
      },
      "process" : {
        "refresh_interval_in_millis" : 1000,
        "id" : 1346,
        "max_file_descriptors" : 65535,
        "mlockall" : true
      },
      "jvm" : {
        "pid" : 1346,
        "version" : "1.8.0_45",
        "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
        "vm_version" : "25.45-b02",
        "vm_vendor" : "Oracle Corporation",
        "start_time_in_millis" : 1436701177529,
        "mem" : {
          "heap_init_in_bytes" : 4294967296,
          "heap_max_in_bytes" : 4260102144,
          "non_heap_init_in_bytes" : 2555904,
          "non_heap_max_in_bytes" : 0,
          "direct_max_in_bytes" : 4260102144
        },
        "gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
        "memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ]
      },
      "thread_pool" : {
        "percolate" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "1k"
        },
        "fetch_shard_started" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 8,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "listener" : {
          "type" : "fixed",
          "min" : 2,
          "max" : 2,
          "queue_size" : -1
        },
        "index" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "200"
        },
        "refresh" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "suggest" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "1k"
        },
        "generic" : {
          "type" : "cached",
          "keep_alive" : "30s",
          "queue_size" : -1
        },
        "warmer" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "search" : {
          "type" : "fixed",
          "min" : 7,
          "max" : 7,
          "queue_size" : "1k"
        },
        "flush" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "optimize" : {
          "type" : "fixed",
          "min" : 1,
          "max" : 1,
          "queue_size" : -1
        },
        "fetch_shard_store" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 8,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "management" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 5,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "get" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "1k"
        },
        "merge" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        },
        "bulk" : {
          "type" : "fixed",
          "min" : 4,
          "max" : 4,
          "queue_size" : "50"
        },
        "snapshot" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 2,
          "keep_alive" : "5m",
          "queue_size" : -1
        }
      },
      "network" : {
        "refresh_interval_in_millis" : 5000,
        "primary_interface" : {
          "address" : "X.X.X.146",
          "name" : "eth0",
          "mac_address" : "00:18:51:3E:C2:A5"
        }
      },
      "transport" : {
        "bound_address" : "inet[/0:0:0:0:0:0:0:0:9300]",
        "publish_address" : "inet[/X.X.X.146:9300]",
        "profiles" : { }
      },
      "http" : {
        "bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]",
        "publish_address" : "inet[/X.X.X.146:9200]",
        "max_content_length_in_bytes" : 104857600
      },
      "plugins" : [ ]
    }
  }
}

Is it normal? At the previous time I changed a cluster name and it was only 1 node. But now I see 2 again. Really strange. Only 1 'elasticsearch' process is running on the server:

 ps ax | grep elas
 1952 ?        SLsl   0:21 /bin/java -Xms4g -Xmx4g -Djava.awt.headless=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:+DisableExplicitGC -Dfile.encoding=UTF-8 -Delasticsearch -Des.foreground=yes -Des.path.home=/usr/share/elasticsearch -cp :/usr/share/elasticsearch/lib/elasticsearch-1.6.0.jar:/usr/share/elasticsearch/lib/*:/usr/share/elasticsearch/lib/sigar/* -Des.pidfile=/var/run/elasticsearch/elasticsearch.pid -Des.default.path.home=/usr/share/elasticsearch -Des.default.path.logs=/var/log/elasticsearch -Des.default.path.data=/var/lib/elasticsearch -Des.default.config=/etc/elasticsearch/elasticsearch.yml -Des.default.path.conf=/etc/elasticsearch org.elasticsearch.bootstrap.Elasticsearch

[root@hostname ~]# netstat -nlp | grep \:9
tcp6       0      0 :::9200                 :::*                    LISTEN      1952/java
tcp6       0      0 :::9300                 :::*                    LISTEN      1952/java
tcp6       0      0 :::9350                 :::*                    LISTEN      587/java
tcp6       0      0 :::9000                 :::*                    LISTEN      26118/java

(Dmitriy S) #2

Forgot to show logs:

[2015-07-12 17:39:42,422][INFO ][node                     ] [es_node_1] starting ...
[2015-07-12 17:39:42,630][INFO ][transport                ] [es_node_1] bound_address {inet[/0:0:0:0:0:0:0:0:9300]}, publish_address {inet[/X.X.X.146:9300]}
[2015-07-12 17:39:42,648][INFO ][discovery                ] [es_node_1] es-my-cluster/Ohqfp4iaQnSTAWxJD6Pg8g
[2015-07-12 17:39:45,680][INFO ][cluster.service          ] [es_node_1] new_master [es_node_1][Ohqfp4iaQnSTAWxJD6Pg8g][hostname.my.com][inet[/X.X.X.146:9300]], reason: zen-disco-join (elected_as_
master)
[2015-07-12 17:39:45,685][INFO ][cluster.service          ] [es_node_1] added {[es_node_1][Jc9vyUdrT4uh1Flyecy-Xg][hostname.my.com][inet[/X.X.X.146:9350]]{client=true, data=false, master=false},}
, reason: zen-disco-receive(join from node[[es_node_1][Jc9vyUdrT4uh1Flyecy-Xg][hostname.my.com][inet[/X.X.X.146:9350]]{client=true, data=false, master=false}])
[2015-07-12 17:39:45,723][INFO ][http                     ] [es_node_1] bound_address {inet[/0:0:0:0:0:0:0:0:9200]}, publish_address {inet[/X.X.X.146:9200]}
[2015-07-12 17:39:45,723][INFO ][node                     ] [es_node_1] started
[2015-07-12 17:39:45,775][INFO ][gateway                  ] [es_node_1] recovered [1] indices into cluster_state

(Dmitriy S) #3

I figured out why it happen. Couple of days ago I renamed node.name in my elasticsearch.yml

# curl -XGET 'http://localhost:9200/_cat/nodes?v&h=host,node.role,master,name'
host               node.role master name
hostname.my.com c         -      es_node_1
hostname.my.com d         *      es_node_01

So what is the correct way to rename node? Should I remove the old node before renaming?


(Dmitriy S) #4

I really have no idea how to remove this fake host. Tried to shutdown it but it is still present:


(Dmitriy S) #5

Finally after server reboot it is showing 1 node:

curl -XGET 'http://localhost:9200/_cat/nodes?v&h=host,node.role,master,name'
host               node.role master name

hostname.my.com d         *      Antiphon the Overseer

(system) #6