Hi all,
I have an odd behavior with my Elasticsearch instance. I installed the latest 1/6 version on the clean server and configured it within Graylog. So I have only one virtual server in 1 cluster. But in the cluster state I see 2 nodes on the same IP address:
[root@hostname ~]# curl 'localhost:9200/_cat/nodes?v'
host ip heap.percent ram.percent load node.role master name
hostname.my.com 199.115.104.146 3 73 0.71 d * es_node_1
hostname.my.com 199.115.104.146 46 73 0.71 c - es_node_1
# curl -XGET 'http://127.0.0.1:9200/_cluster/health?pretty=true'
{
"cluster_name" : "es-my-cluster",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 2,
"number_of_data_nodes" : 1,
"active_primary_shards" : 1,
"active_shards" : 1,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0
}
# curl -XGET 'http://localhost:9200/_nodes?pretty=true'
{
"cluster_name" : "es-my-cluster",
"nodes" : {
"Jc9vyUdrT4uh1Flyecy-Xg" : {
"name" : "es_node_1",
"transport_address" : "inet[/X.X.X.146:9350]",
"host" : "hostname.my.com",
"ip" : "X.X.X.146",
"version" : "1.5.2",
"build" : "62ff986",
"attributes" : {
"client" : "true",
"data" : "false",
"master" : "false"
},
"settings" : {
"node" : {
"name" : "es_node_1",
"client" : "true",
"data" : "false",
"master" : "false"
},
"cluster" : {
"name" : "es-my-cluster"
},
"path" : {
"data" : "/var/lib/elasticsearch",
"logs" : "/var/log/elasticsearch",
"conf" : "/etc/elasticsearch"
},
"discovery" : {
"zen" : {
"ping" : {
"unicasts" : {
"hosts" : [ "127.0.0.1" ]
},
"multicast" : {
"enabled" : "false"
},
"unicast" : {
"hosts" : "127.0.0.1:9300"
}
}
},
"initial_state_timeout" : "3s"
},
"plugins" : {
"mandatory" : "graylog2-monitor"
},
"name" : "es_node_1",
"http" : {
"enabled" : "false"
},
"action" : {
"auto_create_index" : "false"
},
"client" : {
"type" : "node"
},
"bootstrap" : {
"mlockall" : "true"
},
"transport" : {
"tcp" : {
"port" : "9350"
}
}
},
"os" : {
"refresh_interval_in_millis" : 1000,
"available_processors" : 4,
"cpu" : {
"vendor" : "Intel",
"model" : "Xeon",
"mhz" : 2133,
"total_cores" : 32,
"total_sockets" : 1,
"cores_per_socket" : 32,
"cache_size_in_bytes" : 12288
},
"mem" : {
"total_in_bytes" : 8589934592
},
"swap" : {
"total_in_bytes" : 0
}
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 587,
"max_file_descriptors" : 4096,
"mlockall" : false
},
"jvm" : {
"pid" : 587,
"version" : "1.8.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.45-b02",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1436699558179,
"mem" : {
"heap_init_in_bytes" : 1073741824,
"heap_max_in_bytes" : 1020067840,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 1020067840
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ]
},
"thread_pool" : {
"percolate" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"listener" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "200"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 12,
"max" : 12,
"queue_size" : "1k"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"bulk" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "50"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"network" : {
"refresh_interval_in_millis" : 5000,
"primary_interface" : {
"address" : "X.X.X.146",
"name" : "eth0",
"mac_address" : "00:18:51:3E:C2:A5"
}
},
"transport" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0%0:9350]",
"publish_address" : "inet[/X.X.X.146:9350]",
"profiles" : { }
},
"plugins" : [ {
"name" : "graylog2-monitor",
"version" : "0.1.0",
"description" : "Monitors the current cluster state.",
"jvm" : true,
"site" : false
} ]
},
"Ohqfp4iaQnSTAWxJD6Pg8g" : {
"name" : "es_node_1",
"transport_address" : "inet[/X.X.X.146:9300]",
"host" : "hostname.my.com",
"ip" : "X.X.X.146",
"version" : "1.6.0",
"build" : "cdd3ac4",
"http_address" : "inet[/X.X.X.146:9200]",
"settings" : {
"pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
"path" : {
"conf" : "/etc/elasticsearch",
"data" : "/var/lib/elasticsearch",
"logs" : "/var/log/elasticsearch",
"home" : "/usr/share/elasticsearch"
},
"cluster" : {
"name" : "es-my-cluster"
},
"node" : {
"name" : "es_node_1"
},
"discovery" : {
"zen" : {
"ping" : {
"unicasts" : {
"hosts" : [ "127.0.0.1" ]
},
"multicast" : {
"enabled" : "false"
}
}
}
},
"name" : "es_node_1",
"client" : {
"type" : "node"
},
"foreground" : "yes",
"bootstrap" : {
"mlockall" : "true"
},
"config.ignore_system_properties" : "true",
"config" : "/etc/elasticsearch/elasticsearch.yml"
},
"os" : {
"refresh_interval_in_millis" : 1000,
"available_processors" : 4,
"cpu" : {
"vendor" : "Intel",
"model" : "Xeon",
"mhz" : 2133,
"total_cores" : 32,
"total_sockets" : 1,
"cores_per_socket" : 32,
"cache_size_in_bytes" : 12288
},
"mem" : {
"total_in_bytes" : 8589934592
},
"swap" : {
"total_in_bytes" : 0
}
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 1346,
"max_file_descriptors" : 65535,
"mlockall" : true
},
"jvm" : {
"pid" : 1346,
"version" : "1.8.0_45",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.45-b02",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1436701177529,
"mem" : {
"heap_init_in_bytes" : 4294967296,
"heap_max_in_bytes" : 4260102144,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 4260102144
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Compressed Class Space", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ]
},
"thread_pool" : {
"percolate" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"fetch_shard_started" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m",
"queue_size" : -1
},
"listener" : {
"type" : "fixed",
"min" : 2,
"max" : 2,
"queue_size" : -1
},
"index" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "200"
},
"refresh" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"suggest" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"generic" : {
"type" : "cached",
"keep_alive" : "30s",
"queue_size" : -1
},
"warmer" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"search" : {
"type" : "fixed",
"min" : 7,
"max" : 7,
"queue_size" : "1k"
},
"flush" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"optimize" : {
"type" : "fixed",
"min" : 1,
"max" : 1,
"queue_size" : -1
},
"fetch_shard_store" : {
"type" : "scaling",
"min" : 1,
"max" : 8,
"keep_alive" : "5m",
"queue_size" : -1
},
"management" : {
"type" : "scaling",
"min" : 1,
"max" : 5,
"keep_alive" : "5m",
"queue_size" : -1
},
"get" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "1k"
},
"merge" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
},
"bulk" : {
"type" : "fixed",
"min" : 4,
"max" : 4,
"queue_size" : "50"
},
"snapshot" : {
"type" : "scaling",
"min" : 1,
"max" : 2,
"keep_alive" : "5m",
"queue_size" : -1
}
},
"network" : {
"refresh_interval_in_millis" : 5000,
"primary_interface" : {
"address" : "X.X.X.146",
"name" : "eth0",
"mac_address" : "00:18:51:3E:C2:A5"
}
},
"transport" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0:9300]",
"publish_address" : "inet[/X.X.X.146:9300]",
"profiles" : { }
},
"http" : {
"bound_address" : "inet[/0:0:0:0:0:0:0:0:9200]",
"publish_address" : "inet[/X.X.X.146:9200]",
"max_content_length_in_bytes" : 104857600
},
"plugins" : [ ]
}
}
}
Is it normal? At the previous time I changed a cluster name and it was only 1 node. But now I see 2 again. Really strange. Only 1 'elasticsearch' process is running on the server:
ps ax | grep elas
1952 ? SLsl 0:21 /bin/java -Xms4g -Xmx4g -Djava.awt.headless=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:+DisableExplicitGC -Dfile.encoding=UTF-8 -Delasticsearch -Des.foreground=yes -Des.path.home=/usr/share/elasticsearch -cp :/usr/share/elasticsearch/lib/elasticsearch-1.6.0.jar:/usr/share/elasticsearch/lib/*:/usr/share/elasticsearch/lib/sigar/* -Des.pidfile=/var/run/elasticsearch/elasticsearch.pid -Des.default.path.home=/usr/share/elasticsearch -Des.default.path.logs=/var/log/elasticsearch -Des.default.path.data=/var/lib/elasticsearch -Des.default.config=/etc/elasticsearch/elasticsearch.yml -Des.default.path.conf=/etc/elasticsearch org.elasticsearch.bootstrap.Elasticsearch
[root@hostname ~]# netstat -nlp | grep \:9
tcp6 0 0 :::9200 :::* LISTEN 1952/java
tcp6 0 0 :::9300 :::* LISTEN 1952/java
tcp6 0 0 :::9350 :::* LISTEN 587/java
tcp6 0 0 :::9000 :::* LISTEN 26118/java