Hi,
This is Production system with around 2.5 TB of Data.
Elasticsearch version is 5.0.2.
The Server has rebooted somehow and we are still doing an RCA on that.
Elasticsearch is not coming up.
Please suggest the needful.
[2017-03-22T21:14:15,666][INFO ][o.e.n.Node ] [node-rpt1] initializing ...
[2017-03-22T21:14:15,793][INFO ][o.e.e.NodeEnvironment ] [node-rpt1] using [2] data paths, mounts [[/newdata1 (/dev/mapper/newdata1), /data1 (/dev/mapper/data2)]], net usable_space [4.9tb], net total_space [9.3tb], spins? [possibly], types [ext4, ext3]
[2017-03-22T21:14:15,794][INFO ][o.e.e.NodeEnvironment ] [node-rpt1] heap size [30.8gb], compressed ordinary object pointers [true]
[2017-03-22T21:14:16,643][INFO ][o.e.n.Node ] [node-rpt1] version[5.0.0], pid[28829], build[253032b/2016-10-26T04:37:51.531Z], OS[Linux/2.6.32-220.el6.x86_64/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13]
[2017-03-22T21:14:17,538][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [aggs-matrix-stats]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [ingest-common]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [lang-expression]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [lang-groovy]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [lang-mustache]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [lang-painless]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [percolator]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [reindex]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [transport-netty3]
[2017-03-22T21:14:17,539][INFO ][o.e.p.PluginsService ] [node-rpt1] loaded module [transport-netty4]
[2017-03-22T21:14:17,540][INFO ][o.e.p.PluginsService ] [node-rpt1] no plugins loaded
[2017-03-22T21:14:19,976][WARN ][o.e.c.u.IndexFolderUpgrader] [/data1/esdata/nodes/0/indices/ctVB6IYsSAKjCTAhY28p4A] no index state found - ignoring
[2017-03-22T21:14:19,988][ERROR][o.e.g.GatewayMetaState ] [node-rpt1] failed to read local state, exiting...
org.elasticsearch.ElasticsearchException: java.io.IOException: failed to read [id:11, legacy:false, file:/data1/esdata/nodes/0/indices/Gy6M6ijSSwqyZO15jl7RmQ/_state/state-11.st]
at org.elasticsearch.ExceptionsHelper.maybeThrowRuntimeAndSuppress(ExceptionsHelper.java:150) ~[elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:333) ~[elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.util.IndexFolderUpgrader.upgrade(IndexFolderUpgrader.java:89) ~[elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.util.IndexFolderUpgrader.upgradeIndicesIfNeeded(IndexFolderUpgrader.java:127) ~[elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.gateway.GatewayMetaState.(GatewayMetaState.java:89) [elasticsearch-5.0.0.jar:5.0.0]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) [?:1.8.0_101]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) [?:1.8.0_101]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) [?:1.8.0_101]
at org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:49) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:116) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:825) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:50) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:116) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:825) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:50) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:191) [elasticsearch-5.0.0.jar:5.0.0]
at org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:183) [elasticsearch-5.0.0.jar:5.0.0]