Ask you have not encountered under the ES operation of the file lock is lost, and can not lead to the problem, I encountered several times. Every time you see the file lock is lost and then the new index begins to go wrong
[2017-04-30T12:07:26,705][WARN ][o.e.e.NodeEnvironment ] [es_infra_bjza_1] lock assertion failed
java.nio.file.NoSuchFileException: /data/elasticsearch/data/es_infra_bjza/nodes/0/node.lock
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:55) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144) ~[?:?]
at sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99) ~[?:?]
at java.nio.file.Files.readAttributes(Files.java:1737) ~[?:1.8.0_111]
at org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid(NativeFSLockFactory.java:177) ~[lucene-core-6.3.0.jar:6.3.0 a66a44513ee8191e25b477372094bfa846450316 - shalin - 2016-11-02 19:47:11]
at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:901) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.env.NodeEnvironment.nodePaths(NodeEnvironment.java:753) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsProbe.stats(FsProbe.java:55) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService.stats(FsService.java:60) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService.access$200(FsService.java:33) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService$FsInfoCache.refresh(FsService.java:78) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService$FsInfoCache.refresh(FsService.java:67) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService.stats(FsService.java:55) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.node.service.NodeService.stats(NodeService.java:114) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:77) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:42) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:145) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:270) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:266) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.transport.TransportService$6.doRun(TransportService.java:577) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:527) [elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.1.1.jar:5.1.1]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_111]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_111]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_111]
[2017-04-30T12:07:26,706][DEBUG][o.e.a.a.c.n.s.TransportNodesStatsAction] [es_infra_bjza_1] failed to execute on node [-bZFv7AcSc6hK94oEmnTYA]
org.elasticsearch.transport.RemoteTransportException: [es_infra_bjza_1][10.120.6.246:9300][cluster:monitor/nodes/stats[n]]
Caused by: java.lang.IllegalStateException: environment is not locked
at org.elasticsearch.env.NodeEnvironment.assertEnvIsLocked(NodeEnvironment.java:904) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.env.NodeEnvironment.nodePaths(NodeEnvironment.java:753) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsProbe.stats(FsProbe.java:55) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService.stats(FsService.java:60) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService.access$200(FsService.java:33) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService$FsInfoCache.refresh(FsService.java:78) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService$FsInfoCache.refresh(FsService.java:67) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.monitor.fs.FsService.stats(FsService.java:55) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.node.service.NodeService.stats(NodeService.java:114) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:77) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:42) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:145) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:270) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:266) ~[elasticsearch-5.1.1.jar:5.1.1]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) ~[elasticsearch-5.1.1.jar:5.1.1]