Hi Team,
I am using Elasticsearch 6.4.0 and my beats are sending log/ event directly to ES. But when i am trying to create a Machine Learning job its giving me below error.
Elasticsearch output:
[2018-09-12T17:33:40,521][INFO ][o.e.x.m.j.p.a.AutodetectProcessManager] [J14-npm] Opening job [testml]
[2018-09-12T17:33:40,534][INFO ][o.e.x.m.j.p.a.AutodetectProcessManager] [J14-npm] [testml] Loading model snapshot [N/A], job latest_record_timestamp [N/A]
[2018-09-12T17:33:40,542][ERROR][o.e.x.m.j.p.a.NativeAutodetectProcessFactory] Failed to launch autodetect for job testml
[2018-09-12T17:33:40,589][WARN ][r.suppressed ] path: /_xpack/ml/anomaly_detectors/testml/_open, params: {job_id=testml}
org.elasticsearch.ElasticsearchException: Unexpected job state [failed] while waiting for job to be opened
at org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper.serverError(ExceptionsHelper.java:34) ~[?:?]
at org.elasticsearch.xpack.ml.action.TransportOpenJobAction$JobPredicate.test(TransportOpenJobAction.java:814) ~[?:?]
at org.elasticsearch.xpack.ml.action.TransportOpenJobAction$JobPredicate.test(TransportOpenJobAction.java:774) ~[?:?]
at org.elasticsearch.persistent.PersistentTasksService.lambda$waitForPersistentTaskCondition$2(PersistentTasksService.java:169) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.clusterChanged(ClusterStateObserver.java:186) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateListeners$7(ClusterApplierService.java:506) ~[elasticsearch-6.4.0.jar:6.4.0]
at java.util.concurrent.ConcurrentHashMap$KeySpliterator.forEachRemaining(ConcurrentHashMap.java:3527) [?:1.8.0_181]
at java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:743) [?:1.8.0_181]
at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580) [?:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
[2018-09-12T17:33:40,596][WARN ][o.e.p.PersistentTasksNodeService] [J14-npm] task job-testml failed with an exception
org.elasticsearch.ElasticsearchException: Failed to launch autodetect for job testml
at org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper.serverError(ExceptionsHelper.java:38) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.NativeAutodetectProcessFactory.createNativeProcess(NativeAutodetectProcessFactory.java:104) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.NativeAutodetectProcessFactory.createAutodetectProcess(NativeAutodetectProcessFactory.java:60) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.create(AutodetectProcessManager.java:504) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.createProcessAndSetRunning(AutodetectProcessManager.java:458) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager.access$800(AutodetectProcessManager.java:90) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager$2.doRun(AutodetectProcessManager.java:427) ~[?:?]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723) ~[elasticsearch-6.4.0.jar:6.4.0]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.4.0.jar:6.4.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
Caused by: java.nio.file.NoSuchFileException: /tmp/elasticsearch.W0dkkJp9/limitconfig774444647943567679.conf
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_181]
at java.nio.file.Files.createFile(Files.java:632) ~[?:1.8.0_181]
at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) ~[?:1.8.0_181]
at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) ~[?:1.8.0_181]
at java.nio.file.Files.createTempFile(Files.java:852) ~[?:1.8.0_181]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder.buildLimits(AutodetectBuilder.java:114) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder.build(AutodetectBuilder.java:103) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.NativeAutodetectProcessFactory.createNativeProcess(NativeAutodetectProcessFactory.java:99) ~[?:?]
... 10 more
Caused by: java.nio.file.NoSuchFileException: /tmp/elasticsearch.W0dkkJp9/limitconfig774444647943567679.conf
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_181]
at java.nio.file.Files.createFile(Files.java:632) ~[?:1.8.0_181]
at java.nio.file.TempFileHelper.create(TempFileHelper.java:138) ~[?:1.8.0_181]
at java.nio.file.TempFileHelper.createTempFile(TempFileHelper.java:161) ~[?:1.8.0_181]
at java.nio.file.Files.createTempFile(Files.java:852) ~[?:1.8.0_181]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder.buildLimits(AutodetectBuilder.java:114) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectBuilder.build(AutodetectBuilder.java:103) ~[?:?]
at org.elasticsearch.xpack.ml.job.process.autodetect.NativeAutodetectProcessFactory.createNativeProcess(NativeAutodetectProcessFactory.java:99) ~[?:?]
... 10 more
[2018-09-12T17:33:40,838][INFO ][o.e.x.m.a.TransportPutDatafeedAction] [J14-npm] Created datafeed [datafeed-testml]