Elasticsearch 5.6.3 upgrade to 6.0 Node will not start "failed to read local state"

Elastic Stack Elasticsearch
1

We've updated our Dev system and our monitoring stack to 6.0 with minimal issues. When we attempt to update production node they refuse to start. Attempted the upgrade on two nodes now. Both nodes result in the same error, failed to read state file. I rolled the first node back to 5.6.3.

[2017-11-18T15:53:47,133][INFO ][o.e.p.PluginsService ] [ES-data-warm01] loaded plugin [x-pack]
[2017-11-18T15:53:49,544][DEBUG][o.e.a.ActionModule ] Using REST wrapper from plugin org.elasticsearch.xpack.XPackPlugin
[2017-11-18T15:53:50,813][INFO ][o.e.x.m.j.p.l.CppLogMessageHandler] [controller/13410] [Main.cc@128] controller (64 bit): Version 6.0.0 (Build 8e6ab35cf803a2) Copyright (c) 2017 Elasticsearch BV
[2017-11-18T15:53:50,854][INFO ][o.e.d.DiscoveryModule ] [ES-data-warm01] using discovery type [zen]
[2017-11-18T15:53:52,028][ERROR][o.e.g.GatewayMetaState ] [ES-data-warm01] failed to read local state, exiting...
org.elasticsearch.ElasticsearchException: java.io.IOException: failed to read [id:1355, legacy:false, file:/data/sda1/nodes/0/_state/global-1355.st]
at org.elasticsearch.ExceptionsHelper.maybeThrowRuntimeAndSuppress(ExceptionsHelper.java:150) ~[elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:334) ~[elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.gateway.MetaStateService.loadGlobalState(MetaStateService.java:113) ~[elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.gateway.MetaStateService.loadFullState(MetaStateService.java:57) ~[elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.gateway.GatewayMetaState.(GatewayMetaState.java:93) [elasticsearch-6.0.0.jar:6.0.0]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) [?:1.8.0_131]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) [?:1.8.0_131]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) [?:1.8.0_131]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) [?:1.8.0_131]
at org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:49) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:116) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:825) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:50) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:116) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:825) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:50) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:191) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:183) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:818) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:183) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:173) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:161) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:96) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.Guice.createInjector(Guice.java:96) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.Guice.createInjector(Guice.java:70) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.common.inject.ModulesBuilder.createInjector(ModulesBuilder.java:42) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.node.Node.(Node.java:492) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.node.Node.(Node.java:245) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Bootstrap$5.(Bootstrap.java:212) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:212) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:322) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:130) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:121) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:69) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:134) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.cli.Command.main(Command.java:90) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92) [elasticsearch-6.0.0.jar:6.0.0]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:85) [elasticsearch-6.0.0.jar:6.0.0]

Any thoughts on what could be causing this problem.

2

Did you run the Upgrade Assistant before the upgrade?

3

Yes I did. It didn't indicate any problems that needed to be fixed before upgrading.

4

Can you do an ls -l /data/sda1/nodes/0/_state/global-1355.st and post your config?

Also, please format your code using the </> button, or markdown style back ticks as it makes things much easier to read :slight_smile:

5

I've rolled that node back just a few minutes ago. But here is the results. I'm going to try another node in a few minutes. One of the 3 masters.

/etc/elasticsearch# ls -l /data/sda1/nodes/0/_state/global-1355.st
ls: cannot access '/data/sda1/nodes/0/_state/global-1355.st': No such file or directory

/etc/elasticsearch# cat elasticsearch.yml

cluster.name: #############

node.name: ES-data-warm01
# Rack local awareness for shard placement. Prevents cluster failure due to single rack failure
node.attr.rack: r1
# Node performance type
node.attr.box_type: warm
#
node.master: false
node.data: true
node.ingest: false
# ----------------------------------- Paths ------------------------------------
path.data: /data/sda1,/data/sdb1,/data/sdc1,/data/sdd1,/data/sde1,/data/sdf1
path.logs: /var/log/elasticsearch
# ----------------------------------- Memory -----------------------------------
# Lock the memory on startup:
bootstrap.memory_lock: true
# ---------------------------------- Network -----------------------------------
network.host: XXX.XX.XX.16
http.port: 9200
# --------------------------------- Discovery ----------------------------------
discovery.zen.ping.unicast.hosts: ["ES-data-hot01","ES-data-hot02","ES-data-hot03","ES-data-hot04","ES-data-hot05","ES-data-warm01","ES-data-warm02","ES-data-warm03","ES-master01","ES-master02","ES-master03"]
discovery.zen.minimum_master_nodes: 2
# ---------------------------------- Gateway -----------------------------------
gateway.recover_after_nodes: 2
# ---------------------------------- Various -----------------------------------
# Require explicit names when deleting indices:
#action.destructive_requires_name: true
# ---------------------------------- x-pack security --------------------------
xpack.security.audit.enabled: true

xpack.ssl.key:                     /etc/elasticsearch/x-pack/warm1.key
xpack.ssl.certificate:             /etc/elasticsearch/x-pack/warm1.crt
xpack.ssl.certificate_authorities: [ "/etc/elasticsearch/x-pack/ca.crt" ]
xpack.security.transport.ssl.enabled: true
xpack.security.http.ssl.enabled: true

#--------- Xpack Monitoring ---------------
#action.auto_create_index: .security,.monitoring*,.watches,.triggered_watches,.watcher-history*,.ml*

xpack.monitoring.exporters:
  id1:
    type: http
    host: ["https://st-vm-monitor02.coxhealth.com:9200","https://st-vm-monitor01.coxhealth.com:9200"]
    ssl:
      certificate_authorities: [ "/etc/elasticsearch/x-pack/ca.crt" ]
    auth:
      username: remote_monitor
      password: XXXXXXXXXXXXXXXXXXXXXXXX
6

Same results on one of the master nodes. Checked and the state file listed in the error is there.

In both previous cases I can remove x-pack, reinstall 5.6.3, install x-pack and remove the 6.0 keypass.seed file and restart the service. It rejoins the cluster and everything looks perfect.

ls -l /var/lib/elasticsearch/nodes/0/_state/global-1377.st
-rw-r--r-- 1 elasticsearch elasticsearch 84740 Nov 18 13:40 /var/lib/elasticsearch/nodes/0/_state/global-1377.st


2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [lang-expression]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [lang-mustache]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [lang-painless]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [parent-join]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [percolator]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [reindex]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [repository-url]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [transport-netty4]
[2017-11-18T18:03:25,525][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded module [tribe]
[2017-11-18T18:03:25,526][INFO ][o.e.p.PluginsService     ] [ES-master03] loaded plugin [x-pack]
[2017-11-18T18:03:27,976][DEBUG][o.e.a.ActionModule       ] Using REST wrapper from plugin org.elasticsearch.xpack.XPackPlugin
[2017-11-18T18:03:29,418][INFO ][o.e.d.DiscoveryModule    ] [ES-master03] using discovery type [zen]
[2017-11-18T18:03:29,421][INFO ][o.e.x.m.j.p.l.CppLogMessageHandler] [controller/27781] [Main.cc@128] controller (64 bit): Version 6.0.0 (Build 8e6ab35cf803a2) Copyright (c) 2017 Elasticsearch BV
[2017-11-18T18:03:31,419][ERROR][o.e.g.GatewayMetaState   ] [ES-master03] failed to read local state, exiting...
org.elasticsearch.ElasticsearchException: java.io.IOException: failed to read [id:1377, legacy:false, file:/var/lib/elasticsearch/nodes/0/_state/global-1377.st]
7

I've re-run the Upgrade Assistant. Below are the only issues it lists. One is a false positive. The path settings are in place. The other shouldn't be a problem. Everything else shows green.

Default path settings are removed
This issue must be resolved to upgrade. Read Documentation
Details: nodes with settings: [ES-master01]

one or more templates use deprecated mapping settings
Resolving this issue is advised but not required to upgrade. Read Documentation
Details: {.monitoring-es=[Coercion of boolean fields], security-index-template-v6=[Coercion of boolean fields], .monitoring-kibana-2=[Coercion of boolean fields], metricbeat=[Coercion of boolean fields], .monitoring-logstash-2=[Coercion of boolean fields], .watch-history-6=[Coercion of boolean fields], .monitoring-kibana=[Coercion of boolean fields], security-index-template=[Coercion of boolean fields], .monitoring-logstash=[Coercion of boolean fields], logstash=[The _all meta field is disabled by default on indices created in 6.0, The [include_in_all] mapping parameter is now disallowed], .monitoring-es-2=[Coercion of boolean fields], packetbeat=[Coercion of boolean fields], .ml-anomalies-=[Coercion of boolean fields]}
8

Can you share the state file privately with us? (i.e. /var/lib/elasticsearch/nodes/0/_state/global-1377.st). If so, please e-mail it to firstname@elastic.co where firstname is yannick. This will help us figure out what's wrong. Thanks.

9

Thanks for sharing the file, @kwslavens. I've attached the full stack trace below. It looks like there is an ML query that needs to be upgraded before moving to 6.0 (possibly not taken into account by the upgrade assistant). I'll reach out to the ML folks for help.

[2017-11-20T16:19:10,011][ERROR][o.e.g.GatewayMetaState   ] [VyA8xgf] failed to read local state, exiting...
org.elasticsearch.ElasticsearchException: java.io.IOException: failed to read [id:1377, legacy:false, file:/Users/ywelsch/Downloads/elasticsearch-6.0.0/data/nodes/0/_state/global-1377.st]
	at org.elasticsearch.ExceptionsHelper.maybeThrowRuntimeAndSuppress(ExceptionsHelper.java:150) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:334) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaStateService.loadGlobalState(MetaStateService.java:113) ~[elasticsearch-6.0.0.jar:6.0.0]
...
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92) [elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:85) [elasticsearch-6.0.0.jar:6.0.0]
Caused by: java.io.IOException: failed to read [id:1377, legacy:false, file:/Users/ywelsch/Downloads/elasticsearch-6.0.0/data/nodes/0/_state/global-1377.st]
	at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:327) ~[elasticsearch-6.0.0.jar:6.0.0]
	... 46 more
Caused by: org.elasticsearch.common.ParsingException: [ml_metadata] failed to parse field [datafeeds]
	at org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:316) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.ObjectParser.parseArray(ObjectParser.java:308) ~[elasticsearch-6.0.0.jar:6.0.0]
	...
	at org.elasticsearch.xpack.ml.MachineLearning.lambda$getNamedXContent$0(MachineLearning.java:287) ~[?:?]
	at org.elasticsearch.common.xcontent.NamedXContentRegistry$Entry.lambda$new$0(NamedXContentRegistry.java:68) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.NamedXContentRegistry.parseNamedObject(NamedXContentRegistry.java:143) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.support.AbstractXContentParser.namedObject(AbstractXContentParser.java:394) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.cluster.metadata.MetaData$Builder.fromXContent(MetaData.java:1085) ~[elasticsearch-6.0.0.jar:6.0.0]
	...
	... 46 more
Caused by: org.elasticsearch.common.ParsingException: [datafeed_config] failed to parse field [query]
	at org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:316) ~[elasticsearch-6.0.0.jar:6.0.0]
...
	at org.elasticsearch.xpack.ml.MachineLearning.lambda$getNamedXContent$0(MachineLearning.java:287) ~[?:?]
	at 
...
	at org.elasticsearch.gateway.MetaDataStateFormat.read(MetaDataStateFormat.java:202) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:322) ~[elasticsearch-6.0.0.jar:6.0.0]
	... 46 more
Caused by: org.elasticsearch.common.ParsingException: [match] query does not support [type]
	at org.elasticsearch.index.query.MatchQueryBuilder.fromXContent(MatchQueryBuilder.java:494) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.search.SearchModule.lambda$registerQuery$10(SearchModule.java:753) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.NamedXContentRegistry.parseNamedObject(NamedXContentRegistry.java:143) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.support.AbstractXContentParser.namedObject(AbstractXContentParser.java:394) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder(AbstractQueryBuilder.java:314) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.index.query.BoolQueryBuilder.fromXContent(BoolQueryBuilder.java:319) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.search.SearchModule.lambda$registerQuery$10(SearchModule.java:753) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.NamedXContentRegistry.parseNamedObject(NamedXContentRegistry.java:143) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.support.AbstractXContentParser.namedObject(AbstractXContentParser.java:394) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder(AbstractQueryBuilder.java:314) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.xpack.ml.datafeed.DatafeedConfig.lambda$static$2(DatafeedConfig.java:109) ~[?:?]
	at ...
	at org.elasticsearch.xpack.ml.MlMetadata.lambda$static$3(MlMetadata.java:71) ~[?:?]
	at org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$null$7(AbstractObjectParser.java:186) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.AbstractObjectParser.parseArray(AbstractObjectParser.java:230) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.AbstractObjectParser.lambda$declareObjectArray$8(AbstractObjectParser.java:186) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.ObjectParser.lambda$declareField$1(ObjectParser.java:214) ~[elasticsearch-6.0.0.jar:6.0.0]
...
org.elasticsearch.xpack.ml.MachineLearning.lambda$getNamedXContent$0(MachineLearning.java:287) ~[?:?]
	at org.elasticsearch.common.xcontent.NamedXContentRegistry$Entry.lambda$new$0(NamedXContentRegistry.java:68) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.NamedXContentRegistry.parseNamedObject(NamedXContentRegistry.java:143) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.support.AbstractXContentParser.namedObject(AbstractXContentParser.java:394) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.cluster.metadata.MetaData$Builder.fromXContent(MetaData.java:1085) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.cluster.metadata.MetaData$2.fromXContent(MetaData.java:1128) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.cluster.metadata.MetaData$2.fromXContent(MetaData.java:1119) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaDataStateFormat.read(MetaDataStateFormat.java:202) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:322) ~[elasticsearch-6.0.0.jar:6.0.0]
	... 46 more
10

Thanks, I've opened a ticket on this. I'm assuming if I were to delete our 2 ML jobs then that would allow the upgrade to process and complete.

11

We found the offending ML job and deleted it. We had to role back the node to get it to update to the current state file. Then repeated the upgrade and this time the node starts as expected!

Thanks for the quick help its very appreciated!

12

Having the same issue upgrading, but no longer have ML license. Created ML jobs when on trial, now on basic and don't have access to ML to delete the job. Any way to list and delete everything ML-related through API in this state?

13

Hi Mike,

If you can't access ML through Kibana then use the REST APIs to delete jobs and datafeeds

14

I get three parsing exceptions, which I assume are related to ML. It looks like someone on our side deleted the .ml related indexes manually. None of the ML APIs show datafeeds or jobs in the list, give an index missing error. Are things still recoverable to go to 6.0.0?

[2017-11-21T03:13:12,933][INFO ][o.e.n.Node               ] [elasticsearch_LNX02] initializing ...
[2017-11-21T03:13:13,252][INFO ][o.e.e.NodeEnvironment    ] [elasticsearch_LNX02] using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/mapper/B00208--LNX01--vg-root)]], net usable_space [10.6gb], net total_space [59.9gb], types [ext4]
[2017-11-21T03:13:13,253][INFO ][o.e.e.NodeEnvironment    ] [elasticsearch_LNX02] heap size [11.9gb], compressed ordinary object pointers [true]

.........

[2017-11-21T03:13:53,719][WARN ][o.e.d.c.ParseField       ] Deprecated field [split_on_whitespace] used, replaced by [This setting is ignored, the parser always splits on operator]
[2017-11-21T03:13:53,725][ERROR][o.e.g.GatewayMetaState   ] [elasticsearch_LNX02] failed to read local state, exiting...
org.elasticsearch.ElasticsearchException: java.io.IOException: failed to read [id:176, legacy:false, file:/usr/share/elasticsearch/data/nodes/0/_state/global-176.st]
	at org.elasticsearch.ExceptionsHelper.maybeThrowRuntimeAndSuppress(ExceptionsHelper.java:150) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:334) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.gateway.MetaStateService.loadGlobalState(MetaStateService.java:113) ~[elasticsearch-6.0.0.jar:6.0.0]

.........

	... 46 more
Caused by: org.elasticsearch.common.ParsingException: [ml_metadata] failed to parse field [datafeeds]
	at org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:316) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.ObjectParser.parseArray(ObjectParser.java:308) ~[elasticsearch-6.0.0.jar:6.0.0]

.........

	... 46 more
Caused by: org.elasticsearch.common.ParsingException: [datafeed_config] failed to parse field [query]
	at org.elasticsearch.common.xcontent.ObjectParser.parseValue(ObjectParser.java:316) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.common.xcontent.ObjectParser.parseSub(ObjectParser.java:325) ~[elasticsearch-6.0.0.jar:6.0.0]

.........

	... 46 more
Caused by: org.elasticsearch.common.ParsingException: [match] query does not support [type]
	at org.elasticsearch.index.query.MatchQueryBuilder.fromXContent(MatchQueryBuilder.java:494) ~[elasticsearch-6.0.0.jar:6.0.0]
	at org.elasticsearch.search.SearchModule.lambda$registerQuery$10(SearchModule.java:753) ~[elasticsearch-6.0.0.jar:6.0.0]

.........

	... 46 more
15

Hi @dkyle,

My cluster is down, cant access via REST API or kibana. So, have another way to fix it ?
Reinstall last version before upgrade? Delete ML job and update ? ?
I upgrade from 5.6.2 to 6.0.0

16

Hi @tatdat ,

Unfortunately you will have to roll back the upgrade to 5.6.2 and delete the ML job then you can upgrade normally.

17

Thanks @dkyle,
I'm trying to downgrade from 6.0.0 to 5.6.2 via install rpm package but it's conflict with version 6.0.0.
So maybe i need backup config and remove ES 6.0 and install 5.6.2 ?

18

@mjett

Did you manage to fix the problem? Were you able to delete the ML jobs and datafeeds after reverting to 5.6?

19

Yep, i managed them. if rollback success, i 'll delete ML via cerebro plugin

1 Like
20

Hi @dkyle,

I rollback my cluster to 5.6.2 but can't start cluster ... I got error. After that few second, node will be down.

[2017-11-24T18:01:39,171][INFO ][o.e.n.Node               ] [Elastic-03] started
[2017-11-24T18:01:41,557][INFO ][o.e.c.s.ClusterSettings  ] [Elastic-03] updating [indices.store.throttle.max_bytes_per_sec] from [0b] to [100mb]
[2017-11-24T18:01:45,953][ERROR][o.e.x.m.c.i.IndexRecoveryCollector] [Elastic-03] collector [index-recovery] failed to collect data
org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];
        at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:114) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction.checkGlobalBlock(TransportRecoveryAction.java:52) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$AsyncAction.<init>(TransportBroadcastByNodeAction.java:256) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:234) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction.doExecute(TransportBroadcastByNodeAction.java:79) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:408) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.client.FilterClient.doExecute(FilterClient.java:67) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.xpack.security.InternalClient.doExecute(InternalClient.java:91) ~[?:?]
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:408) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1256) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.action.ActionRequestBuilder.get(ActionRequestBuilder.java:69) ~[elasticsearch-5.6.2.jar:5.6.2]
        at org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryCollector.doCollect(IndexRecoveryCollector.java:65) ~[x-pack-5.6.2.jar:5.6.2]
        at org.elasticsearch.xpack.monitoring.collector.Collector.collect(Collector.java:83) [x-pack-5.6.2.jar:5.6.2]
        at org.elasticsearch.xpack.monitoring.MonitoringService$MonitoringExecution$1.doRun(MonitoringService.java:195) [x-pack-5.6.2.jar:5.6.2]
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.6.2.jar:5.6.2]