Rolling upgrade did not work as documented, I have 1 node, elastic log shows this warning below. Any other info needed pls let me know. We are sort of stuck right now and i'm so new to this that i'm not sure how to proceed.
Also, now old logs have been ingested and after 7 days are gzipped, if i need to re-parse old logs they have to be uncompressed.
[2018-08-07T13:50:12,153][WARN ][r.suppressed ] path: /.kibana/_search, params: {ignore_unavailable=true, index=.kibana, filter_path=aggregations.types.buckets}
org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
Here are all the curl commands and their responses:
-
curl "localhost:9200/.kibana/_search?q=type:index-pattern&size=9999&_source=false&pretty"
{
"error" : {
"root_cause" : [ ],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [ ]
},
"status" : 503
} -
curl elastic_ip:9200/_cluster/health?pretty
{
"cluster_name" : "elasticsearch",
"status" : "red",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 1306,
"active_shards" : 1306,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 1309,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 49.942638623326964
} -
curl localhost:9200/_cat/templates?pretty
.monitoring-beats [.monitoring-beats-6-] 0 6020099
.monitoring-alerts [.monitoring-alerts-6] 0 6020099
.ml-state [.ml-state] 0 6030199
security_audit_log [.security_audit_log] 1000
.watches [.watches*] 2147483647
.ml-notifications [.ml-notifications] 0 6030199
.triggered_watches [.triggered_watches*] 2147483647
logstash-index-template [.logstash] 0
.monitoring-es [.monitoring-es-6-] 0 6020099
.ml-anomalies- [.ml-anomalies-] 0 6030199
security-index-template [.security-] 1000
.monitoring-kibana [.monitoring-kibana-6-] 0 6020099
.monitoring-logstash [.monitoring-logstash-6-] 0 6020099
logstash [logstash-] 0 60001
.watch-history-7 [.watcher-history-7*] 2147483647
kibana_index_template:.kibana [.kibana] 0
.ml-meta [.ml-meta] 0 6030199 -
curl localhost:9200/_cat/indices?pretty (tons of these)
yellow open logstash-2018.05.15 pfJad-p5SreuojqCD6--tA 5 1 4173661 0 1.3gb 1.3gb
yellow open logstash-2018.01.31 WFx6NRT8TFqNhZSoMmQVyw 5 1 4067388 0 1.2gb 1.2gb
yellow open logstash-2018.02.02 18cueP5aQ2y0Cyo_qJYP0A 5 1 4585440 0 1.4gb 1.4gb
yellow open logstash-2017.12.01 -U4ribxcSUms08FrkOJlUQ 5 1 21758379 0 5.6gb 5.6gb -
curl localhost:9200/logstash-2018.08.06/_mapping
{"logstash-2018.08.06":{"mappings":{"default":{"dynamic_templates":[{"message_field":{"path_match":"message","match_mapping_type":"string","mapping":{"norms":false,"type":"text"}}},{"string_fields":{"match":"","match_mapping_type":"string","mapping":{"fields":{"keyword":{"ignore_above":256,"type":"keyword"}},"norms":false,"type":"text"}}}],"properties":{"@timestamp":{"type":"date"},"@version":{"type":"keyword"},"geoip":{"dynamic":"true","properties":{"ip":{"type":"ip"},"latitude":{"type":"half_float"},"location":{"type":"geo_point"},"longitude":{"type":"half_float"}}}}},"doc":{"dynamic_templates":[{"message_field":{"path_match":"message","match_mapping_type":"string","mapping":{"norms":false,"type":"text"}}},{"string_fields":{"match":"","match_mapping_type":"string","mapping":{"fields":{"keyword":{"ignore_above":256,"type":"keyword"}},"norms":false,"type":"text"}}}],"properties":{"@timestamp":{"type":"date"},"@version":{"type":"keyword"},"appname":{"type":"text","norms":false,"fields":{"keyword":{"type":"keyword","ignore_above":256}}},"date2":{"type":"text","norms":false,"fields":{"keyword":{"type":"keyword","ignore_above":256}}},"geoip":{"dynamic":"true","properties":{"ip": -
curl -s 'localhost:9200/_cat/allocation?v'
shards disk.indices disk.used disk.avail disk.total disk.percent host ip node
1306 655.7gb 3.3tb 4.1tb 7.4tb 45 172.20.0.19 172.20.0.19 X4gBF-q
1309 UNASSIGNED
8.curl -XGET 'localhost:9200/_cluster/health?pretty'
{
"cluster_name" : "elasticsearch",
"status" : "red",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 1306,
"active_shards" : 1306,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 1309,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 49.942638623326964
} -
curl -XGET 'http://localhost:9200/_settings?pretty' (one for every log)
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{
"logstash-2018.05.15" : {
"settings" : {
"index" : {
"refresh_interval" : "5s",
"number_of_shards" : "5",
"provided_name" : "logstash-2018.05.15",
"creation_date" : "1526342400295",
"number_of_replicas" : "1",
"uuid" : "pfJad-p5SreuojqCD6--tA",
"version" : {
"created" : "6000099",
"upgraded" : "6030199"
}
}
}
},
"logstash-2018.02.02" : {
"settings" : {
"index" : {
"refresh_interval" : "5s",
"number_of_shards" : "5",
"provided_name" : "logstash-2018.02.02",
"creation_date" : "1517529600318",
"number_of_replicas" : "1",
"uuid" : "18cueP5aQ2y0Cyo_qJYP0A",
"version" : {
"created" : "6000099",
"upgraded" : "6030199"
}
}
}
}