Flood storage Disk Watermark exceeded error

Elastic Stack Elasticsearch
1

I am getting the following error

Elasticsearch error: cluster_block_exception [FORBIDDEN/12/index read-only / allow delete (api)], flood stage disk watermark exceeded

Even though I have three data nodes each with 1TB storage. But still, I am getting this error. The total storage make up by all the indexes and their replica is less than 100 GB. Why I am getting this error?

For the time being, I deleted the previous indexes and make this API call and things are working fine but how to fix it permanently.

$ curl -XPUT -H "Content-Type: application/json" https://[YOUR_ELASTICSEARCH_ENDPOINT]:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'

Below is the storage status of all my data nodes.

image
image838×314 10.5 KB

image

image

Will be really thankful for your help

2

Welcome to our community! :smiley:
Please don't post pictures of text, they are difficult to read, impossible to search and replicate (if it's code), and some people may not be even able to see them :slight_smile:

What is the output from the _cluster/stats?pretty&human API?

3

@warkolm will take care regarding the image thing. Thank you for your feedback.

Below is the response for above API request

{
    "_nodes": {
        "total": 7,
        "successful": 7,
        "failed": 0
    },
    "cluster_name": "elasticsearch-cluster",
    "cluster_uuid": "idkidkidk",
    "timestamp": 1617788216638,
    "status": "green",
    "indices": {
        "count": 24,
        "shards": {
            "total": 52,
            "primaries": 26,
            "replication": 1.0,
            "index": {
                "shards": {
                    "min": 2,
                    "max": 6,
                    "avg": 2.1666666666666665
                },
                "primaries": {
                    "min": 1,
                    "max": 3,
                    "avg": 1.0833333333333333
                },
                "replication": {
                    "min": 1.0,
                    "max": 1.0,
                    "avg": 1.0
                }
            }
        },
        "docs": {
            "count": 33626258,
            "deleted": 1147924
        },
        "store": {
            "size": "36.1gb",
            "size_in_bytes": 38762394094
        },
        "fielddata": {
            "memory_size": "3.1kb",
            "memory_size_in_bytes": 3248,
            "evictions": 0
        },
        "query_cache": {
            "memory_size": "22.3mb",
            "memory_size_in_bytes": 23432992,
            "total_count": 58115,
            "hit_count": 10335,
            "miss_count": 47780,
            "cache_size": 531,
            "cache_count": 1980,
            "evictions": 1449
        },
        "completion": {
            "size": "0b",
            "size_in_bytes": 0
        },
        "segments": {
            "count": 356,
            "memory": "36.4mb",
            "memory_in_bytes": 38179752,
            "terms_memory": "17.4mb",
            "terms_memory_in_bytes": 18336832,
            "stored_fields_memory": "17.9mb",
            "stored_fields_memory_in_bytes": 18830992,
            "term_vectors_memory": "0b",
            "term_vectors_memory_in_bytes": 0,
            "norms_memory": "15kb",
            "norms_memory_in_bytes": 15424,
            "points_memory": "0b",
            "points_memory_in_bytes": 0,
            "doc_values_memory": "973.1kb",
            "doc_values_memory_in_bytes": 996504,
            "index_writer_memory": "370.8mb",
            "index_writer_memory_in_bytes": 388823312,
            "version_map_memory": "35.3mb",
            "version_map_memory_in_bytes": 37113284,
            "fixed_bit_set": "856kb",
            "fixed_bit_set_memory_in_bytes": 876600,
            "max_unsafe_auto_id_timestamp": 1617753609029,
            "file_sizes": {}
        }
    },
    "nodes": {
        "count": {
            "total": 7,
            "coordinating_only": 0,
            "data": 3,
            "ingest": 7,
            "master": 3,
            "ml": 7,
            "voting_only": 0
        },
        "versions": [
            "7.6.2"
        ],
        "os": {
            "available_processors": 28,
            "allocated_processors": 28,
            "names": [
                {
                    "name": "Linux",
                    "count": 7
                }
            ],
            "pretty_names": [
                {
                    "pretty_name": "CentOS Linux 7 (Core)",
                    "count": 5
                },
                {
                    "pretty_name": "CentOS Linux 8 (Core)",
                    "count": 2
                }
            ],
            "mem": {
                "total": "61.3gb",
                "total_in_bytes": 65848094720,
                "free": "2.6gb",
                "free_in_bytes": 2890088448,
                "used": "58.6gb",
                "used_in_bytes": 62958006272,
                "free_percent": 4,
                "used_percent": 96
            }
        },
        "process": {
            "cpu": {
                "percent": 3
            },
            "open_file_descriptors": {
                "min": 359,
                "max": 553,
                "avg": 445
            }
        },
        "jvm": {
            "max_uptime": "343d",
            "max_uptime_in_millis": 29638730662,
            "versions": [
                {
                    "version": "13.0.2",
                    "vm_name": "OpenJDK 64-Bit Server VM",
                    "vm_version": "13.0.2+8",
                    "vm_vendor": "AdoptOpenJDK",
                    "bundled_jdk": true,
                    "using_bundled_jdk": true,
                    "count": 7
                }
            ],
            "mem": {
                "heap_used": "14.2gb",
                "heap_used_in_bytes": 15250386360,
                "heap_max": "31.7gb",
                "heap_max_in_bytes": 34115682304
            },
            "threads": 484
        },
        "fs": {
            "total": "332.9gb",
            "total_in_bytes": 357519785984,
            "free": "257.9gb",
            "free_in_bytes": 276981882880,
            "available": "257.9gb",
            "available_in_bytes": 276981882880
        },
        "plugins": [],
        "network_types": {
            "transport_types": {
                "security4": 7
            },
            "http_types": {
                "security4": 7
            }
        },
        "discovery_types": {
            "zen": 7
        },
        "packaging_types": [
            {
                "flavor": "default",
                "type": "rpm",
                "count": 7
            }
        ],
        "ingest": {
            "number_of_pipelines": 3,
            "processor_stats": {
                "date": {
                    "count": 114547188,
                    "failed": 0,
                    "current": 0,
                    "time": "30.4m",
                    "time_in_millis": 1827848
                },
                "geoip": {
                    "count": 343641564,
                    "failed": 750534,
                    "current": 0,
                    "time": "6.8h",
                    "time_in_millis": 24647790
                },
                "gsub": {
                    "count": 0,
                    "failed": 0,
                    "current": 0,
                    "time": "0s",
                    "time_in_millis": 0
                },
                "json": {
                    "count": 114548008,
                    "failed": 820,
                    "current": 0,
                    "time": "22.4m",
                    "time_in_millis": 1347927
                },
                "remove": {
                    "count": 1718207820,
                    "failed": 0,
                    "current": 0,
                    "time": "29.3m",
                    "time_in_millis": 1759563
                },
                "script": {
                    "count": 0,
                    "failed": 0,
                    "current": 0,
                    "time": "0s",
                    "time_in_millis": 0
                }
            }
        }
    }
}
4

In my elastic search cluster, I tried to have two dedicated master node, two nodes which will acts as both ingest and coordinating node and three dedicated data nodes one of which is also master enabled node.

5

What is the output of the cat nodes API? Can you verify that the dedicated master and ingest/coordination nodes are not by mistake also defined as data nodes?

6

@Christian_Dahlqvist Thank you for your response.

This is the response of the API call

IP 58 89  0 0.00 0.01 0.05 dil  - data-node-2
IP 55 98  0 0.00 0.01 0.05 ilm  - master-node-1
IP 55 98  0 0.00 0.01 0.05 ilm  * master-node-2
IP 38 93  0 0.00 0.01 0.05 dilm - data-node-3
IP 22 85 13 1.06 1.09 1.06 dil  - data-node-1
IP 59 97 26 1.03 1.15 1.17 il   - coordinating-ingest-node-2
IP 62 96  2 0.14 0.05 0.06 il   - coordinating-ingest-node-1
7

That looks fine. Not sure what is going on.

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.