Elastic does not reindex all indicies while using a wildcard

Elastic Stack Elasticsearch
1

Hi!

I'm trying to reindex our indicies for them to use updated mappings.

Here is the query:

    {
        "source": {
            "index": "cards_*"
        },
       "dest": {
            "index": "cards_temp"
       },
      "script": {
            "lang": "painless",
            "source": "ctx._index = 'cards_' + (ctx._index.substring('cards_'.length(), ctx._index.length())) + '_temp'"
        }
    }

We have about 1500 of them in our test env.

I'm doing a status check like this:

/_tasks?actions=*reindex&detailed&group_by=none

And after the response is empty I expected for all of indicies to be done with and their respective _temp "copies" to be created.

But after a rather short wait the task is finished but only 178 of _temp indicies where created.

Is there a way to make elastic reindex all of them?

PS: I've checked all the original indicies for their size and number of docs (and doc minus deleted) just in case - still the number should be much higher

2

Upon further investigation, this is the problem:

"cause": {
       "type": "process_cluster_event_timeout_exception",
       "reason": "failed to process cluster event (create-index [mms_offer_cards_1171_temp], cause [auto(bulk api)]) within 1m"
},

Is there any way to avoid it?

3

What is the full output of the cluster stats API? Might you have hit the cluster shard limit?

4 
{
    "_nodes": {
        "total": 5,
        "successful": 5,
        "failed": 0
    },
    "cluster_name": "elastic_aux",
    "cluster_uuid": "OgAYQCZbRwy1wBLQf5R4Ew",
    "timestamp": 1614939861435,
    "status": "green",
    "indices": {
        "count": 1906,
        "shards": {
            "total": 3868,
            "primaries": 1934,
            "replication": 1.0,
            "index": {
                "shards": {
                    "min": 2,
                    "max": 15,
                    "avg": 2.0293809024134313
                },
                "primaries": {
                    "min": 1,
                    "max": 5,
                    "avg": 1.0146904512067156
                },
                "replication": {
                    "min": 0.0,
                    "max": 2.0,
                    "avg": 1.0
                }
            }
        },
        "docs": {
            "count": 53673167,
            "deleted": 10321319
        },
        "store": {
            "size_in_bytes": 167656194612
        },
        "fielddata": {
            "memory_size_in_bytes": 832760,
            "evictions": 0
        },
        "query_cache": {
            "memory_size_in_bytes": 30015457,
            "total_count": 305918503,
            "hit_count": 7334867,
            "miss_count": 298583636,
            "cache_size": 73927,
            "cache_count": 206116,
            "evictions": 132189
        },
        "completion": {
            "size_in_bytes": 0
        },
        "segments": {
            "count": 10168,
            "memory_in_bytes": 570332099,
            "terms_memory_in_bytes": 477533559,
            "stored_fields_memory_in_bytes": 64666384,
            "term_vectors_memory_in_bytes": 0,
            "norms_memory_in_bytes": 7598144,
            "points_memory_in_bytes": 5269148,
            "doc_values_memory_in_bytes": 15264864,
            "index_writer_memory_in_bytes": 315058024,
            "version_map_memory_in_bytes": 29925380,
            "fixed_bit_set_memory_in_bytes": 1546336,
            "max_unsafe_auto_id_timestamp": -1,
            "file_sizes": {}
        }
    },
    "nodes": {
        "count": {
            "total": 5,
            "data": 5,
            "coordinating_only": 0,
            "master": 5,
            "ingest": 5
        },
        "versions": [
            "6.5.4"
        ],
        "os": {
            "available_processors": 30,
            "allocated_processors": 38,
            "names": [
                {
                    "name": "Linux",
                    "count": 5
                }
            ],
            "mem": {
                "total_in_bytes": 168619151360,
                "free_in_bytes": 2732904448,
                "used_in_bytes": 165886246912,
                "free_percent": 2,
                "used_percent": 98
            }
        },
        "process": {
            "cpu": {
                "percent": 17
            },
            "open_file_descriptors": {
                "min": 3028,
                "max": 3111,
                "avg": 3059
            }
        },
        "jvm": {
            "max_uptime_in_millis": 9415251481,
            "versions": [
                {
                    "version": "11.0.1",
                    "vm_name": "OpenJDK 64-Bit Server VM",
                    "vm_version": "11.0.1+13",
                    "vm_vendor": "Oracle Corporation",
                    "count": 5
                }
            ],
            "mem": {
                "heap_used_in_bytes": 41037453352,
                "heap_max_in_bytes": 85899345920
            },
            "threads": 1106
        },
        "fs": {
            "total_in_bytes": 472895528960,
            "free_in_bytes": 283427737600,
            "available_in_bytes": 259185725440
        },
        "plugins": [
            {
                "name": "ingest-user-agent",
                "version": "6.5.4",
                "elasticsearch_version": "6.5.4",
                "java_version": "1.8",
                "description": "Ingest processor that extracts information from a user agent",
                "classname": "org.elasticsearch.ingest.useragent.IngestUserAgentPlugin",
                "extended_plugins": [],
                "has_native_controller": false
            },
            {
                "name": "analysis-phonetic",
                "version": "6.5.4",
                "elasticsearch_version": "6.5.4",
                "java_version": "1.8",
                "description": "The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch.",
                "classname": "org.elasticsearch.plugin.analysis.AnalysisPhoneticPlugin",
                "extended_plugins": [],
                "has_native_controller": false
            },
            {
                "name": "ingest-geoip",
                "version": "6.5.4",
                "elasticsearch_version": "6.5.4",
                "java_version": "1.8",
                "description": "Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database",
                "classname": "org.elasticsearch.ingest.geoip.IngestGeoIpPlugin",
                "extended_plugins": [],
                "has_native_controller": false
            },
            {
                "name": "analysis-icu",
                "version": "6.5.4",
                "elasticsearch_version": "6.5.4",
                "java_version": "1.8",
                "description": "The ICU Analysis plugin integrates Lucene ICU module into elasticsearch, adding ICU relates analysis components.",
                "classname": "org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin",
                "extended_plugins": [],
                "has_native_controller": false
            }
        ],
        "network_types": {
            "transport_types": {
                "netty4": 5
            },
            "http_types": {
                "netty4": 5
            }
        }
    }
}

Seems like we are low on mem

5

You have a lot of very small shards, which is very inefficient. I would recommend you reduce this dramatically and would expect that to help with reindexing as well as memory usage. It does however not look like you are hitting the shard limits at the moment so I am not sure why reindexing is failing.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.