Blocked by: [FORBIDDEN/8/index write (api)]; Reindex

Elastic Stack Elasticsearch
1

Hey team,
I was reindexing one of my index but suddenly it stopped because shard size reaches to 60gb and it rollover to new index.I am getting an error now i cannot write after index get rollover

"cause" : {
          "type" : "cluster_block_exception",
          "reason" : "index [log-wlb-sysmon-2021.10.04-000015] blocked by: [FORBIDDEN/8/index write (api)];"
        },
        "status" : 403

Following are steps that i did to have this reindexing problem.
1.

PUT %3Clog-wlb-sysmon-%7Bnow%2Fd%7D-000015%3E 
{
"aliases": {
   "log-wlb-sysmon": {
      "is_write_index": false      
   }
 }
 , "settings": {
   "number_of_replicas": 0,
   "refresh_interval": "-1",
   "index.lifecycle.parse_origination_date":true,
  "index.lifecycle.name":"winlogbeat_sysmon_policy",
  "index.lifecycle.rollover_alias":"log-wlb-sysmon"
 }
}

POST _aliases
{
  "actions": [
    {
      "add": {
        "index": "log-wlb-sysmon-2021.09.23-000019",
        "alias": "log-wlb-sysmon",
        "is_write_index": false
      }
    },
    {
      "add": {
        "index": "log-wlb-sysmon-2021.10.04-000015",
        "alias": "log-wlb-sysmon",
        "is_write_index": true
      }
    }
    
  ]
}

POST _reindex?slices=5&wait_for_completion=false&max_docs=102154111
{
   "source": {
    "index": "log-wlb-sysmon-2021.08.16-000015"
    , "size": 3000
    
  }
  , "dest": {
    "index": "log-wlb-sysmon-2021.10.04-000015"
  }
}

GET _tasks/x8nHc1dDSgiTh3G9rD8raQ:712742016

{
  "completed" : true,
  "task" : {
    "node" : "x8nHc1dDSgiTh3G9rD8raQ",
    "id" : 712742016,
    "type" : "transport",
    "action" : "indices:data/write/reindex",
    "status" : {
      "total" : 102147101,
      "updated" : 0,
      "created" : 73800000,
      "deleted" : 0,
      "batches" : 24605,
      "version_conflicts" : 0,
      "noops" : 0,
      "retries" : {
        "bulk" : 0,
        "search" : 0
      },
      "throttled_millis" : 0,
      "requests_per_second" : -1.0,
      "throttled_until_millis" : 0,
      "slices" : [
        {
          "slice_id" : 0,
          "total" : 20429863,
          "updated" : 0,
          "created" : 14826000,
          "deleted" : 0,
          "batches" : 4943,
          "version_conflicts" : 0,
          "noops" : 0,
          "retries" : {
            "bulk" : 0,
            "search" : 0
          },
          "throttled_millis" : 0,
          "requests_per_second" : -1.0,
          "throttled_until_millis" : 0
        },
        {
          "slice_id" : 1,
          "total" : 20430455,
          "updated" : 0,
          "created" : 14628000,
          "deleted" : 0,
          "batches" : 4877,
          "version_conflicts" : 0,
          "noops" : 0,
          "retries" : {
            "bulk" : 0,
            "search" : 0
          },
          "throttled_millis" : 0,
          "requests_per_second" : -1.0,
          "throttled_until_millis" : 0
        },
        {
          "slice_id" : 2,
          "total" : 20430822,
          "updated" : 0,
          "created" : 14862000,
          "deleted" : 0,
          "batches" : 4955,
          "version_conflicts" : 0,
          "noops" : 0,
          "retries" : {
            "bulk" : 0,
            "search" : 0
          },
          "throttled_millis" : 0,
          "requests_per_second" : -1.0,
          "throttled_until_millis" : 0
        },
        {
          "slice_id" : 3,
          "total" : 20425139,
          "updated" : 0,
          "created" : 14685000,
          "deleted" : 0,
          "batches" : 4896,
          "version_conflicts" : 0,
          "noops" : 0,
          "retries" : {
            "bulk" : 0,
            "search" : 0
          },
          "throttled_millis" : 0,
          "requests_per_second" : -1.0,
          "throttled_until_millis" : 0
        },
        {
          "slice_id" : 4,
          "total" : 20430822,
          "updated" : 0,
          "created" : 14799000,
          "deleted" : 0,
          "batches" : 4934,
          "version_conflicts" : 0,
          "noops" : 0,
          "retries" : {
            "bulk" : 0,
            "search" : 0
          },
          "throttled_millis" : 0,
          "requests_per_second" : -1.0,
          "throttled_until_millis" : 0
        }
      ]
    },
    "description" : "reindex from [log-wlb-sysmon-2021.08.16-000015] to [log-wlb-sysmon-2021.10.04-000015][_doc]",
    "start_time_in_millis" : 1633325995144,
    "running_time_in_nanos" : 10884886700980,
    "cancellable" : true,
    "headers" : { }
  },
  "response" : {
    "took" : 10884885,
    "timed_out" : false,
    "total" : 102147101,
    "updated" : 0,
    "created" : 73800000,
    "deleted" : 0,
    "batches" : 24605,
    "version_conflicts" : 0,
    "noops" : 0,
    "retries" : {
      "bulk" : 0,
      "search" : 0
    },
    "throttled" : "0s",
    "throttled_millis" : 0,
    "requests_per_second" : -1.0,
    "throttled_until" : "0s",
    "throttled_until_millis" : 0,
    "slices" : [
      {
        "slice_id" : 0,
        "total" : 20429863,
        "updated" : 0,
        "created" : 14826000,
        "deleted" : 0,
        "batches" : 4943,
        "version_conflicts" : 0,
        "noops" : 0,
        "retries" : {
          "bulk" : 0,
          "search" : 0
        },
        "throttled" : "0s",
        "throttled_millis" : 0,
        "requests_per_second" : -1.0,
        "throttled_until" : "0s",
        "throttled_until_millis" : 0
      },
      {
        "slice_id" : 1,
        "total" : 20430455,
        "updated" : 0,
        "created" : 14628000,
        "deleted" : 0,
        "batches" : 4877,
        "version_conflicts" : 0,
        "noops" : 0,
        "retries" : {
          "bulk" : 0,
          "search" : 0
        },
        "throttled" : "0s",
        "throttled_millis" : 0,
        "requests_per_second" : -1.0,
        "throttled_until" : "0s",
        "throttled_until_millis" : 0
      },
      {
        "slice_id" : 2,
        "total" : 20430822,
        "updated" : 0,
        "created" : 14862000,
        "deleted" : 0,
        "batches" : 4955,
        "version_conflicts" : 0,
        "noops" : 0,
        "retries" : {
          "bulk" : 0,
          "search" : 0
        },
        "throttled" : "0s",
        "throttled_millis" : 0,
        "requests_per_second" : -1.0,
        "throttled_until" : "0s",
        "throttled_until_millis" : 0
      },
      {
        "slice_id" : 3,
        "total" : 20425139,
        "updated" : 0,
        "created" : 14685000,
        "deleted" : 0,
        "batches" : 4896,
        "version_conflicts" : 0,
        "noops" : 0,
        "retries" : {
          "bulk" : 0,
          "search" : 0
        },
        "throttled" : "0s",
        "throttled_millis" : 0,
        "requests_per_second" : -1.0,
        "throttled_until" : "0s",
        "throttled_until_millis" : 0
      },
      {
        "slice_id" : 4,
        "total" : 20430822,
        "updated" : 0,
        "created" : 14799000,
        "deleted" : 0,
        "batches" : 4934,
        "version_conflicts" : 0,
        "noops" : 0,
        "retries" : {
          "bulk" : 0,
          "search" : 0
        },
        "throttled" : "0s",
        "throttled_millis" : 0,
        "requests_per_second" : -1.0,
        "throttled_until" : "0s",
        "throttled_until_millis" : 0
      }
    ],
    "failures" : [
      {
        "index" : "log-wlb-sysmon-2021.10.04-000015",
        "type" : "_doc",
        "id" : "e2f422b1661ab22535ba77f4fc3099a1d2c8cffe",
        "cause" : {
          "type" : "cluster_block_exception",
          "reason" : "index [log-wlb-sysmon-2021.10.04-000015] blocked by: [FORBIDDEN/8/index write (api)];"
        },
        "status" : 403
      },
      {
        "index" : "log-wlb-sysmon-2021.10.04-000015",
        "type" : "_doc",
        "id" : "edacaad8c936e94104da98bca78dfd0a32978668",
        "cause" : {
          "type" : "cluster_block_exception",
          "reason" : "index [log-wlb-sysmon-2021.10.04-000015] blocked by: [FORBIDDEN/8/index write (api)];"
        },
        "status" : 403
      },
      {
        "index" : "log-wlb-sysmon-2021.10.04-000015",
        "type" : "_doc",
        "id" : "cac2bb28cc0592ca7c075b8cc3b58cbbe0df95db",
        "cause" : {
          "type" : "cluster_block_exception",
          "reason" : "index [log-wlb-sysmon-2021.10.04-000015] blocked by: [FORBIDDEN/8/index write (api)];"
        },
        "status" : 403
     ```
2

GET /_ilm/policy/winlogbeat_sysmon_policy

{
  "winlogbeat_sysmon_policy" : {
    "version" : 12,
    "modified_date" : "2021-07-07T07:19:18.923Z",
    "policy" : {
      "phases" : {
        "hot" : {
          "min_age" : "0ms",
          "actions" : {
            "forcemerge" : {
              "max_num_segments" : 1
            },
            "rollover" : {
              "max_size" : "50gb",
              "max_age" : "30d"
            },
            "set_priority" : {
              "priority" : 100
            }
          }
        },
        "delete" : {
          "min_age" : "60d",
          "actions" : {
            "delete" : {
              "delete_searchable_snapshot" : true
            }
          }
        }
      }
    }
  }
}

GET /_cat/indices/log-wlb-sysmon-*?v

health status index                            uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   log-wlb-sysmon-2021.10.01-000014 OaDubXBDTFuwVqYjLIIcXg   1   1   87312747            0    114.4gb         57.2gb
green  open   log-wlb-sysmon-2021.09.30-000013 _ukLur8LR0aY4pUJPnGIcw   1   1   85912426            0    113.9gb         56.9gb
green  open   log-wlb-sysmon-2021.08.16-000015 EtF6odgpQGGRjQ0Nywqtxg   1   1  102154111            0    102.2gb         51.1gb
green  open   log-wlb-sysmon-2021.10.04-000015 dL6aRy2qQZ6J01Mmuxs7Fg   1   0   73800000            0    101.1gb        101.1gb
green  open   log-wlb-sysmon-2021.10.04-000016 AxlE3DUYSV6dgRyZgo5n4g   1   1          0            0       416b           208b
green  open   log-wlb-sysmon-2021.09.23-000019 tevWknvdRY-XwrEARoNY9g   1   1   49458446        89797     59.1gb         29.5gb
green  open   log-wlb-sysmon-2021.08.27-000017 hAGeCkqhRlCBvmwBLonjaQ   1   1   93303600            0      102gb           51gb
green  open   log-wlb-sysmon-2021.08.19-000016 mkBAnhNMReyHzw0PJwkCHg   1   1   83527835            0      102gb           51gb
green  open   log-wlb-sysmon-2021.09.07-000018 HuDxDct7RGKOcuYFXCKA8w   1   1   92614553            0    101.7gb         50.8gb
3

I guess that you should use the alias name as the destination instead of the concrete index name?

4

sorry @dadoonet i could not get it , i want to change here something and what

did you mean this ?

POST _reindex?slices=5&wait_for_completion=false&max_docs=102154111
{
   "source": {
    "index": "log-wlb-sysmon-2021.08.16-000015"
    , "size": 3000
    
  }
  , "dest": {
    "index": "log-wlb-sysmon"
  }
}
5

Yes. This is what I meant.

Or use a concrete index name which is not rolled over?

6

ya i am understanding your point but why its size increasing if we are disabling replica. its actual size is 102gb replica including and replica size is 50gb but in current situation its size(primary size) is 70gb
We want our index based on ILM

7

That's not the same question. You are mixing questions which makes everything difficult to follow IMO.

Did you try to forcemerge the index after the reindex operation?

I have no idea of what you are trying to do. More than that, I don't know what problem you are trying to solve.

8

Let me try to explain you
I am taking one index to explain what problem i am facing and why this write error happened.

  1. log-wlb-sysmon-2021.08.16-000015
    This index is using ilm properties.If primary shard reaches to 50gb it rollover to new index we have one replica also so total size in this case is 102.2gb

  2. We are reindexing because we have mapping problem. We reindex log-wlb-sysmon-2021.08.16-000015 to log-wlb-sysmon-2021.10.04-000015

  3. So reindexing 102154111 documents of log-wlb-sysmon-2021.08.16-000015 to log-wlb-sysmon-2021.10.04-000015 leads to creation of another index because it's reaches to 50gb shard size

  4. Somehow reindexing process get stopped because alias it was writing for was log-wlb-sysmon-2021.08.16-000015 but as i said it size becomes 50gb and documents it contains is incompleted.And we faced error writing forbidden because new index called log-wlb-sysmon-2021.10.04-000016 has is_write_index false

  5. we cannot forcemerge as it has incomplete documents.So we want that our index log-wlb-sysmon-2021.10.04-000015 has 102154111 documents and it should use previous ilm settings

9

Did you have a date issue?

Anyway, I'd probably reindex in another brand new index scheme naming, like log-wlb-sysmon-new-YYYY-MM-DD using the same winlogbeat_sysmon_policy so it will manage automatically the rollover for you.

10

Screenshot 2021-10-05 153551
Screenshot 2021-10-05 1535511903×960 61.3 KB

no, i just want log-wlb-sysmon-2021.10.05-000015 should have 102154111 documents instead of 73707000 it just create another index because of shard limit that i specified in ilm.Although i changed 50gb to 200gb but nothing works.

11

hey @dadoonet basically we just want to reindex the ilm based index so that it can be rollover if we reindex older indices which is ilm based..we encounter with an issue it created the new index..if it is creating new index then alias should automatically move to the newer index but in this case it is not switching to new index and it show forbidden error.

12

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.