Partial update in a rolled-over index

We're using datetime based index with an alias that is rolled over to a new index daily. We managed to achieve this by creating an ingest pipeline that figures out the physical index behind an alias and updates(read replaces) the original document. Now, requirement is to partially update a document in a rolled-over index, meaning, we might get an additional field to update in a document that resides in an index that has been marked as write=false.
If the document is in current write index, update works fine.

But the same update API throws an "Index missing exception" OR "version_conflict_engine_exception" otherwise.

ILM policy

PUT _ilm/policy/hot-warm-delete
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "1d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "warm": {
        "min_age": "10d",
        "actions": {
          "set_priority": {
            "priority": 50
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {
            "delete_searchable_snapshot": true
          }
        }
      }
    }
  }
}

pipeline

{
  "description": "Adds ingest time to document + figures out the destination index if it's a dated transaction",
  "processors": [
    {
      "set": {
        "field": "_source.ingest_time",
        "value": "{{{_ingest.timestamp}}}"
      }
    },
    {
    "script": {
      "description": "Set index based on `payload_ts` field",
       "lang": "painless",
      "source": """
        long today = new Date().getTime();
        long payloadTs = ctx['payload_ts'] ;
        
        LocalDate payloadDate = Instant.ofEpochMilli(payloadTs).atZone(ZoneId.systemDefault()).toLocalDate();
        LocalDate currentDate = Instant.ofEpochMilli(today).atZone(ZoneId.systemDefault()).toLocalDate();

        if(payloadDate.isBefore(currentDate)){
        //if(payloadTs < today){
          ctx['_index'] = ctx['_index'] + '-' + payloadDate +'-' + '000001';  
        }
      """
      }
    }
  ]
}

bootstrap index

PUT <transactions-{now/d{yyyy-MM-dd}}-000001>
{
    "aliases": {
        "transactions": {
        "is_write_index": "True"
    }
    },
    "mappings": {
    "properties": {
      "executingEntityIdCode":    { "type": "keyword" },  
      "transactionReferenceNumber":  { "type": "keyword"  }, 
      "ingest_time":   { "type": "date"  }     
    }
  },
  "settings" : { 
    "index" : { 
      "lifecycle.origination_date": 1656892800000 
    } 
  }
 }
POST /transactions/_update/1000
{
    "doc" : {
        "myfield": "new value of my field",
        "payload_ts": 1656806400000
    }
}

exception

{
  "error": {
    "root_cause": [
      {
        "type": "version_conflict_engine_exception",
        "reason": "[1000]: version conflict, required seqNo [143], primary term [1]. current document has seqNo [55] and primary term [1]",
        "index_uuid": "l6gM62jwQSq92zhU1im1-w",
        "shard": "0",
        "index": "transactions-2022-07-03-000001"
      }
    ],
    "type": "version_conflict_engine_exception",
    "reason": "[1000]: version conflict, required seqNo [143], primary term [1]. current document has seqNo [55] and primary term [1]",
    "index_uuid": "l6gM62jwQSq92zhU1im1-w",
    "shard": "0",
    "index": "transactions-2022-07-03-000001"
  },
  "status": 409
}

My question is-
Is it possible to achieve this if we don't want to index a fully formed document all the time.

I understand we can achieve this using painless script but I want to avoid runtime processing for performance boost.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.