Enrich policy working only on some documents

Elastic Stack Elasticsearch
1

hello guys!!

I did the simulate pipeline with two documents, one that gets enriched and one that is not.

This is the pipeline

{
  "peruchannelspm-ingest-pipeline": {
    "processors": [
      {
        "enrich": {
          "policy_name": "spm-puch-enriching-policy",
          "field": "G-UCHAN",
          "target_field": "U",
          "max_matches": "1"
        }
      }
    ]
  }
}

This is the simulate

POST /_ingest/pipeline/peruchannelspm-ingest-pipeline/_simulate
{
  "docs": [
    {
  "_index": "perupchan",
  "_id": "hMdNcJABaAFMtSx5Naec",
  "_version": 1,
  "_score": 0,
  "_source": {
    "channel_name": "cable-upstream 1/2/14.0",
    "u_20-29": 70,
    "u_40-49": 21,
    "iso_time_s": "2024-05-04T23:50:00Z",
    "u_60-69": 0,
    "type": "up-util-facts",
    "u_80-89": 0,
    "u_0-9": 13,
    "G-UCHAN": "WAKA-E6K2 cable-upstream 1/2/14.0",
    "cmts_id": "WAKA-E6K2",
    "u_10-19": 121,
    "hub_id": "WAKANDA",
    "@timestamp": "2024-07-01T21:57:06.404784936Z",
    "u_90+": 1,
    "u_30-39": 57,
    "@version": "1",
    "u_50-59": 5,
    "week_number": 17,
    "cable_mac": "cable-mac 2",
    "u_70-79": 0
  }},
  {
  "_index": "perupchan",
  "_id": "uMdicJABaAFMtSx5NfzY",
  "_version": 1,
  "_score": 0,
  "_source": {
    "channel_name": "cable-upstream 1/0/10.0",
    "u_20-29": 99,
    "u_40-49": 34,
    "iso_time_s": "2024-05-04T23:50:00Z",
    "u_60-69": 12,
    "type": "up-util-facts",
    "u_80-89": 1,
    "u_0-9": 15,
    "G-UCHAN": "GOTH-E6K1 cable-upstream 1/0/10.0",
    "cmts_id": "GOTH-E6K1",
    "u_10-19": 58,
    "hub_id": "GOTHAM",
    "@timestamp": "2024-07-01T22:20:02.699800605Z",
    "u_30-39": 46,
    "u_90+": 1,
    "@version": "1",
    "u_50-59": 18,
    "week_number": 17,
    "cable_mac": "cable-mac 100",
    "u_70-79": 4
  }}
  ]
}

this is the result

{
  "docs": [
    {
      "doc": {
        "_index": "perupchan",
        "_version": "1",
        "_id": "hMdNcJABaAFMtSx5Naec",
        "_source": {
          "channel_name": "cable-upstream 1/2/14.0",
          "u_20-29": 70,
          "u_40-49": 21,
          "iso_time_s": "2024-05-04T23:50:00Z",
          "u_60-69": 0,
          "type": "up-util-facts",
          "u_80-89": 0,
          "u_0-9": 13,
          "G-UCHAN": "WAKA-E6K2 cable-upstream 1/2/14.0",
          "cmts_id": "WAKA-E6K2",
          "u_10-19": 121,
          "hub_id": "WAKANDA",
          "@timestamp": "2024-07-01T21:57:06.404784936Z",
          "u_90+": 1,
          "u_30-39": 57,
          "@version": "1",
          "u_50-59": 5,
          "week_number": 17,
          "cable_mac": "cable-mac 2",
          "u_70-79": 0
        },
        "_ingest": {
          "timestamp": "2024-07-01T22:34:56.000813317Z"
        }
      }
    },
    {
      "doc": {
        "_index": "perupchan",
        "_version": "1",
        "_id": "uMdicJABaAFMtSx5NfzY",
        "_source": {
          "channel_name": "cable-upstream 1/0/10.0",
          "u_20-29": 99,
          "u_40-49": 34,
          "iso_time_s": "2024-05-04T23:50:00Z",
          "u_60-69": 12,
          "type": "up-util-facts",
          "u_80-89": 1,
          "u_0-9": 15,
          "G-UCHAN": "GOTH-E6K1 cable-upstream 1/0/10.0",
          "cmts_id": "GOTH-E6K1",
          "u_10-19": 58,
          "hub_id": "GOTHAM",
          "@timestamp": "2024-07-01T22:20:02.699800605Z",
          "U": {
            "G-UCHAN": "GOTH-E6K1 cable-upstream 1/0/10.0",
            "G-USG": "GOTH-E6K1 cable-mac 100 1"
          },
          "u_30-39": 46,
          "u_90+": 1,
          "@version": "1",
          "u_50-59": 18,
          "week_number": 17,
          "cable_mac": "cable-mac 100",
          "u_70-79": 4
        },
        "_ingest": {
          "timestamp": "2024-07-01T22:34:56.000852642Z"
        }
      }
    }
  ]
}

both matching values can be found on the enrich index

one

{
  "_index": "enrich_uchannel_count",
  "_id": "JsYhcJABaAFMtSx5mPhf",
  "_version": 1,
  "_score": 0,
  "_source": {
    "cable_mac": "cable-mac 100",
    "G-USG": "GOTH-E6K1 cable-mac 100 1",
    "G-UCHAN": "GOTH-E6K1 cable-upstream 1/0/10.0",
    "sg_id": 1,
    "@version": "1",
    "type": "up-channel-config",
    "@timestamp": "2024-07-01T21:09:27.923894874Z",
    "cmts_id": "GOTH-E6K1",
    "channel_name": "cable-upstream 1/0/10.0"
  },
  "fields": {
    "G-UCHAN": [
      "GOTH-E6K1 cable-upstream 1/0/10.0"
    ],
    "channel_name": [
      "cable-upstream 1/0/10.0"
    ],
    "cmts_id": [
      "GOTH-E6K1"
    ],
    "sg_id": [
      1
    ],
    "@timestamp": [
      "2024-07-01T21:09:27.923Z"
    ],
    "@version": [
      "1"
    ],
    "@version.keyword": [
      "1"
    ],
    "type": [
      "up-channel-config"
    ],
    "G-USG": [
      "GOTH-E6K1 cable-mac 100 1"
    ],
    "cable_mac": [
      "cable-mac 100"
    ]
  }
}

two

{
  "_index": "enrich_uchannel_count",
  "_id": "vsYhcJABaAFMtSx5mPqA",
  "_version": 1,
  "_score": 0,
  "_source": {
    "cable_mac": "cable-mac 2",
    "G-USG": "WAKA-E6K2 cable-mac 2 1",
    "G-UCHAN": "WAKA-E6K2 cable-upstream 1/2/14.0",
    "sg_id": 1,
    "@version": "1",
    "type": "up-channel-config",
    "@timestamp": "2024-07-01T21:09:27.941073485Z",
    "cmts_id": "WAKA-E6K2",
    "channel_name": "cable-upstream 1/2/14.0"
  },
  "fields": {
    "G-UCHAN": [
      "WAKA-E6K2 cable-upstream 1/2/14.0"
    ],
    "channel_name": [
      "cable-upstream 1/2/14.0"
    ],
    "cmts_id": [
      "WAKA-E6K2"
    ],
    "sg_id": [
      1
    ],
    "@timestamp": [
      "2024-07-01T21:09:27.941Z"
    ],
    "@version": [
      "1"
    ],
    "@version.keyword": [
      "1"
    ],
    "type": [
      "up-channel-config"
    ],
    "G-USG": [
      "WAKA-E6K2 cable-mac 2 1"
    ],
    "cable_mac": [
      "cable-mac 2"
    ]
  }
}

any idea why is this not working

2

It might be caused by the mapping.

Did you try to run a match query on the field to check if it actually matches?

3

I reconfigured the enrich policy and pipeline an it appears to be working now, I did a lot of testing with that pipeline and I suspect there was some configuration leftovers somewhere if that makes sense.

If I ran into similar problems I will try the match query! Thanks
I am going to wait a couple of days to mark this as solved.