Cannot remove field with ingest processor

Elastic Stack Elasticsearch
1

I using ES 7.9.0

My data

{
      "_index": "original-index-2020.11",
      "_type": "_doc",
      "_id": "7K27Bas23a",
      "_version": 1,
      "_score": null,
      "_source": {
        "http.request.auth": "-",
        "http.request.uri": "/xxxxx",
        "test": {
          "type": "SOFTWARE",
     
          "status": true
        },
        "http.request.method": "GET",
        "@version": "1",
        "http.response.status_code": "404",
        "http.version": "1.1",
        "timestamp": "27/Nov/2020:05:46:11 +0700",
        "http.request.referrer": "\"-\"",
        "http.request.ident": "-",
        "http.response.body.bytes": "196",
        "tags": [
          "beats_input_codec_plain_applied"
        ],
        "@timestamp": "2020-11-26T22:46:13.164Z",
        "vg.vc.version": "23.54.2.0",
        "vg.vc.name": "tool-name"
      }
    }

My pipeline

{
  "description" : "remove some field are not necessary",
    {
      "remove" : {
        "field": ["http.request.ident", "http.request.referrer", "http.request.method", "http.request.auth", "http.request.uri", "http.response.body.bytes", "http.response.status_code", "http.version"],
        "on_failure" : [
          {
            "set" : {
              "field" : "error.message1",
              "value" : "{{ _ingest.on_failure_message }}"
            }
          }
        ]
      }
    },
    {
      "remove" : {
        "field": ["vg.vc.version"],
        "on_failure" : [
          {
            "set" : {
              "field" : "error.message2",
              "value" : "{{ _ingest.on_failure_message }}"
            }
          }
        ]
      }
    }
  ]
}

And result :

{
  "_index": "test-2020.11",
  "_type": "_doc",
  "_id": "7K27BnYBEw-zIe_TCKl3",
  "_version": 1,
  "_score": null,
  "_source": {
    "http.request.auth": "-",
    "http.request.uri": "/xxxxx",
    "test": {
      "type": "SOFTWARE",
 
      "status": true
    },
    "error": {
      "message2": "field [vg] not present as part of path [vg.vc.version]",
      "message1": "field [http] not present as part of path [http.request.ident]"
    },
    "http.request.method": "GET",
    "@version": "1",
    "http.response.status_code": "404",
    "http.version": "1.1",
    "timestamp": "27/Nov/2020:05:46:11 +0700",
    "http.request.referrer": "\"-\"",
    "http.request.ident": "-",
    "http.response.body.bytes": "196",
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "@timestamp": "2020-11-26T22:46:13.164Z",
    "vg.vc.version": "23.54.2.0",
    "vg.vc.name": "tool-name"
  },
  "fields": {
    "@timestamp": [
      "2020-11-26T22:46:13.164Z"
    ]
  },
  "sort": [
    1606430773164
  ]
}
2

I believe this is happening because the notation used is interpreted as a JSON object path. For example, take a look at flat_settings.

I believe that the processor is expecting your document to look something like this:

{
      "_index": "original-index-2020.11",
      "_type": "_doc",
      "_id": "7K27Bas23a",
      "_version": 1,
      "_score": null,
      "_source": {
        "http”: {
            "request": {
                "auth": "-",
                "uri": "/xxxxx"
            }
        },
        "test": {
          "type": "SOFTWARE",
     
          "status": true
        },
...

Here’s a seemingly related forum post How to handle dot in field names in ES 7.0.

I think you may have to change the field names.

3

Actually you can use the Dot Expander Processor and then remove processor. It's not graceful, but here's one working version:

{
    "pipeline": {
        "processors": [
            {
                "dot_expander": {
                    "field": "http.version",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.response.status_code",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.response.body.bytes",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.request.uri",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.request.auth",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.request.method",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.request.referrer",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "dot_expander": {
                    "field": "http.request.ident",
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            },
            {
                "remove": {
                    "field": [
                        "http"
                    ],
                    "on_failure": [
                        {
                            "set": {
                                "field": "error.message1",
                                "value": "{{ _ingest.on_failure_message }}"
                            }
                        }
                    ]
                }
            }
        ]
    }
}
1 Like
4

Thank @egalpin, that is news with me.
Im using Dot expander and it worked.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.