Deleting array fields logstash

Elastic Stack Logstash
1

Hi,
In my company we're trying to process a json array sent by filiebeat to logstash. The json that logstash receives is:

{ "logFile": [ { "line": 2, "elements": [ { "line": 4, "name": "", "description": "", "type": "", "keyword": "", "steps": [ { "result": { "duration": "", "status": "" }, "line": 5, "name": "", "match": { "arguments": [ { "val": "", "offset": 2 } ], "location": "" }, "keyword": " " } ] }, { "start_timestamp": "", "before": [ { "result": { "duration": 561992900, "status": "" }, "match": { "location": "" } } ], "line": 8, "name": "", "description": "", "id": "", "after": [ { "result": { "duration": 1327737100, "status": "" }, "match": { "location": "" } } ], "type": "", "keyword": "", "steps": [ { "result": { "duration": 2546265500, "status": "" }, "line": 9, "name": "", "match": { "location": "" }, "keyword": " " }, { "embeddings": [ { "data": "", "mime_type": "", "name": "" } ], "result": { "duration": 8510044800, "status": "" }, "line": 10, "name": "", "match": { "arguments": [ { "val": "", "offset": 25 }, { "val": "", "offset": 44 }, { "val": "", "offset": 52 } ], "location": "" }, "keyword": " " }, { "result": { "duration": 24000600, "status": "" }, "line": 11, "name": "", "match": { "location": "" }, "keyword": " " } ], "tags": [ { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" } ] }, { "line": 4, "name": "", "description": "", "type": "", "keyword": "", "steps": [ { "result": { "duration": 853000700, "status": "" }, "line": 5, "name": "", "match": { "arguments": [ { "val": "", "offset": 2 } ], "location": "" }, "keyword": "" } ] }, { "start_timestamp": "", "before": [ { "result": { "duration": 1000500, "status": "" }, "match": { "location": "" } } ], "line": 15, "name": "", "description": "", "id": "", "after": [ { "result": { "duration": 128004000, "status": "" }, "match": { "location": "" } } ], "type": "", "keyword": "", "steps": [ { "result": { "duration": 1852995400, "status": "" }, "line": 16, "name": "", "match": { "location": "" }, "keyword": " " }, { "embeddings": [ { "data": "", "mime_type": "", "name": "" } ], "result": { "duration": 3733347800, "status": "" }, "line": 17, "name": "", "match": { "arguments": [ { "val": "", "offset": 25 }, { "val": "", "offset": 44 }, { "val": "", "offset": 52 } ], "location": "" }, "keyword": " " }, { "result": { "duration": 9997000, "status": "" }, "line": 18, "name": "", "match": { "location": "" }, "keyword": " " } ], "tags": [ { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" } ] } ], "name": "", "description": "", "id": "", "keyword": "", "uri": "", "tags": [ { "name": "", "type": "", "location": { "line": 1, "column": 1 } }, { "name": "", "type": "", "location": { "line": 1, "column": 8 } } ] } ]}

But with several occurrences inside the main array. With logstash i've splitted the array into every element in the array [logFile][elements] with success using this filter section:


filter {
	json{
		source => "message"
	}
	split{
		field => "logFile"
	}
	split{
		field => "[logFile][elements]"
	}
}

Right now, the json present on each event is something like:

"logFile": {
      "description": "",
      "keyword": "Feature",
      "uri": "",
      "id": "",
      "name": "",
      "elements": {
        "steps": [
          {
            "name": "",
            "keyword": "Given ",
            "line": 85,
            "result": {
              "status": "passed",
              "duration": 6792357600
            },
            "match": {
              "location": ""
            }
          },
          {
            "embeddings": [
              {
                "name": "Session Token",
                "mime_type": "text/plain",
                "data": ""
              }
            ],
            "keyword": "And ",
            "result": {
              "status": "passed",
              "duration": 12536463900
            },
            "match": {
              "arguments": [
                {
                  "val": "",
                  "offset": 25
                },
                {
                  "val": "",
                  "offset": 44
                },
                {
                  "val": "",
                  "offset": 52
                }
              ],
              "location": ""
            },
            "name": "",
            "line": 86
          },
          {
            "name": "",
            "keyword": "When ",
            "line": 87,
            "result": {
              "status": "passed",
              "duration": 358273600
            },
            "match": {
              "arguments": [
                {
                  "val": "",
                  "offset": 8
                },
                {
                  "val": "",
                  "offset": 30
                }
              ],
              "location": ""
            }
          },
          {
            "name": "",
            "keyword": "And ",
            "line": 88,
            "result": {
              "status": "passed",
              "duration": 10050374100
            },
            "match": {
              "arguments": [
                {
                  "val": "",
                  "offset": 5
                },
                {
                  "val": "",
                  "offset": 18
                }
              ],
              "location": ""
            }
          },
          {
            "name": "",
            "keyword": "And ",
            "line": 89,
            "result": {
              "status": "passed",
              "duration": 2262704100
            },
            "match": {
              "location": ""
            }
          },
          {
            "name": "",
            "keyword": "Then ",
            "line": 90,
            "result": {
              "status": "passed",
              "duration": 121674300
            },
            "match": {
              "location": ""
            }
          }
        ],
        "description": "",
        "id": "",
        "line": 84,
        "keyword": "Scenario",
        "before": [
          {
            "result": {
              "status": "passed"
            },
            "match": {
              "location": ""
            }
          }
        ],
        "start_timestamp": "",
        "after": [
          {
            "result": {
              "status": "passed",
              "duration": 2056419700
            },
            "match": {
              "location": "-"
            }
          }
        ],
        "name": "",
        "type": "scenario",
        "tags": [
          {
            "name": "-"
          },
          {
            "name": "-"
          },
          {
            "name": "-"
          },
          {
            "name": "-"
          }
        ]
      },
      "line": 2,
      "tags": [
        {
          "name": "-",
          "type": "Tag",
          "location": {
            "line": 1,
            "column": 1
          }
        },
        {
          "name": "-",
          "type": "Tag",
          "location": {
            "line": 1,
            "column": 11
          }
        }
      ]
    },

How can I remove for example the "embeddings" field from "[elements][steps]"

Note: Already tried with mutate {remove_field => "[logFile][elements][steps][embeddings]") and some ruby code without success. Thank you in advance.

2

Since steps is also an array you will may need to use a Ruby filter.

Does it have multiple elements or is always a one element array? If it is always a one element array you may remove it using [logfile][elements][steps][0][embeddings].

If it have multiple elements, then you will need to use a ruby filter.

What ruby code you used? You didn't share.

3

Yes, "steps" have a variable size. But the same problem persist if i try to delete all the [steps] field. I've tried with mutate{ remove_field => "[logFile][elements][steps]" }. Doesn´t return any error but simply does nothing. Also tried with just "elements" and just "[elements][steps]" with the same result.

Something like this:

ruby {
    code => '
        elementsArray = event.get("[logFile][elements]")
        if elementsArray
            newElementsArray = []
            elementsArray.each { |x|
                x.delete("[logFile][elements][steps]")
                newElementsArray << x
            }
            event.set("[logFile][elements]", newElementsArray)
        end
    '
}

And it was returning an error about the replacing. I have to make the disclaimer that i have zero ruby bases and this code is copied form another discuss topic that I found :slight_smile:

4

Try

    ruby {
        code => '
            #event.remove("[logFile][elements][steps]")

            steps = event.get("[logFile][elements][steps]")
            if steps.respond_to? "each_index"
                steps.each_index { |x| event.remove("[logFile][elements][steps][#{x}][embeddings]") }
            end
        '
    }
7

This is working fine. Thanks! Another question. Would be nice if instead of display the result for every step (possible values are skipped, passed, failed), display a global result. Like if steps have the results passed, passed, failed, the global result should be failed. If the steps have results passed, passed, skipped, the global result should be skipped. In general, the result should be passed only if the array have all steps as passed. Is this possible?

8

Try

    ruby {
        code => '
            steps = event.get("[logFile][elements][steps]")
            if steps.respond_to? "each_index"
                result = steps[0]["result"]["status"]
                steps.each_index { |x|
                    event.remove("[logFile][elements][steps][#{x}][embeddings]")
                    if steps[x]["result"]["status"] != "passed"
                        result = steps[x]["result"]["status"]
                    end
                }
                event.set("overallResult", result)
            end
        '
    }
9

Perfect :mechanical_arm: ! Exactly what i wanted

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.