Deleting array fields logstash

andrejcoliveira · January 25, 2024, 10:38am

Hi,
In my company we're trying to process a json array sent by filiebeat to logstash. The json that logstash receives is:

{ "logFile": [ { "line": 2, "elements": [ { "line": 4, "name": "", "description": "", "type": "", "keyword": "", "steps": [ { "result": { "duration": "", "status": "" }, "line": 5, "name": "", "match": { "arguments": [ { "val": "", "offset": 2 } ], "location": "" }, "keyword": " " } ] }, { "start_timestamp": "", "before": [ { "result": { "duration": 561992900, "status": "" }, "match": { "location": "" } } ], "line": 8, "name": "", "description": "", "id": "", "after": [ { "result": { "duration": 1327737100, "status": "" }, "match": { "location": "" } } ], "type": "", "keyword": "", "steps": [ { "result": { "duration": 2546265500, "status": "" }, "line": 9, "name": "", "match": { "location": "" }, "keyword": " " }, { "embeddings": [ { "data": "", "mime_type": "", "name": "" } ], "result": { "duration": 8510044800, "status": "" }, "line": 10, "name": "", "match": { "arguments": [ { "val": "", "offset": 25 }, { "val": "", "offset": 44 }, { "val": "", "offset": 52 } ], "location": "" }, "keyword": " " }, { "result": { "duration": 24000600, "status": "" }, "line": 11, "name": "", "match": { "location": "" }, "keyword": " " } ], "tags": [ { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" } ] }, { "line": 4, "name": "", "description": "", "type": "", "keyword": "", "steps": [ { "result": { "duration": 853000700, "status": "" }, "line": 5, "name": "", "match": { "arguments": [ { "val": "", "offset": 2 } ], "location": "" }, "keyword": "" } ] }, { "start_timestamp": "", "before": [ { "result": { "duration": 1000500, "status": "" }, "match": { "location": "" } } ], "line": 15, "name": "", "description": "", "id": "", "after": [ { "result": { "duration": 128004000, "status": "" }, "match": { "location": "" } } ], "type": "", "keyword": "", "steps": [ { "result": { "duration": 1852995400, "status": "" }, "line": 16, "name": "", "match": { "location": "" }, "keyword": " " }, { "embeddings": [ { "data": "", "mime_type": "", "name": "" } ], "result": { "duration": 3733347800, "status": "" }, "line": 17, "name": "", "match": { "arguments": [ { "val": "", "offset": 25 }, { "val": "", "offset": 44 }, { "val": "", "offset": 52 } ], "location": "" }, "keyword": " " }, { "result": { "duration": 9997000, "status": "" }, "line": 18, "name": "", "match": { "location": "" }, "keyword": " " } ], "tags": [ { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" }, { "name": "" } ] } ], "name": "", "description": "", "id": "", "keyword": "", "uri": "", "tags": [ { "name": "", "type": "", "location": { "line": 1, "column": 1 } }, { "name": "", "type": "", "location": { "line": 1, "column": 8 } } ] } ]}

But with several occurrences inside the main array. With logstash i've splitted the array into every element in the array [logFile][elements] with success using this filter section:


filter {
	json{
		source => "message"
	}
	split{
		field => "logFile"
	}
	split{
		field => "[logFile][elements]"
	}
}

Right now, the json present on each event is something like:

"logFile": {
      "description": "",
      "keyword": "Feature",
      "uri": "",
      "id": "",
      "name": "",
      "elements": {
        "steps": [
          {
            "name": "",
            "keyword": "Given ",
            "line": 85,
            "result": {
              "status": "passed",
              "duration": 6792357600
            },
            "match": {
              "location": ""
            }
          },
          {
            "embeddings": [
              {
                "name": "Session Token",
                "mime_type": "text/plain",
                "data": ""
              }
            ],
            "keyword": "And ",
            "result": {
              "status": "passed",
              "duration": 12536463900
            },
            "match": {
              "arguments": [
                {
                  "val": "",
                  "offset": 25
                },
                {
                  "val": "",
                  "offset": 44
                },
                {
                  "val": "",
                  "offset": 52
                }
              ],
              "location": ""
            },
            "name": "",
            "line": 86
          },
          {
            "name": "",
            "keyword": "When ",
            "line": 87,
            "result": {
              "status": "passed",
              "duration": 358273600
            },
            "match": {
              "arguments": [
                {
                  "val": "",
                  "offset": 8
                },
                {
                  "val": "",
                  "offset": 30
                }
              ],
              "location": ""
            }
          },
          {
            "name": "",
            "keyword": "And ",
            "line": 88,
            "result": {
              "status": "passed",
              "duration": 10050374100
            },
            "match": {
              "arguments": [
                {
                  "val": "",
                  "offset": 5
                },
                {
                  "val": "",
                  "offset": 18
                }
              ],
              "location": ""
            }
          },
          {
            "name": "",
            "keyword": "And ",
            "line": 89,
            "result": {
              "status": "passed",
              "duration": 2262704100
            },
            "match": {
              "location": ""
            }
          },
          {
            "name": "",
            "keyword": "Then ",
            "line": 90,
            "result": {
              "status": "passed",
              "duration": 121674300
            },
            "match": {
              "location": ""
            }
          }
        ],
        "description": "",
        "id": "",
        "line": 84,
        "keyword": "Scenario",
        "before": [
          {
            "result": {
              "status": "passed"
            },
            "match": {
              "location": ""
            }
          }
        ],
        "start_timestamp": "",
        "after": [
          {
            "result": {
              "status": "passed",
              "duration": 2056419700
            },
            "match": {
              "location": "-"
            }
          }
        ],
        "name": "",
        "type": "scenario",
        "tags": [
          {
            "name": "-"
          },
          {
            "name": "-"
          },
          {
            "name": "-"
          },
          {
            "name": "-"
          }
        ]
      },
      "line": 2,
      "tags": [
        {
          "name": "-",
          "type": "Tag",
          "location": {
            "line": 1,
            "column": 1
          }
        },
        {
          "name": "-",
          "type": "Tag",
          "location": {
            "line": 1,
            "column": 11
          }
        }
      ]
    },

How can I remove for example the "embeddings" field from "[elements][steps]"

Note: Already tried with mutate {remove_field => "[logFile][elements][steps][embeddings]") and some ruby code without success. Thank you in advance.

leandrojmp · January 25, 2024, 12:19pm

Since steps is also an array you will may need to use a Ruby filter.

Does it have multiple elements or is always a one element array? If it is always a one element array you may remove it using [logfile][elements][steps][0][embeddings].

If it have multiple elements, then you will need to use a ruby filter.

What ruby code you used? You didn't share.

andrejcoliveira · January 25, 2024, 2:25pm

Yes, "steps" have a variable size. But the same problem persist if i try to delete all the [steps] field. I've tried with mutate{ remove_field => "[logFile][elements][steps]" }. Doesn´t return any error but simply does nothing. Also tried with just "elements" and just "[elements][steps]" with the same result.

Something like this:

ruby {
    code => '
        elementsArray = event.get("[logFile][elements]")
        if elementsArray
            newElementsArray = []
            elementsArray.each { |x|
                x.delete("[logFile][elements][steps]")
                newElementsArray << x
            }
            event.set("[logFile][elements]", newElementsArray)
        end
    '
}

And it was returning an error about the replacing. I have to make the disclaimer that i have zero ruby bases and this code is copied form another discuss topic that I found

Badger · January 25, 2024, 6:34pm

Try

    ruby {
        code => '
            #event.remove("[logFile][elements][steps]")

            steps = event.get("[logFile][elements][steps]")
            if steps.respond_to? "each_index"
                steps.each_index { |x| event.remove("[logFile][elements][steps][#{x}][embeddings]") }
            end
        '
    }

andrejcoliveira · January 26, 2024, 12:13pm

Badger:

    ruby {
        code => '
            #event.remove("[logFile][elements][steps]")

            steps = event.get("[logFile][elements][steps]")
            if steps.respond_to? "each_index"
                steps.each_index { |x| event.remove("[logFile][elements][steps][#{x}][embeddings]") }
            end
        '
    }

This is working fine. Thanks! Another question. Would be nice if instead of display the result for every step (possible values are skipped, passed, failed), display a global result. Like if steps have the results passed, passed, failed, the global result should be failed. If the steps have results passed, passed, skipped, the global result should be skipped. In general, the result should be passed only if the array have all steps as passed. Is this possible?

Badger · January 26, 2024, 6:20pm

Try

    ruby {
        code => '
            steps = event.get("[logFile][elements][steps]")
            if steps.respond_to? "each_index"
                result = steps[0]["result"]["status"]
                steps.each_index { |x|
                    event.remove("[logFile][elements][steps][#{x}][embeddings]")
                    if steps[x]["result"]["status"] != "passed"
                        result = steps[x]["result"]["status"]
                    end
                }
                event.set("overallResult", result)
            end
        '
    }

andrejcoliveira · January 29, 2024, 11:37am

Badger:

Try

    ruby {
        code => '
            steps = event.get("[logFile][elements][steps]")
            if steps.respond_to? "each_index"
                result = steps[0]["result"]["status"]
                steps.each_index { |x|
                    event.remove("[logFile][elements][steps][#{x}][embeddings]")
                    if steps[x]["result"]["status"] != "passed"
                        result = steps[x]["result"]["status"]
                    end
                }
                event.set("overallResult", result)
            end
        '
    }

Perfect ! Exactly what i wanted

system · February 26, 2024, 11:38am

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.