Extract JSON log from JSON

Elastic Stack Logstash
1

Hi everyone,
I´m trying to create an index based on a script output. The script itself creates an NDJSON like:

{"packages/current_version":"3.7.3-2+deb10u5","packages/candidate_version":"3.7.3-2+deb10u6","packages/priority":"optional","packages/security":"false","timestamp":"10/31/2023 10:42:28","hostname":"azure-demo-host","packages/name":"libpython3.7-minimal","packages/section":"python"}
{"packages/current_version":"0.21","packages/candidate_version":"0.21+deb10u1","packages/priority":"optional","packages/security":"false","timestamp":"10/31/2023 10:42:28","hostname":"azure-demo-host","packages/name":"python3-distro-info","packages/section":"python"}
{"packages/current_version":"1.8.0-2.1","packages/candidate_version":"1.8.0-2.1+deb10u1","packages/priority":"optional","packages/security":"false","timestamp":"10/31/2023 10:42:28","hostname":"azure-demo-host","packages/name":"libssh2-1","packages/section":"libs"}
{"packages/current_version":"1.17-3+deb10u5","packages/candidate_version":"1.17-3+deb10u6","packages/priority":"standard","packages/security":"false","timestamp":"10/31/2023 10:42:28","hostname":"azure-demo-host","packages/name":"libkrb5-3","packages/section":"libs"}
{"packages/current_version":"1.17-3+deb10u5","packages/candidate_version":"1.17-3+deb10u6","packages/priority":"standard","packages/security":"false","timestamp":"10/31/2023 10:42:28","hostname":"azure-demo-host","packages/name":"libgssapi-krb5-2","packages/section":"libs"}

The script will be executed by Ansible AWX. Which logs to Logstash, which sends the logs to Elasticsearch.

My current Logstash config is as follows:

        input {
          tcp {
            codec => json
            port => 5055
          }
        }

        filter {
          if [event_data][task] == "List Available Updates" and [event_data][res][stdout] {
            json {
              source => "[event_data][res][stdout]"
            }
          } else {
              drop {}
            }
          }

        output {
            elasticsearch {
              hosts => [ "${ECK_ES_HOSTS}" ]
              user => "${ECK_ES_USER}"
              password => "${ECK_ES_PASSWORD}"
              cacert => "${ECK_ES_SSL_CERTIFICATE_AUTHORITY}"
              index => "logs-apt"
            }
        }

When a log hit´s the endpoint, I would expect an index to be created based on the content of "[event_data][res][stdout]". However the complete log is indexed, and not the output.

Indexed Log 
  "_source": {
    "parent_uuid": "d2e6a7c9-8213-8052-1940-000000000004",
    "id": null,
    "changed": true,
    "job_created": "2023-11-01T18:45:37.940Z",
    "packages_security": "false",
    "packages_name": "libcurl3-gnutls",
    "verbosity": 0,
    "packages_section": "libs",
    "modified": null,
    "logger_name": "awx.analytics.job_events",
    "task": "List Available Updates",
    "host_name": "20.127.230.105",
    "stdout": "\u001b[0;33mchanged: [20.127.230.105]\u001b[0m",
    "created": "2023-11-01T18:46:39.811Z",
    "packages_priority": "optional",
    "playbook": "task.yml",
    "uuid": "6f751077-0477-4931-b446-3433bfc6c4d3",
    "failed": false,
    "host": 2,
    "level": "INFO",
    "guid": "0d3fff4e73dd4da3ae66a7cb39db202b",
    "event_display": "Host OK",
    "message": "Event data saved.",
    "job": 63,
    "port": 40712,
    "cluster_host_id": "awx-demo-task-7bd8ff7d8d-z5wvb",
    "play": "Python-APT",
    "start_line": 8,
    "timestamp": "11/01/2023 18:46:39",
    "hostname": "blob.internal.cloudapp.net",
    "event": "runner_on_ok",
    "role": "",
    "counter": 9,
    "end_line": 9,
    "tower_uuid": null,
    "packages_current_version": "7.64.0-4+deb10u6",
    "packages_candidate_version": "7.64.0-4+deb10u7",
    "@timestamp": "2023-11-01T18:46:40.138Z",
    "@version": "1",
    "event_data": {
      "event_loop": null,
      "play_uuid": "d2e6a7c9-8213-8052-1940-000000000002",
      "host": "20.127.230.105",
      "task_action": "ansible.builtin.script",
      "res": {
        "stderr": "Shared connection to 20.127.230.105 closed.\r\n",
        "changed": true,
        "stdout": "{\"packages_current_version\":\"3.7.3-2+deb10u5\",\"packages_candidate_version\":\"3.7.3-2+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libpython3.7-minimal\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"0.21\",\"packages_candidate_version\":\"0.21+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python3-distro-info\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"1.8.0-2.1\",\"packages_candidate_version\":\"1.8.0-2.1+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libssh2-1\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.17-3+deb10u5\",\"packages_candidate_version\":\"1.17-3+deb10u6\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libkrb5-3\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.17-3+deb10u5\",\"packages_candidate_version\":\"1.17-3+deb10u6\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libgssapi-krb5-2\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.36.0-2+deb10u1\",\"packages_candidate_version\":\"1.36.0-2+deb10u2\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libnghttp2-14\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.12.24-0+deb10u1\",\"packages_candidate_version\":\"1.12.28-0+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libdbus-1-3\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"4.19.289-1\",\"packages_candidate_version\":\"4.19.289-2\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"linux-image-4.19.0-25-cloud-amd64\",\"packages_section\":\"kernel\"}\r\n{\"packages_current_version\":\"2:8.1.0875-5+deb10u5\",\"packages_candidate_version\":\"2:8.1.0875-5+deb10u6\",\"packages_priority\":\"important\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"vim-common\",\"packages_section\":\"editors\"}\r\n{\"packages_current_version\":\"7.64.0-4+deb10u6\",\"packages_candidate_version\":\"7.64.0-4+deb10u7\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libcurl4\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.1.1n-0+deb10u5\",\"packages_candidate_version\":\"1.1.1n-0+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"openssl\",\"packages_section\":\"utils\"}\r\n{\"packages_current_version\":\"2.06-3~deb10u3\",\"packages_candidate_version\":\"2.06-3~deb10u4\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"grub-common\",\"packages_section\":\"admin\"}\r\n{\"packages_current_version\":\"1.12.24-0+deb10u1\",\"packages_candidate_version\":\"1.12.28-0+deb10u1\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"dbus\",\"packages_section\":\"admin\"}\r\n{\"packages_current_version\":\"2.7.16-2+deb10u2\",\"packages_candidate_version\":\"2.7.16-2+deb10u3\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python2.7-minimal\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"0.176-1.1\",\"packages_candidate_version\":\"0.176-1.1+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libelf1\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.24.1-1\",\"packages_candidate_version\":\"1.24.1-1+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python3-urllib3\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"6.1+20181013-2+deb10u3\",\"packages_candidate_version\":\"6.1+20181013-2+deb10u4\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libncurses6\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"2.7.16-2+deb10u2\",\"packages_candidate_version\":\"2.7.16-2+deb10u3\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python2.7\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"3.7.3-2+deb10u5\",\"packages_candidate_version\":\"3.7.3-2+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python3.7\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"1:7.9p1-10+deb10u2\",\"packages_candidate_version\":\"1:7.9p1-10+deb10u3\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"openssh-sftp-server\",\"packages_section\":\"net\"}\r\n{\"packages_current_version\":\"1.17-3+deb10u5\",\"packages_candidate_version\":\"1.17-3+deb10u6\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libk5crypto3\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"2.06-3~deb10u3\",\"packages_candidate_version\":\"2.06-3~deb10u4\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"grub2-common\",\"packages_section\":\"admin\"}\r\n{\"packages_current_version\":\"6.1+20181013-2+deb10u3\",\"packages_candidate_version\":\"6.1+20181013-2+deb10u4\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libncursesw6\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"3.7.3-2+deb10u5\",\"packages_candidate_version\":\"3.7.3-2+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libpython3.7-stdlib\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"3.7.3-2+deb10u5\",\"packages_candidate_version\":\"3.7.3-2+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python3.7-minimal\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"2.06-3~deb10u3\",\"packages_candidate_version\":\"2.06-3~deb10u4\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"grub-pc-bin\",\"packages_section\":\"admin\"}\r\n{\"packages_current_version\":\"2.06-3~deb10u3\",\"packages_candidate_version\":\"2.06-3~deb10u4\",\"packages_priority\":\"extra\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"grub-efi-amd64-bin\",\"packages_section\":\"admin\"}\r\n{\"packages_current_version\":\"6.1+20181013-2+deb10u3\",\"packages_candidate_version\":\"6.1+20181013-2+deb10u4\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libtinfo6\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1.17-3+deb10u5\",\"packages_candidate_version\":\"1.17-3+deb10u6\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libkrb5support0\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"1:3.1+dfsg-8+deb10u10\",\"packages_candidate_version\":\"1:3.1+dfsg-8+deb10u11\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"qemu-utils\",\"packages_section\":\"otherosfs\"}\r\n{\"packages_current_version\":\"2:8.1.0875-5+deb10u5\",\"packages_candidate_version\":\"2:8.1.0875-5+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"vim-runtime\",\"packages_section\":\"editors\"}\r\n{\"packages_current_version\":\"2:8.1.0875-5+deb10u5\",\"packages_candidate_version\":\"2:8.1.0875-5+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"vim\",\"packages_section\":\"editors\"}\r\n{\"packages_current_version\":\"0.41+deb10u7\",\"packages_candidate_version\":\"0.41+deb10u8\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"distro-info-data\",\"packages_section\":\"devel\"}\r\n{\"packages_current_version\":\"2:8.1.0875-5+deb10u5\",\"packages_candidate_version\":\"2:8.1.0875-5+deb10u6\",\"packages_priority\":\"extra\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"xxd\",\"packages_section\":\"editors\"}\r\n{\"packages_current_version\":\"6.1+20181013-2+deb10u3\",\"packages_candidate_version\":\"6.1+20181013-2+deb10u4\",\"packages_priority\":\"required\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"ncurses-bin\",\"packages_section\":\"utils\"}\r\n{\"packages_current_version\":\"1:7.9p1-10+deb10u2\",\"packages_candidate_version\":\"1:7.9p1-10+deb10u3\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"openssh-server\",\"packages_section\":\"net\"}\r\n{\"packages_current_version\":\"1:7.9p1-10+deb10u2\",\"packages_candidate_version\":\"1:7.9p1-10+deb10u3\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"openssh-client\",\"packages_section\":\"net\"}\r\n{\"packages_current_version\":\"6.1+20181013-2+deb10u3\",\"packages_candidate_version\":\"6.1+20181013-2+deb10u4\",\"packages_priority\":\"required\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"ncurses-base\",\"packages_section\":\"misc\"}\r\n{\"packages_current_version\":\"2.7.16-2+deb10u2\",\"packages_candidate_version\":\"2.7.16-2+deb10u3\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libpython2.7-minimal\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"2:8.1.0875-5+deb10u5\",\"packages_candidate_version\":\"2:8.1.0875-5+deb10u6\",\"packages_priority\":\"important\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"vim-tiny\",\"packages_section\":\"editors\"}\r\n{\"packages_current_version\":\"1.1.1n-0+deb10u5\",\"packages_candidate_version\":\"1.1.1n-0+deb10u6\",\"packages_priority\":\"important\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libssl1.1\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"2.7.16-2+deb10u2\",\"packages_candidate_version\":\"2.7.16-2+deb10u3\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libpython2.7-stdlib\",\"packages_section\":\"python\"}\r\n{\"packages_current_version\":\"7.64.0-4+deb10u6\",\"packages_candidate_version\":\"7.64.0-4+deb10u7\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"curl\",\"packages_section\":\"web\"}\r\n{\"packages_current_version\":\"2.58.3-2+deb10u4\",\"packages_candidate_version\":\"2.58.3-2+deb10u5\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libglib2.0-0\",\"packages_section\":\"libs\"}\r\n{\"packages_current_version\":\"7.64.0-4+deb10u6\",\"packages_candidate_version\":\"7.64.0-4+deb10u7\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libcurl3-gnutls\",\"packages_section\":\"libs\"}\r\n",
        "stderr_lines": [
          "Shared connection to 20.127.230.105 closed."
        ],
        "_ansible_no_log": null,
        "stdout_lines": [
          "{\"packages_current_version\":\"3.7.3-2+deb10u5\",\"packages_candidate_version\":\"3.7.3-2+deb10u6\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libpython3.7-minimal\",\"packages_section\":\"python\"}",
          "{\"packages_current_version\":\"0.21\",\"packages_candidate_version\":\"0.21+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"python3-distro-info\",\"packages_section\":\"python\"}",
          "{\"packages_current_version\":\"1.8.0-2.1\",\"packages_candidate_version\":\"1.8.0-2.1+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libssh2-1\",\"packages_section\":\"libs\"}",
          "{\"packages_current_version\":\"1.17-3+deb10u5\",\"packages_candidate_version\":\"1.17-3+deb10u6\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libkrb5-3\",\"packages_section\":\"libs\"}",
          "{\"packages_current_version\":\"1.17-3+deb10u5\",\"packages_candidate_version\":\"1.17-3+deb10u6\",\"packages_priority\":\"standard\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libgssapi-krb5-2\",\"packages_section\":\"libs\"}",
          "{\"packages_current_version\":\"1.36.0-2+deb10u1\",\"packages_candidate_version\":\"1.36.0-2+deb10u2\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libnghttp2-14\",\"packages_section\":\"libs\"}",
          "{\"packages_current_version\":\"1.12.24-0+deb10u1\",\"packages_candidate_version\":\"1.12.28-0+deb10u1\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"libdbus-1-3\",\"packages_section\":\"libs\"}",
          "{\"packages_current_version\":\"4.19.289-1\",\"packages_candidate_version\":\"4.19.289-2\",\"packages_priority\":\"optional\",\"packages_security\":\"false\",\"timestamp\":\"11/01/2023 18:46:39\",\"hostname\":\"blob.internal.cloudapp.net\",\"packages_name\":\"linux-image-4.19.0-25-cloud-amd64\",\"packages_section\":\"kernel\"}",
        ],
        "rc": 0
      },
      "start": "2023-11-01T18:46:35.718444",
      "play": "Python-APT",
      "end": "2023-11-01T18:46:39.809042",
      "task_uuid": "d2e6a7c9-8213-8052-1940-000000000004",
      "task": "List Available Updates",
      "duration": 4.090598,
      "resolved_action": "ansible.builtin.script",
      "task_args": "",
      "playbook": "task.yml",
      "play_pattern": "all",
      "remote_addr": "20.127.230.105",
      "uuid": "6f751077-0477-4931-b446-3433bfc6c4d3",
      "task_path": "/runner/project/task.yml:4",
      "playbook_uuid": "d88a2d21-8130-48b1-9ece-c3e8f2abc13e"
    }
  }

Logstash itself is producing the following log, which makes me think that the field cannot be read.

[ERROR][logstash.agent           ] Failed to execute action {:id=>:main, :action_type=>LogStash::ConvergeResult::FailedAction, :message=>"Expected one of [ \\t\\r\\n], \"#\", \"}\" at line 11, column 29 (byte 191) after filter {\n  if [event_data][task] == \"List Available Updates\" and [event_data][res][stdout] {\n    json {\n      source => [event_data]", :backtrace=>["/usr/share/logstash/logstash-core/lib/logstash/compiler.rb:32:in `compile_imperative'", "org/logstash/execution/AbstractPipelineExt.java:239:in `initialize'", "org/logstash/execution/AbstractPipelineExt.java:173:in `initialize'", "org/jruby/RubyClass.java:911:in `new'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/reload.rb:51:in `execute'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:386:in `block in converge_state'"]}

Has anyone an idea how i can produce an index based on the script output that is stored in [event_data][res][stdout] ?

Any help i highly appreciated!

2

Did you had a different configuration and changed it? This error means that Logstash could not even start your pipeline, it shows an error on your filter.

message=>"Expected one of [ \t\r\n], "#", "}" at line 11, column 29 (byte 191) after filter {\n if [event_data][task] == "List Available Updates" and [event_data][res][stdout] {\n json {\n source => [event_data]"

The source for the json filter on this error log is different from the one you shared, and your pipeline is not running, not sure how you are receiving logs.

Do you have anything else in Logstash logs? Can you stop Logstash and start it again to get fresh logs and share them?

3

Hey, thanks for your input!

you may be right that the that the error message is not part of the configuration, as i had quite often changed the config to try out different settings.

However, I have solved it as well now.
First I needed to split the data, and then parse it as JSON. Like so.

        filter {
          if [event_data][task] == "List Available Updates" and [event_data][res][stdout] {
            split {
              field => "[event_data][res][stdout]"
              target => "parsed_message"
            }
            json {
              source => "parsed_message"
              remove_field => ["parsed_message", "event_data"]
            }
          } else {
              drop {}
            }
          }
4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.