Hello,
Logstash version: 7.17
Aggregate filter plugin: v2.10.0
I have the following input of logs:
{"@timestamp": "2023-07-27T08:40:27.849Z", "message": "Activity Stream update entry for job", "host": "tower-host", "level": "INFO", "logger_name": "awx.analytics.activity_stream", "stack_info": null, "changes": {"name": "SK_connectivity_Test", "description": "", "job_type": "run", "inventory": "SK_inv-143", "project": "SK_connectivity_test-541", "playbook": "connectivity_test.yml", "scm_branch": "", "forks": 0, "limit": "", "verbosity": 0, "extra_vars": "{}", "job_tags": "", "force_handlers": false, "skip_tags": "", "start_at_task": "", "timeout": 0, "use_fact_cache": false, "job_template": "SK_connectivity_Test-542", "allow_simultaneous": false, "instance_group": null, "diff_mode": false, "job_slice_number": 0, "job_slice_count": 1, "webhook_service": "", "webhook_credential": null, "webhook_guid": "", "id": 102684, "credentials": ["SK_Ansible (218)"], "labels": []}, "relationship": "", "actor": "sk@test.com", "operation": "create", "object1": "job", "object2": "", "summary_fields": {"job": [{"id": 102684, "name": "SK_connectivity_Test", "description": "", "status": "new", "failed": false, "elapsed": "0.000"}], "job_template": [{"id": 542, "name": "SK_connectivity_Test", "description": ""}], "actor": {"id": 79, "username": "sk@test.com", "first_name": "S", "last_name": "K"}}, "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:32.037Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "verbose", "event_data": {}, "failed": false, "changed": false, "uuid": "43fdsf-3fdsb11b9", "playbook": "", "play": "", "role": "", "task": "", "counter": 2, "stdout": "BECOME password[defaults to SSH password]: ", "verbosity": 0, "start_line": 1, "end_line": 2, "created": null, "job": 102684, "host_name": "", "parent_uuid": "", "event_display": "Verbose", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:32.545Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "playbook_on_start", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "uuid": "32134-fdfs-bdfghb-432554"}, "failed": false, "changed": false, "uuid": "32134-fdfs-bdfghb-432554", "playbook": "connectivity_test.yml", "play": "", "role": "", "task": "", "counter": 3, "stdout": "", "verbosity": 0, "start_line": 2, "end_line": 2, "created": "2023-07-27T08:40:32.542Z", "job": 102684, "host_name": "", "parent_uuid": "", "event_display": "Playbook Started", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:32.549Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "playbook_on_play_start", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "play": "connectivitiy Test", "play_uuid": "32134-fdfs-bdfghb-432554-543erfse", "play_pattern": "all", "name": "connectivitiy Test", "pattern": "all", "uuid": "32134-fdfs-bdfghb-432554-543erfse"}, "failed": false, "changed": false, "uuid": "32134-fdfs-bdfghb-432554-543erfse", "playbook": "connectivity_test.yml", "play": "connectivitiy Test", "role": "", "task": "", "counter": 4, "stdout": "\r\nPLAY [connectivitiy Test] ******************************************************", "verbosity": 0, "start_line": 2, "end_line": 4, "created": "2023-07-27T08:40:32.545Z", "job": 102684, "host_name": "", "parent_uuid": "32134-fdfs-bdfghb-432554", "event_display": "Play Started (connectivitiy Test)", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:32.569Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "playbook_on_task_start", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "play": "connectivitiy Test", "play_uuid": "32134-fdfs-bdfghb-432554-543erfse", "play_pattern": "all", "task": "ping Test", "task_uuid": "32134-gfdgddgfg-543erfse", "task_action": "wait_for_connection", "task_args": "", "task_path": "/tmp/gfdgddgfg_543erfse_resrf/awx_gfdgddgfg_543erfse/project/connectivity_test.yml:8", "name": "ping Test", "is_conditional": false, "uuid": "32134-gfdgddgfg-543erfse"}, "failed": false, "changed": false, "uuid": "32134-gfdgddgfg-543erfse", "playbook": "connectivity_test.yml", "play": "connectivitiy Test", "role": "", "task": "ping Test", "counter": 5, "stdout": "\r\nTASK [ping Test] ***************************************************************", "verbosity": 0, "start_line": 4, "end_line": 6, "created": "2023-07-27T08:40:32.564Z", "job": 102684, "host_name": "", "parent_uuid": "32134-fdfs-bdfghb-432554-543erfse", "event_display": "Task Started (ping Test)", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:32.574Z", "message": "Event data saved.", "host": 17502, "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "runner_on_start", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "play": "connectivitiy Test", "play_uuid": "32134-fdfs-bdfghb-432554-543erfse", "play_pattern": "all", "task": "ping Test", "task_uuid": "32134-gfdgddgfg-543erfse", "task_action": "wait_for_connection", "task_args": "", "task_path": "/tmp/gfdgddgfg_543erfse_resrf/awx_gfdgddgfg_543erfse/project/connectivity_test.yml:8", "host": "google.com", "uuid": "fgdgdfgd"}, "failed": false, "changed": false, "uuid": "fgdgdfgd", "playbook": "connectivity_test.yml", "play": "connectivitiy Test", "role": "", "task": "ping Test", "counter": 6, "stdout": "", "verbosity": 0, "start_line": 6, "end_line": 6, "created": "2023-07-27T08:40:32.565Z", "job": 102684, "host_name": "google.com", "parent_uuid": "32134-gfdgddgfg-543erfse", "event_display": "Host Started", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:32.624Z", "message": "Event data saved.", "host": 17452, "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "runner_on_start", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "play": "connectivitiy Test", "play_uuid": "32134-fdfs-bdfghb-432554-543erfse", "play_pattern": "all", "task": "ping Test", "task_uuid": "32134-gfdgddgfg-543erfse", "task_action": "wait_for_connection", "task_args": "", "task_path": "/tmp/gfdgddgfg_543erfse_resrf/awx_gfdgddgfg_543erfse/project/connectivity_test.yml:8", "host": "lab-host", "uuid": "fdsfs-gfdg-gfdgd-tret"}, "failed": false, "changed": false, "uuid": "fdsfs-gfdg-gfdgd-tret", "playbook": "connectivity_test.yml", "play": "connectivitiy Test", "role": "", "task": "ping Test", "counter": 7, "stdout": "", "verbosity": 0, "start_line": 6, "end_line": 6, "created": "2023-07-27T08:40:32.580Z", "job": 102684, "host_name": "lab-host", "parent_uuid": "32134-gfdgddgfg-543erfse", "event_display": "Host Started", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:40:33.799Z", "message": "Event data saved.", "host": 17452, "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "runner_on_ok", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "play": "connectivitiy Test", "play_uuid": "32134-fdfs-bdfghb-432554-543erfse", "play_pattern": "all", "task": "ping Test", "task_uuid": "32134-gfdgddgfg-543erfse", "task_action": "wait_for_connection", "task_args": "", "task_path": "/tmp/gfdgddgfg_543erfse_resrf/awx_gfdgddgfg_543erfse/project/connectivity_test.yml:8", "host": "lab-host", "remote_addr": "lab-host", "res": {"elapsed": 1, "_ansible_no_log": false, "changed": false}, "start": "2023-07-27T08:40:32.580441", "end": "2023-07-27T08:40:33.796096", "duration": 1.215655, "event_loop": null, "uuid": "0e75c038-6131-485c-beed-055077cfe5b0"}, "failed": false, "changed": false, "uuid": "0e75c038-6131-485c-beed-055077cfe5b0", "playbook": "connectivity_test.yml", "play": "connectivitiy Test", "role": "", "task": "ping Test", "counter": 8, "stdout": "\u001b[0;32mok: [lab-host]\u001b[0m", "verbosity": 0, "start_line": 6, "end_line": 7, "created": "2023-07-27T08:40:33.796Z", "job": 102684, "host_name": "lab-host", "parent_uuid": "32134-gfdgddgfg-543erfse", "event_display": "Host OK", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:42:57.067Z", "message": "Event data saved.", "host": 17502, "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "runner_on_failed", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "play": "connectivitiy Test", "play_uuid": "32134-fdfs-bdfghb-432554-543erfse", "play_pattern": "all", "task": "ping Test", "task_uuid": "32134-gfdgddgfg-543erfse", "task_action": "wait_for_connection", "task_args": "", "task_path": "/tmp/gfdgddgfg_543erfse_resrf/awx_gfdgddgfg_543erfse/project/connectivity_test.yml:8", "host": "google.com", "remote_addr": "google.com", "res": {"msg": "timed out waiting for ping module test success: Failed to connect to the host via ssh: socket: Address family not supported by protocol\r\nsocket: Address family not supported by protocol\r\nsocket: Address family not supported by protocol\r\nsocket: Address family not supported by protocol\r\nssh: connect to host google.com port 22: failure", "elapsed": 144, "_ansible_no_log": false, "changed": false}, "start": "2023-07-27T08:40:32.565183", "end": "2023-07-27T08:42:57.062821", "duration": 144.497638, "ignore_errors": null, "event_loop": null, "uuid": "6014373b-7486-4696-932e-6d0dae260848"}, "failed": true, "changed": false, "uuid": "6014373b-7486-4696-932e-6d0dae260848", "playbook": "connectivity_test.yml", "play": "connectivitiy Test", "role": "", "task": "ping Test", "counter": 9, "stdout": "\u001b[0;31mfatal: [google.com]: FAILED! => {\"changed\": false, \"elapsed\": 144, \"msg\": \"timed out waiting for ping module test success: Failed to connect to the host via ssh: socket: Address family not supported by protocol\\r\\nsocket: Address family not supported by protocol\\r\\nsocket: Address family not supported by protocol\\r\\nsocket: Address family not supported by protocol\\r\\nssh: connect to host google.com port 22: failure\"}\u001b[0m", "verbosity": 0, "start_line": 7, "end_line": 8, "created": "2023-07-27T08:42:57.062Z", "job": 102684, "host_name": "google.com", "parent_uuid": "32134-gfdgddgfg-543erfse", "event_display": "Host Failed", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:42:57.072Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "warning", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "warning": true, "uuid": "cdaea5b4-86ff-4884-b46f-1c414a1173ea"}, "failed": false, "changed": false, "uuid": "cdaea5b4-86ff-4884-b46f-1c414a1173ea", "playbook": "connectivity_test.yml", "play": "", "role": "", "task": "", "counter": 11, "stdout": "\u001b[1;35m[WARNING]: Failure using method (v2_playbook_on_stats) in callback plugin\u001b[0m\r\n\u001b[1;35m(<ansible.plugins.callback.snow-callback.py.CallbackModule object at\u001b[0m\r\n\u001b[1;35m0x7fa918853940>): '_run'\u001b[0m", "verbosity": 0, "start_line": 13, "end_line": 16, "created": "2023-07-27T08:42:57.067Z", "job": 102684, "host_name": "", "parent_uuid": "32134-fdfs-bdfghb-432554", "event_display": "Warning", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:42:57.191Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "playbook_on_stats", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "changed": {}, "dark": {}, "failures": {"google.com": 1}, "ignored": {}, "ok": {"lab-host": 1}, "processed": {"lab-host": 1, "google.com": 1}, "rescued": {}, "skipped": {}, "artifact_data": {}, "uuid": "5480fce1-13e8-40c5-8708-b8d0c0563b7c"}, "failed": true, "changed": false, "uuid": "5480fce1-13e8-40c5-8708-b8d0c0563b7c", "playbook": "connectivity_test.yml", "play": "", "role": "", "task": "", "counter": 10, "stdout": "\r\nPLAY RECAP *********************************************************************\r\n\u001b[0;31mgoogle.com\u001b[0m : ok=0 changed=0 unreachable=0 \u001b[0;31mfailed=1 \u001b[0m skipped=0 rescued=0 ignored=0 \r\n\u001b[0;32mlab-host\u001b[0m : \u001b[0;32mok=1 \u001b[0m changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 \r\n", "verbosity": 0, "start_line": 8, "end_line": 13, "created": "2023-07-27T08:42:57.066Z", "job": 102684, "host_name": "", "parent_uuid": "32134-fdfs-bdfghb-432554", "event_display": "Playbook Complete", "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:42:57.311Z", "message": "Event processing is finished for Job 102684, sending notifications", "host": "tower-host", "level": "INFO", "logger_name": "awx.main.commands.run_callback_receiver", "stack_info": null, "cluster_host_id": "tower-host", "tower_uuid": null}
{"@timestamp": "2023-07-27T08:42:58.329Z", "message": "job 102684 (failed) encountered an error (rc=2), please see task stdout for details.", "host": "tower-host", "level": "WARNING", "logger_name": "awx.main.dispatch", "stack_info": null, "cluster_host_id": "tower-host", "tower_uuid": null}
The task is to create an aggregated log that would contain the following fields:
- Start_time
- the timestamp from the first log:
{"@timestamp": "2023-07-27T08:40:27.849Z", "message": "Activity Stream update entry for job", "host": "tower-host", "level": "INFO", "logger_name": "awx.analytics.activity_stream", "stack_info": null, "changes": {"name": "SK_connectivity_Test", "description": "", "job_type": "run", "inventory": "SK_inv-143", "project": "SK_connectivity_test-541", "playbook": "connectivity_test.yml", "scm_branch": "", "forks": 0, "limit": "", "verbosity": 0, "extra_vars": "{}", "job_tags": "", "force_handlers": false, "skip_tags": "", "start_at_task": "", "timeout": 0, "use_fact_cache": false, "job_template": "SK_connectivity_Test-542", "allow_simultaneous": false, "instance_group": null, "diff_mode": false, "job_slice_number": 0, "job_slice_count": 1, "webhook_service": "", "webhook_credential": null, "webhook_guid": "", "id": 102684, "credentials": ["SK_Ansible (218)"], "labels": []}, "relationship": "", "actor": "sk@test.com", "operation": "create", "object1": "job", "object2": "", "summary_fields": {"job": [{"id": 102684, "name": "SK_connectivity_Test", "description": "", "status": "new", "failed": false, "elapsed": "0.000"}], "job_template": [{"id": 542, "name": "SK_connectivity_Test", "description": ""}], "actor": {"id": 79, "username": "sk@test.com", "first_name": "S", "last_name": "K"}}, "cluster_host_id": "tower-host", "tower_uuid": null}
In the logstash configuration that will follow I didn't include a map for Start_time because I didn't know how.
- actor, changes
- extracted also from the first log:
{"@timestamp": "2023-07-27T08:40:27.849Z", "message": "Activity Stream update entry for job", "host": "tower-host", "level": "INFO", "logger_name": "awx.analytics.activity_stream", "stack_info": null, "changes": {"name": "SK_connectivity_Test", "description": "", "job_type": "run", "inventory": "SK_inv-143", "project": "SK_connectivity_test-541", "playbook": "connectivity_test.yml", "scm_branch": "", "forks": 0, "limit": "", "verbosity": 0, "extra_vars": "{}", "job_tags": "", "force_handlers": false, "skip_tags": "", "start_at_task": "", "timeout": 0, "use_fact_cache": false, "job_template": "SK_connectivity_Test-542", "allow_simultaneous": false, "instance_group": null, "diff_mode": false, "job_slice_number": 0, "job_slice_count": 1, "webhook_service": "", "webhook_credential": null, "webhook_guid": "", "id": 102684, "credentials": ["SK_Ansible (218)"], "labels": []}, "relationship": "", "actor": "sk@test.com", "operation": "create", "object1": "job", "object2": "", "summary_fields": {"job": [{"id": 102684, "name": "SK_connectivity_Test", "description": "", "status": "new", "failed": false, "elapsed": "0.000"}], "job_template": [{"id": 542, "name": "SK_connectivity_Test", "description": ""}], "actor": {"id": 79, "username": "sk@test.com", "first_name": "S", "last_name": "K"}}, "cluster_host_id": "tower-host", "tower_uuid": null}
- event_data.task, stdout
- extracted from logs that have logger_name:awx.analytics.job_events and event:runner_on_failed
- End_time
- the timestamp extracted from the last log which may be one of the following:
{"@timestamp": "2023-07-27T08:42:57.191Z", "message": "Event data saved.", "host": "", "level": "INFO", "logger_name": "awx.analytics.job_events", "id": null, "modified": null, "event": "playbook_on_stats", "event_data": {"playbook": "connectivity_test.yml", "playbook_uuid": "32134-fdfs-bdfghb-432554", "changed": {}, "dark": {}, "failures": {"google.com": 1}, "ignored": {}, "ok": {"lab-host": 1}, "processed": {"lab-host": 1, "google.com": 1}, "rescued": {}, "skipped": {}, "artifact_data": {}, "uuid": "5480fce1-13e8-40c5-8708-b8d0c0563b7c"}, "failed": true, "changed": false, "uuid": "5480fce1-13e8-40c5-8708-b8d0c0563b7c", "playbook": "connectivity_test.yml", "play": "", "role": "", "task": "", "counter": 10, "stdout": "\r\nPLAY RECAP *********************************************************************\r\n\u001b[0;31mgoogle.com\u001b[0m : ok=0 changed=0 unreachable=0 \u001b[0;31mfailed=1 \u001b[0m skipped=0 rescued=0 ignored=0 \r\n\u001b[0;32mlab-host\u001b[0m : \u001b[0;32mok=1 \u001b[0m changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 \r\n", "verbosity": 0, "start_line": 8, "end_line": 13, "created": "2023-07-27T08:42:57.066Z", "job": 102684, "host_name": "", "parent_uuid": "32134-fdfs-bdfghb-432554", "event_display": "Playbook Complete", "cluster_host_id": "tower-host", "tower_uuid": null}
or
{"@timestamp": "2023-07-27T08:42:57.311Z", "message": "Event processing is finished for Job 102684, sending notifications", "host": "tower-host", "level": "INFO", "logger_name": "awx.main.commands.run_callback_receiver", "stack_info": null, "cluster_host_id": "tower-host", "tower_uuid": null}
or
{"@timestamp": "2023-07-27T08:42:58.329Z", "message": "job 102684 (failed) encountered an error (rc=2), please see task stdout for details.", "host": "tower-host", "level": "WARNING", "logger_name": "awx.main.dispatch", "stack_info": null, "cluster_host_id": "tower-host", "tower_uuid": null}
The logstash configuration that I have:
input {
file {
path => "/var/log/logstash/input.log"
type => "tower-logs"
}
}
filter {
if [type] == "tower-logs" {
fingerprint {
method => "SHA1"
key => "KEY"
}
mutate {
add_field => { "original_message" => "%{message}" }
}
json {
source => "message"
target => "linca"
}
if [linca][summary_fields][job][0][id] {
mutate {
add_field => { "job_id" => "%{[linca][summary_fields][job][0][id]}" }
}
}
if [linca][job] {
mutate {
add_field => { "job_id" => "%{[linca][job]}" }
}
}
if [linca][logger_name] == "awx.main.commands.run_callback_receiver" {
grok {
match => {"message" => "%{GREEDYDATA}\s+Job\s+%{NUMBER:job_id}\,\s+%{GREEDYDATA}"}
}
}
if [linca][logger_name] == "awx.main.dispatch" {
grok {
match => {"message" => "job\s+%{NUMBER:job_id}\s+%{GREEDYDATA}"}
}
}
aggregate {
task_id => "%{job_id}"
code => '
map["actor_new"] ||= ""
map["changesId"] ||= ""
map["changesJobTemplate"] ||= ""
map["changesName"] ||= ""
map["Task"] ||= []
map["stdout_new"] ||= []
if [linca][logger_name] == "awx.analytics.activity_stream"
map["actor_new"] = event.get("[linca][actor]")
map["changesId"] = event.get("[linca][changes][id]")
map["changesJobTemplate"] = event.get("[linca][changes][job_template]")
map["changesName"] = event.get("[linca][changes][name]")
end
if [linca][logger_name] == "awx.analytics.job_events" and [linca][event] == "runner_on_failed"
map["Task"] << event.get("[linca][event_data][task]")
map["stdout_new"] << event.get("[linca][stdout]")
end
'
timeout => 300
push_map_as_event_on_timeout => true
timeout_task_id_field => "[linca][job]"
}
}
}
output {
file {
path => "/var/log/logstash/tavi.log"
}
}
The exception that I see in /var/log/logstash/logstash-plain.log is:
[2023-09-05T07:44:06,328][ERROR][logstash.filters.aggregate][main][b4676d599be5e0e8477e4c9816d0a8f8a040f19ba53af726d893ab7617d1399d] Aggregate exception occurred {:error=>#<NameError: undefined local variable or method `linca' for #<LogStash::Filters::Aggregate:0x59a95202>>, :code=>"\n map[\"actor_new\"] ||= \"\"\n map[\"changesId\"] ||= \"\"\n map[\"changesJobTemplate\"] ||= \"\"\n map[\"changesName\"] ||= \"\"\n map[\"Task\"] ||= []\n map[\"stdout_new\"] ||= []\n if [linca][logger_name] == \"awx.analytics.activity_stream\" \n map[\"actor_new\"] = event.get(\"[linca][actor]\")\n map[\"changesId\"] = event.get(\"[linca][changes][id]\")\n map[\"changesJobTemplate\"] = event.get(\"[linca][changes][job_template]\")\n map[\"changesName\"] = event.get(\"[linca][changes][name]\")\n end\n if [linca][logger_name] == \"awx.analytics.job_events\" and [linca][event] == \"runner_on_failed\"\n map[\"Task\"] << event.get(\"[linca][event_data][task]\") \n map[\"stdout_new\"] << event.get(\"[linca][stdout]\") \n end\n ", :map=>{"actor_new"=>"", "changesId"=>"", "changesJobTemplate"=>"", "changesName"=>"", "Task"=>[], "stdout_new"=>[]}, :event_data=>{"path"=>"/var/log/logstash/input.log", "@timestamp"=>2023-09-05T07:44:06.208Z, "job_id"=>"102684", "@metadata"=>{"host"=>"ip-10-7-188-157.ec2.internal", "path"=>"/var/log/logstash/input.log"}, "host"=>"ip-10-7-188-157.ec2.internal", "@version"=>"1", "fingerprint"=>"86db06a958b73eaff2ce62d52e3d4caf2dd3da98", "original_message"=>"{\"@timestamp\": \"2023-07-27T08:40:27.849Z\", \"message\": \"Activity Stream update entry for job\", \"host\": \"tower-host\", \"level\": \"INFO\", \"logger_name\": \"awx.analytics.activity_stream\", \"stack_info\": null, \"changes\": {\"name\": \"SK_connectivity_Test\", \"description\": \"\", \"job_type\": \"run\", \"inventory\": \"SK_inv-143\", \"project\": \"SK_connectivity_test-541\", \"playbook\": \"connectivity_test.yml\", \"scm_branch\": \"\", \"forks\": 0, \"limit\": \"\", \"verbosity\": 0, \"extra_vars\": \"{}\", \"job_tags\": \"\", \"force_handlers\": false, \"skip_tags\": \"\", \"start_at_task\": \"\", \"timeout\": 0, \"use_fact_cache\": false, \"job_template\": \"SK_connectivity_Test-542\", \"allow_simultaneous\": false, \"instance_group\": null, \"diff_mode\": false, \"job_slice_number\": 0, \"job_slice_count\": 1, \"webhook_service\": \"\", \"webhook_credential\": null, \"webhook_guid\": \"\", \"id\": 102684, \"credentials\": [\"SK_Ansible (218)\"], \"labels\": []}, \"relationship\": \"\", \"actor\": \"sk@test.com\", \"operation\": \"create\", \"object1\": \"job\", \"object2\": \"\", \"summary_fields\": {\"job\": [{\"id\": 102684, \"name\": \"SK_connectivity_Test\", \"description\": \"\", \"status\": \"new\", \"failed\": false, \"elapsed\": \"0.000\"}], \"job_template\": [{\"id\": 542, \"name\": \"SK_connectivity_Test\", \"description\": \"\"}], \"actor\": {\"id\": 79, \"username\": \"sk@test.com\", \"first_name\": \"S\", \"last_name\": \"K\"}}, \"cluster_host_id\": \"tower-host\", \"tower_uuid\": null}", "message"=>"{\"@timestamp\": \"2023-07-27T08:40:27.849Z\", \"message\": \"Activity Stream update entry for job\", \"host\": \"tower-host\", \"level\": \"INFO\", \"logger_name\": \"awx.analytics.activity_stream\", \"stack_info\": null, \"changes\": {\"name\": \"SK_connectivity_Test\", \"description\": \"\", \"job_type\": \"run\", \"inventory\": \"SK_inv-143\", \"project\": \"SK_connectivity_test-541\", \"playbook\": \"connectivity_test.yml\", \"scm_branch\": \"\", \"forks\": 0, \"limit\": \"\", \"verbosity\": 0, \"extra_vars\": \"{}\", \"job_tags\": \"\", \"force_handlers\": false, \"skip_tags\": \"\", \"start_at_task\": \"\", \"timeout\": 0, \"use_fact_cache\": false, \"job_template\": \"SK_connectivity_Test-542\", \"allow_simultaneous\": false, \"instance_group\": null, \"diff_mode\": false, \"job_slice_number\": 0, \"job_slice_count\": 1, \"webhook_service\": \"\", \"webhook_credential\": null, \"webhook_guid\": \"\", \"id\": 102684, \"credentials\": [\"SK_Ansible (218)\"], \"labels\": []}, \"relationship\": \"\", \"actor\": \"sk@test.com\", \"operation\": \"create\", \"object1\": \"job\", \"object2\": \"\", \"summary_fields\": {\"job\": [{\"id\": 102684, \"name\": \"SK_connectivity_Test\", \"description\": \"\", \"status\": \"new\", \"failed\": false, \"elapsed\": \"0.000\"}], \"job_template\": [{\"id\": 542, \"name\": \"SK_connectivity_Test\", \"description\": \"\"}], \"actor\": {\"id\": 79, \"username\": \"sk@test.com\", \"first_name\": \"S\", \"last_name\": \"K\"}}, \"cluster_host_id\": \"tower-host\", \"tower_uuid\": null}", "type"=>"tower-logs", "linca"=>{"object2"=>"", "object1"=>"job", "level"=>"INFO", "changes"=>{"instance_group"=>nil, "skip_tags"=>"", "job_type"=>"run", "force_handlers"=>false, "credentials"=>["SK_Ansible (218)"], "job_template"=>"SK_connectivity_Test-542", "description"=>"", "project"=>"SK_connectivity_test-541", "inventory"=>"SK_inv-143", "timeout"=>0, "diff_mode"=>false, "job_slice_number"=>0, "limit"=>"", "id"=>102684, "start_at_task"=>"", "playbook"=>"connectivity_test.yml", "forks"=>0, "allow_simultaneous"=>false, "webhook_guid"=>"", "use_fact_cache"=>false, "job_tags"=>"", "webhook_service"=>"", "labels"=>[], "extra_vars"=>"{}", "job_slice_count"=>1, "webhook_credential"=>nil, "scm_branch"=>"", "name"=>"SK_connectivity_Test", "verbosity"=>0}, "summary_fields"=>{"actor"=>{"last_name"=>"K", "id"=>79, "first_name"=>"S", "username"=>"sk@test.com"}, "job_template"=>[{"name"=>"SK_connectivity_Test", "description"=>"", "id"=>542}], "job"=>[{"elapsed"=>"0.000", "name"=>"SK_connectivity_Test", "description"=>"", "failed"=>false, "id"=>102684, "status"=>"new"}]}, "message"=>"Activity Stream update entry for job", "actor"=>"sk@test.com", "@timestamp"=>"2023-07-27T08:40:27.849Z", "cluster_host_id"=>"tower-host", "host"=>"tower-host", "stack_info"=>nil, "logger_name"=>"awx.analytics.activity_stream", "relationship"=>"", "operation"=>"create", "tower_uuid"=>nil}}}
Could you please point me to where I am doing something wrong? Thank you!