Continuing the discussion from Add field from JSON / logstash filter:
Hi there,
I am having difficulty extracting a field from a log message using the json filter, I hope you can help.
I am using logstash 5.2.1, the logs come from Journalbeat. This is my configuration:
input {
beats {
port => 5044
}
}
filter {
json {
source => "message"
skip_on_invalid_json => true
remove_field => [ "timestamp" ]
add_field => {
"env" => "%{[_source][fields][environment]}"
"product" => "%{[_source][fields][product]}"
"service" => "%{[_source][fields][service]}"
"team" => "%{[_source][fields][team]}"
}
}
}
output {
elasticsearch {
hosts => ["http://elasticsearchhost:9200"]
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
The incoming log is like:
{
"_index": "journalbeat-2017.05.03",
"_type": "journal",
"_id": "AVvOb3kfj92j5Xhsa2ux",
"_score": null,
"_source": {
"gid": "6007",
"syslog_identifier": "npm",
"pid": "33943",
"type": "journal",
"uid": "1054",
"hostname": "Web01",
"cmdline": "npm ",
"exe": "/usr/local/bin/node",
"boot_id": "5a9d8d2b5428",
"@version": "1",
"beat": {
"hostname": "Web01",
"name": "journalbeat",
"version": "5.1.1"
},
"host": "Web01",
"systemd_slice": "system.slice",
"comm": "npm",
"syslog_facility": "3",
"input_type": "journal",
"machine_id": "5dbf4ddf84a14008b3e56a68cd9a02c0",
"transport": "stdout",
"message": "{"level":"ERROR","message":"Error: Page /version/12345678 does not exist\n at new NotFoundError (/opt/frontend/src/main/errors.ts:9:5)\n at /opt/frontend/src/main/app.ts:83:8\n at Layer.handle [as handle_request] (/opt/frontend/node_modules/express/lib/router/layer.js:95:5)\n at trim_prefix (/opt/frontend/node_modules/express/lib/router/index.js:317:13)\n at /opt/frontend/node_modules/express/lib/router/index.js:284:7\n ","rootRequestId":"","requestId":"","originRequestId":"","responseCode":"","fields":[],"timestamp":"2017-05-03T13:11:03+00:00","type":"nodejs","microservice":"frontend","team":"cmc","environment":"development","hostname":"Web01"}",
"priority": "6",
"tags": [
"application",
"beats_input_codec_plain_applied"
],
"@timestamp": "2017-05-03T13:11:03.695Z",
"systemd_unit": "cmc-citizen-frontend.service",
"systemd_cgroup": "/system.slice/frontend.service",
"selinux_context": "system_u:system_r:init_t:s0",
"cap_effective": "0",
"fields": {
"product": "myproduct",
"environment": "dev",
"service": "web",
"team": "myteam"
}
},
..
I need to extract the fields:
"fields": {
"product": "myproduct",
"environment": "dev",
"service": "web",
"team": "myteam"
}
and add them to the output message.
What i get is the actual string like:
"fields.team": "%{[_source][fields][team]}",
not the value of the field.team
Thanks in advance
Laura