Hi, I am really new to logstash and was spending quite a few hours working on this but couldn't make any progress so hopefully, someone can point me out in some correct direction.
The event eventually in the kabana is like this:
{
"_index": "logstash-2021.03.11",
"_type": "_doc",
"_id": "8ep7I3gBigcDOtaplgHA",
"_version": 1,
"_score": null,
"_source": {
"severity": "INFO",
"agent": {
"version": "7.10.2",
"name": "gitlab2-be",
"ephemeral_id": "1e93d37d-de86-479a-955c-513277bd3c0d",
"type": "filebeat",
"id": "8060cd79-3aed-41f4-886c-f1c2e84ffa29",
"hostname": "gitlab2-be"
},
"tags": [
"beats_input_raw_event"
],
"gitaly_duration_s": 0.007361,
"gitaly_calls": 1,
"remote_ip": "150.203.212.84, 150.203.212.84",
"log_id": "api_json",
"log": {
"file": {
"path": "/var/log/gitlab/gitlab-rails/api_json.log"
},
"offset": 99111208
},
"redis_calls": 2,
"@timestamp": "2021-03-11T22:50:04.249Z",
"method": "GET",
"host": {
"name": "gitlab2-be"
},
"meta.user": "engn8535-2021-marker",
"route": "/api/:version/projects/:id/repository/commits",
"meta.project": "u7006861/2021-s1-lab-1",
"path": "/api/v4/projects/115231/repository/commits",
"log_group": "gitlab_teach",
"duration_s": 0.03516,
"meta.caller_id": "/api/:version/projects/:id/repository/commits",
"input": {
"type": "log"
},
"user_id": 7725,
"params": [
{
"key": "ref_name",
"value": "master"
}
],
"time": "2021-03-11T22:50:04.249Z",
"ecs": {
"version": "1.6.0"
},
"status": 200,
"redis_duration_s": 0.000883,
"db_duration_s": 0.0106,
"view_duration_s": 0.02456,
"queue_duration_s": 0.004865,
"meta.root_namespace": "u7006861",
"@version": "1",
"correlation_id": "rQA6wX9CNba",
"username": "engn8535-2021-marker",
"ua": "python-gitlab/2.6.0"
},
"fields": {
"@timestamp": [
"2021-03-11T22:50:04.249Z"
],
"time": [
"2021-03-11T22:50:04.249Z"
]
},
"sort": [
1615503004249
]
}
There is a field called params
"params": [
{
"key": "ref_name",
"value": "master"
}
],
which I hope to transfer to json so that I can extract key and value in the chart.
I used the following in the logstash config file. When I used date, geoip, mutate filter, they were all working okay. Only json filter can't work as I expected.
filter {
if [log_group] =~ "gitlab" {
date {
match => [ "time", "ISO8601" ]
target => "@timestamp"
}
if [log_id] =~ "production_json" {
geoip {
source => "remote_ip"
}
}
if [log_id] =~ "api_json" {
json {
source => "params"
target => "params_json"
skip_on_invalid_json => true
}
}
if [log_id] == "diskusage_json" {
mutate {
convert => {
"spaceavail" => "integer"
"spacetotal" => "integer"
}
}
}
}
}
By using JSON filter, I would expect there could be a new field called "params_json", but this never happens. I set up skip_on_invalid_json as true so this event should be field should be filtered correctly?
