Logstash not parsing json properly

Elastic Stack Logstash
1 
2018-05-29T06:25:02.589432+00:00 ovthick_121 bpradhcp: {"msg": "200 GET /api/v1/sessions?fields=id%2Cchildren HTTP/1.1 reqid=UBRGU4FRB7FDXBB6U4EYGE763UE", "namespace": "rasdk.rest.logger", "http_query": "fields=id%2Cchildren", "msg_id": "e912b173eb9d4520b39c2bde2dd7e906", "http_method": "GET", "duration": 0.0007190704345703125, "src_ip": "172.16.0.51", "http_path": "/api/v1/sessions", "http_status": "200", "priority": 6, "pid": 1, "tid": 140331636516608, "code_file": "/usr/local/lib/python2.7/dist-packages/rasdk/rest/logger.py", "code_line": 80, "code_func": "_log_request", "timestamp": "2018-05-29T06:25:02.589005Z", "app": "bpradhcp", "app_instance": "0", "host": "ovthick_121", "container": "660389ac29b9"}

Tried to parse the above log using grok, json and mutate.

input {
	stdin {
	}
}

filter {
	grok {
		match => {"message" => '%{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:hostname} "%{WORD:container_name}\: %{GREEDYDATA:unparsedjson}"'}
	}

	json {
		source => "unparsedjson"
		target => "parsedjson"
		#remove_field => ["unparsedjson"]
	}
	mutate {
		add_field => {
			"log_message" => "%{[parsedjson][msg]}"
			"http_status_code" => "%{[parsedjson][http_status]}" 
		}
	}
}

output {
	stdout {
		codec => rubydebug
	}
}

But it's giving me an error.

{
                "host" => "HAM-VIAGGARW-02",
                "tags" => [
        [0] "_grokparsefailure"
    ],
          "@timestamp" => 2018-05-29T16:24:56.049Z,
             "message" => "2018-05-29T06:25:02.589432+00:00 ovthick_121 bpradhcp: {\"msg\": \"200 GET /api/v1/sessions?fields=id%2Cchildren HTTP/1.1 reqid=UBRGU4FRB7FDXBB6U4EYGE763UE\", \"namespace\": \"rasdk.rest.logger\", \"http_query\": \"fields=id%2Cchildren\", \"msg_id\": \"e912b173eb9d4520b39c2bde2dd7e906\", \"http_method\": \"GET\", \"duration\": 0.0007190704345703125, \"src_ip\": \"172.16.0.51\", \"http_path\": \"/api/v1/sessions\", \"http_status\": \"200\", \"priority\": 6, \"pid\": 1, \"tid\": 140331636516608, \"code_file\": \"/usr/local/lib/python2.7/dist-packages/rasdk/rest/logger.py\", \"code_line\": 80, \"code_func\": \"_log_request\", \"timestamp\": \"2018-05-29T06:25:02.589005Z\", \"app\": \"bpradhcp\", \"app_instance\": \"0\", \"host\": \"ovthick_121\", \"container\": \"660389ac29b9\"}",
         "log_message" => "%{[parsedjson][msg]}",
    "http_status_code" => "%{[parsedjson][http_status]}",
            "@version" => "1"
}
{
                "host" => "HAM-VIAGGARW-02",
                "tags" => [
        [0] "_grokparsefailure"
    ],
          "@timestamp" => 2018-05-29T16:24:56.048Z,
             "message" => "",
         "log_message" => "%{[parsedjson][msg]}",
    "http_status_code" => "%{[parsedjson][http_status]}",
            "@version" => "1"
}

Can anyone please help me to understand what I am doing wrong here?

2

Your pattern expects a double quote before container_name and after unparsedjson, but your data does not show that. Also, a HOSTNAME consists of groups of letters, numbers, and hyphens. It does not contain underscores. NOTSPACE would work there.

3

Finally I used this one to complete my task

input {
	stdin {
	}
}

filter {
	grok {
		match => {"message" => '%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:hostname} %{WORD:container_name}\: %{GREEDYDATA:unparsedjson}'}
	}

	json {
		source => "unparsedjson"
		target => "parsedjson"
		remove_field => ["unparsedjson", "message", "@version", "@timestamp"]
	}
	mutate {
		convert => {
		"[parsedjson][http_status]" => "integer"
		"[parsedjson][app_instance]" => "integer"}
	}
}

output {
	stdout {
		codec => rubydebug
	}
}

and output is

{
          "hostname" => "ovthick_121",
         "timestamp" => "2018-05-29T06:25:02.584783+00:00",
    "container_name" => "bpradhcp",
              "host" => "HAM-VIAGGARW-02",
        "parsedjson" => {
                 "app" => "bpradhcp",
            "duration" => 0.0007190704345703125,
           "code_file" => "/usr/local/lib/python2.7/dist-packages/rasdk/rest/logger.py",
           "namespace" => "rasdk.rest.logger",
           "code_line" => 80,
           "timestamp" => "2018-05-29T06:25:02.589005Z",
           "container" => "660389ac29b9",
                 "pid" => 1,
           "code_func" => "_log_request",
              "msg_id" => "e912b173eb9d4520b39c2bde2dd7e906",
         "http_method" => "GET",
              "src_ip" => "172.16.0.51",
                 "tid" => 140331636516608,
        "app_instance" => 0,
           "http_path" => "/api/v1/sessions",
                 "msg" => "200 GET /api/v1/sessions?fields=id%2Cchildren HTTP/1.1 reqid=UBRGU4FRB7FDXBB6U4EYGE763UE",
         "http_status" => 200,
          "http_query" => "fields=id%2Cchildren",
            "priority" => 6,
                "host" => "ovthick_121"
    }
}
4

Appreciate your help :slight_smile:

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.