Parse csv data wrapped in "message" from filebeat in logstash

Elastic Stack Logstash
1

Hi
i'm using filebeat to send jmeter logs to logstash but i have a problem
log data are wrapped in "message" key and the value is a csv formatted text
i tried to unwrap data with CSV filter plugin in logstash.
i can parse headers but i couldnt parse the values

this is my filter code:
if [fields][app_name] == "jmeter-tests" {
csv {
columns => ["timeStamp","elapsed","label","responseCode","responseMessage","threadName","dataType","success","failureMessage","bytes","sentBytes","grpThreads","allThreads","Latency","IdleTime","Connect"]
separator => ","
convert => {
"elapsed" => "integer"
"success" => "boolean"
"bytes" => "integer"
"sentBytes" => "integer"
"grpThreads" => "integer"
"allThreads" => "integer"
"Latency" => "integer"
"IdleTime" => "integer"
"Connect" => "integer"
}
}
}

and my outputs is something like this:
{
"_index": "jmeter-2020.01.28",
"_type": "doc",
"_id": "AFUU7G8Bo7Kec4CTnddm",
"_score": 1,
"_source": {
"grpThreads": "grpThreads",
"dataType": "dataType",
"timeStamp": "timeStamp",
"tags": [
"beats_input_codec_plain_applied"
],
"source": "/home/p188-mobiletradingpro-stresstest/stresstest/logs/999-326-jmeter-Remain-services-GET-Gateway.org.jmx-run.jmx-2020-01-28--1523-IRST.log",
"Latency": "Latency",
"label": "label",
"host": "server10",
"threadName": "threadName",
"fields": {
"app_name": "jmeter-tests"
},
"responseCode": "responseCode",
"responseMessage": "responseMessage",
"sentBytes": "sentBytes",
"message": "timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,Latency,IdleTime,Connect\n1580212435610,240,remain,200,,remain 1-4,text,true,,457,323,6,6,240,0,166\n1580212435506,342,remain,200,,remain 1-3,text,true,,457,323,6,6,342,0,269\n1580212435391,461,remain,200,,remain 1-1,text,true,,457,323,6,6,461,0,385\n1580212435401,450,remain,200,,remain 1-2,text,true,,457,323,6,6,450,0,375\n1580212435710,140,remain,200,,remain 1-5,text,true,,457,323,6,6,140,0,65\n1580212435827,48,remain,200,,remain 1-6,text,true,,457,323,1,1,48,0,6\n1580212435922,49,remain,200,,remain 1-7,text,true,,457,323,1,1,49,0,8\n1580212436027,54,remain,200,,remain 1-8,text,true,,457,323,1,1,54,0,13\n1580212436134,48,remain,200,,remain 1-9,text,true,,457,323,1,1,48,0,6\n1580212436242,51,remain,200,,remain 1-10,text,true,,457,323,1,1,51,0,5",
"@version": "1",
"@timestamp": "2020-01-28T12:16:04.036Z",
"success": "success",
"beat": {
"version": "6.0.1",
"name": "server10",
"hostname": "server10"
},
"failureMessage": "failureMessage",
"allThreads": "allThreads",
"bytes": "bytes",
"offset": 880,
"IdleTime": "IdleTime",
"elapsed": "elapsed",
"Connect": "Connect",
"prospector": {
"type": "log"
}
}
}

can anyone help pls.
tnx

2

The [message] field contains multiple lines. You can use a split filter to split that into multiple events. The csv should then work the way you want.

3

can you explain more please or give me an example
I would be grateful

4

Try

split { field => "message" }
5

i did it but nothing changes:

{
  "took": 1,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 1,
    "max_score": 1,
    "hits": [
      {
        "_index": "jmeter-2020.02.01",
        "_type": "doc",
        "_id": "BJGz_28Bo7Kec4CTDu3-",
        "_score": 1,
        "_source": {
          "host": "server10",
          "fields": {
            "app_name": "jmeter-tests"
          },
          "timeStamp": "timeStamp",
          "message": "timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,Latency,IdleTime,Connect\n1580542781100,127350,remain,Non HTTP response code: java.net.ConnectException,Non HTTP response message: Connection timed out,remain 1-1,text,false,\"Test failed: text expected to contain /{\"\"successful\"\":true/\",2039,0,1,1,0,0,127350",
          "tags": [
            "beats_input_codec_plain_applied"
          ],
          "@version": "1",
          "@timestamp": "2020-02-01T07:41:54.619Z",
          "source": "/home/p188-mobiletradingpro-stresstest/stresstest/logs/999-326-jmeter-Remain-services-GET-Gateway.org.jmx-run.jmx-2020-02-01--1107-IRST.log",
          "beat": {
            "version": "6.0.1",
            "name": "server10",
            "hostname": "server10"
          },
          "offset": 392,
          "elapsed": "elapsed",
          "prospector": {
            "type": "log"
          },
          "responseMessage": "responseMessage",
          "success": "success",
          "failureMessage": "failureMessage",
          "Latency": "Latency",
          "IdleTime": "IdleTime",
          "bytes": "bytes",
          "grpThreads": "grpThreads",
          "label": "label",
          "Connect": "Connect",
          "responseCode": "responseCode",
          "sentBytes": "sentBytes",
          "allThreads": "allThreads",
          "threadName": "threadName",
          "dataType": "dataType"
        }
      }
    ]
  }
}

this is my config file:
filter {
if [fields][app_name] == "jmeter-tests" {
split {
field => "message"
terminator => "\n"
}
csv {
columns => ["timeStamp","elapsed","label","responseCode","responseMessage","threadName","dataType","success","failureMessage","bytes","sentBytes","grpThreads","allThreads","Latency","IdleTime","Connect"]
separator => ","
convert => {
"elapsed" => "integer"
"success" => "boolean"
"bytes" => "integer"
"sentBytes" => "integer"
"grpThreads" => "integer"
"allThreads" => "integer"
"Latency" => "integer"
"IdleTime" => "integer"
"Connect" => "integer"
}
}
}
}

6

Try removing the terminator option.

7

it works, thank you
and thanks for your replies

but there is another problem, i want to change @/timestamp
i used Date plugin but i couldnt succeed
i want to change it to value of timestamp which i have in CSV file

8

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.