Field value not populating in kibana dashboard

[Vinay_Kumar2 (Vinay Kumar)]

Hi I am trying to create a visualization using data table in kibana 7.6. Able to see the field value in Discover tab. But not able to create visualization for the one field iws message2.
Below is my logstash configuration like this format
input {
beats {
port => "5044"
}
}
filter {
if "beats_input_codec_plain_applied" in [tags] {
mutate {
remove_tag => ["beats_input_codec_plain_applied"]
}
} }
if [document_type] == "iws" {
grok {
match => { "message" => '%{USERNAME:username}%{SPACE}%{NOTSPACE:hyphon}%{SPACE}%{NOTSPACE:ID}%{SPACE}%{TIMESTAMP_ISO8601:timestamp1}%{SPACE}(.?[%{GREEDYDATA:iwsmessage1}])%{SPACE}%{LOGLEVEL:severity}%{SPACE}((?[^)]))%{SPACE}****%{WORD:requesttype}%{SPACE}%{GREEDYDATA:iwsmessage2}'}
}
# mutate { add_field => { "timestamp1" => "%{year}-%{month}-%{day},000" }
#}
date {
match => [ "timestamp1", "yyyy-MM-dd HH:mm:ss,SSS" ]
timezone => "US/Eastern"
target => "@timestamp"
}
}
else {
grok {
match => { "message" => '%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}\t%{TIME:time}\t%{DATA:method}\t%{USERNAME:ecid}\t%{INT:REL_ID}\t%{URIPATHPARAM:URI}\t%{NUMBER:response_status}\t%{NUMBER:bytes}'}
}
mutate { add_field => { "timestamp6" => "%{year}-%{month}-%{day} %{time}" }
}
date {
match => [ "timestamp6", "yyyy-MM-dd HH:mm:ss" ]
timezone => "US/Eastern"
target => "@timestamp"
}

}           

}
output {

stdout { codec => rubydebug }

elasticsearch {
hosts => ["http://localhost:9200"]
index => "iws"
}

}
Sample log format like this
IISUSERL - 878190-122-1 2020-03-04 14:15:12,811 [[ACTIVE] ExecuteThread: '1' for queue: 'weblogic.kernel.Default (self-tuning)'] INFO (ouaf.iws.trace) ****Request env:Header/<ns1:C1FieldActivityMaintenance xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/" xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><ns1:C1FieldActivityMaintenanceService xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd"><ns1:C1FieldActivityMaintenanceHeader xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" FieldActivityID="7028706612"/><ns1:C1FieldActivityMaintenanceDetails xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" CreateDateTime="2020-03-04T14:11:48-05:00" FieldActivityID="Valueeeeee"><ns1:FaRem xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd"><ns1:FaRemRow xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" FieldActivityID="Valueeeeee" FieldActivityRemark="GASLEAKSRV" FieldActivityRemarkStatus="P" rowAction="Add"/><ns1:FaRemHeader xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" FieldActivityID="Valueeeeee"/></ns1:FaRem><ns1:FaLog xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd"><ns1:FaLogHeader xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" FieldActivityID="Valueeeeee"/><ns1:FaLogRow xmlns:ns1="http://splwg.com/C1FieldActivityMaintenance.xsd" DateTime="2020-03-04T14:11:48-05:00" Details="Description: Send Gas Leak Survey " FieldActivityID="Valueeeeee" LogType="USER" rowAction="Add"/></ns1:FaLog></ns1:C1FieldActivityMaintenanceDetails></ns1:C1FieldActivityMaintenanceService></ns1:C1FieldActivityMaintenance>

I am not able to create visualization on iwsmessage2 field. even not able to see any grok parse failure.

Any one help on the issue why it is not working ....

[nickpeihl]

Can you share the mapping of your iws index? I wonder if the iwsmessage2 field is mapped as text? If so, you would need to add fielddata to the mapping to be able to visualize using a Terms aggregation on that field.

[Vinay_Kumar2 (Vinay Kumar)]

Hi nickpeihl,

please refer the below mapping of my iws index. First i found ignore_above value as 256. After that i have increased it to 100000 and recreated a new index. Still no luck.

my iws index mapping like this

{
"iws" : {
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ID" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"REL_ID" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"URI" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"agent" : {
"properties" : {
"ephemeral_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"bytes" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"day" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"document_type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ecid" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ecs" : {
"properties" : {
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"host" : {
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"hyphon" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"input" : {
"properties" : {
"type" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"iwsmessage1" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"iwsmessage2" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 100000
}
}
},
"log" : {
"properties" : {
"file" : {
"properties" : {
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"flags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"offset" : {
"type" : "long"
}
}
},
"message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"method" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"month" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"requesttype" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"response_status" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"serviceName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"severity" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"time" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timestamp1" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timestamp6" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"username" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"year" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
}

[Vinay_Kumar2 (Vinay Kumar)]

Hi Nickpeihl,
Please refer the below snapshot. After recreating index and enabling field data. Still i am getting no results found in visualization.

Regards,
Vinay Kumar