Hello,
I am trying to use influxdb output plugin v-5.0.5 with logstash v-6.2.3.
I am getting parsing exceptions but on further looking,i see "logstash and a timestamp value" is getting added to all the output logs. When the output is seen in rubydebug format i dont see the timestamp value and logstash. Does anyone know how to get rid of these ?
` [2019-04-09T15:09:33,095][WARN ][logstash.outputs.influxdb] Non recoverable exception while writing to InfluxDB
{:exception=>#<InfluxDB::Error: {"error":"unable to parse 'logstash,host=localhost username="admin",
site="test",timestamp_local="20190409120930",resource_type="/users/admin",timestamp=1554811770000i,
@timestamp=2019-04-09T19:09:32.368Z,repo="api",resource_path="/users/admin",
message="20190409120930|14|REQUEST|...|admin|GET|/users/admin|HTTP/1.1|200|0",
duration="14",clientip="...**",@version="1",statuscode="200",bytes="0",
timestamp_object=2019-04-09T12:09:30.000Z,type="request",env="stage",requesttype="REQUEST",
resource="/users/admin",resource_name="admin",method="GET",protocol="HTTP/1.1" 1554836972368': invalid number"}}
{
"username" => "admin",
"site" => "test",
"timestamp_local" => "20190409120930",
"resource_type" => "/users/admin",
"host" => "localhost",
"timestamp" => 1554811770000,
"@timestamp" => 2019-04-09T19:09:32.368Z,
"repo" => "api",
"resource_path" => "/users/admin",
"message" => "20190409120930|14|REQUEST|...|admin|GET|/users/admin|HTTP/1.1|200|0",
"duration" => "14",
"clientip" => "...**",
"@version" => "1",
"statuscode" => "200",
"bytes" => "0",
"timestamp_object" => 2019-04-09T12:09:30.000Z,
"type" => "request",
"env" => "stage",
"requesttype" => "REQUEST",
"resource" => "/users/admin",
"resource_name" => "admin",
"method" => "GET",
"protocol" => "HTTP/1.1"
}`
My config:
input {
kafka {
id => "kafka1"
group_id => "logstash"
bootstrap_servers => ["localhost:9092"]
topics => ["request.log"]
consumer_threads => 2
type => "request"
}
}
###################################
filter {
if [type] == "request" {
if "/api/test" in [message] { drop{ } }
else {
grok {
# Enable multiple matchers
break_on_match => false
match => { "message" => "%{DATA:timestamp_local}|%{NUMBER:duration}|%{WORD:requesttype}|%{IP:clientip}|%{DATA:username}|%{WORD:method}|%{DATA:resource}|%{DATA:protocol}|%{NUMBER:statuscode}|%{NUMBER:bytes}" }
# Extract repo and path
match => { "resource" => "/%{DATA:repo}/%{GREEDYDATA:resource_path}"}
# Extract resource name
match => { "resource_path" => "(?<resource_name>[^/]+$)" }
# Extract file extension
match => { "resource_path" => "(?<resource_type>[^.]+$)" }
}
}
#Parse date field
date {
timezone => "UTC"
match => [ "timestamp_local" , "yyyyMMddHHmmss" ]
target => "timestamp_object"
} }
ruby {
code => "event.set('timestamp', event.get('timestamp_object').to_i * 1000)"
}
}
#############################
output {
if [type] == "request" {
influxdb {
codec => json
host => "..."
db => "logstash_test"
port => 8086
use_event_fields_for_data_points => true
exclude_fields => ["logstash"]
}
stdout { codec => rubydebug }
}
}