LogStash not sending some data to ElasticSearch

[Ragnaruk]

Hello.

I was running ELK Stack (6.2.3) with MetricBeat (6.2.4) installed on several Windows servers and there were no problems. Then, a need to monitor a Linux server (Red Hat Enterprise Linux 7.4) appeared: I installed MetricBeat (6.2.4), copied configuration files from its Windows version, and launched it. The data was successfully sent to LogStash, into the same pipeline as other Beats, however it didn't go any further — into ElasticSearch index (meanwhile data from Windows servers is still being sent correctly). I combed through logs of LogStash but couldn't find any errors.

It might be a problem with ElasticSearch but I figured that I'll ask here first. Thanks in advance.

My LogStash pipeline:

input {
	beats {
		port => "5044"
	}
}

filter { }

output {
	elasticsearch {
		hosts => [ "elasticsearch:9200" ]
		user => ...
		password => ...
		ssl => true
		ssl_certificate_verification => true
		cacert => "..."
		index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
	}
}

Two messages of the new Beat I picked from LogStash log:

[2018-07-18T10:54:41,503][DEBUG][logstash.pipeline        ] output received {"event"=>{"@version"=>"1", "tags"=>["beats_input_raw_event"], "@timestamp"=>2018-07-18T07:31:51.639Z, "metricset"=>{"name"=>"network", "module"=>"system", "rtt"=>6600}, "system"=>{"network"=>{"in"=>{"dropped"=>0, "errors"=>0, "packets"=>6406412, "bytes"=>5535867771}, "name"=>"docker0", "out"=>{"dropped"=>0, "errors"=>0, "packets"=>6582191, "bytes"=>11885740450}}}, "beat"=>{"hostname"=>"somehost", "name"=>"somehost", "version"=>"6.2.4"}, "host"=>"somehost"}}

[2018-07-18T10:54:41,506][DEBUG][logstash.pipeline        ] output received {"event"=>{"@version"=>"1", "tags"=>["beats_input_raw_event"], "@timestamp"=>2018-07-18T07:31:52.164Z, "system"=>{"process"=>{"cmdline"=>"/usr/lib/jvm/jre-1.8.0-openjdk/bin/java -Xms1g -Xmx1g -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastThrow -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Djava.io.tmpdir=/tmp/elasticsearch.StIwj7iK -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -Xloggc:logs/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=32 -XX:GCLogFileSize=64m -Des.cgroups.hierarchy.override=/ -Xmx256m -Xms256m -Des.path.home=/usr/share/elasticsearch -Des.path.conf=/usr/share/elasticsearch/config -cp /usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch", "pgid"=>28787, "memory"=>{"share"=>21331968, "size"=>8953016320, "rss"=>{"bytes"=>767119360, "pct"=>0.0461}}, "name"=>"java", "state"=>"sleeping", "ppid"=>28766, "cpu"=>{"start_time"=>"2018-07-10T08:20:09.000Z", "total"=>{"norm"=>{"pct"=>0.1026}, "pct"=>0.8208, "value"=>319660240.0}}, "username"=>"someuser", "pid"=>28787, "cgroup"=>{"memory"=>{"kmem"=>{"limit"=>{"bytes"=>9223372036854771712}, "failures"=>0, "usage"=>{"bytes"=>0, "max"=>{"bytes"=>0}}}, "mem"=>{"usage"=>{"max"=>{"bytes"=>2548813824}, "bytes"=>1965522944}, "failures"=>0, "limit"=>{"bytes"=>9223372036854771712}}, "stats"=>{"page_faults"=>57990208, "active_anon"=>{"bytes"=>378302464}, "swap"=>{"bytes"=>1597440}, "active_file"=>{"bytes"=>597491712}, "pages_in"=>61825870, "pages_out"=>61346006, "major_page_faults"=>339, "rss_huge"=>{"bytes"=>0}, "hierarchical_memory_limit"=>{"bytes"=>9223372036854771712}, "rss"=>{"bytes"=>756731904}, "unevictable"=>{"bytes"=>0}, "cache"=>{"bytes"=>1208791040}, "inactive_anon"=>{"bytes"=>378429440}, "hierarchical_memsw_limit"=>{"bytes"=>9223372036854771712}, "mapped_file"=>{"bytes"=>24727552}, "inactive_file"=>{"bytes"=>611299328}}, "memsw"=>{"usage"=>{"bytes"=>1967120384, "max"=>{"bytes"=>2550444032}}, "failures"=>0, "limit"=>{"bytes"=>9223372036854771712}}, "kmem_tcp"=>{"usage"=>{"bytes"=>0, "max"=>{"bytes"=>0}}, "failures"=>0, "limit"=>{"bytes"=>9223372036854771712}}, "path"=>"/docker/6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15", "id"=>"6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15"}, "path"=>"/docker/6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15", "blkio"=>{"total"=>{"ios"=>7199325, "bytes"=>109582420992}, "id"=>"6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15", "path"=>"/docker/6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15"}, "cpu"=>{"cfs"=>{"quota"=>{"us"=>0}, "period"=>{"us"=>100000}, "shares"=>1024}, "rt"=>{"period"=>{"us"=>1000000}, "runtime"=>{"us"=>0}}, "stats"=>{"periods"=>0, "throttled"=>{"periods"=>0, "ns"=>0}}, "id"=>"6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15", "path"=>"/docker/6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15"}, "id"=>"6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15", "cpuacct"=>{"percpu"=>{"4"=>40086669964122, "3"=>43400845566725, "2"=>45455445679121, "7"=>38159451848212, "8"=>37622886213292, "6"=>38043944307962, "5"=>38960181671253, "1"=>37937064393637}, "total"=>{"ns"=>319666489644324}, "stats"=>{"user"=>{"ns"=>309249940000000}, "system"=>{"ns"=>10196080000000}}, "id"=>"6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15", "path"=>"/docker/6f5f7ec115937b7aaf6b96d04863e3689f5eacb340c0ec183df133cd0eff0f15"}}}}, "metricset"=>{"name"=>"process", "module"=>"system", "rtt"=>399757}, "beat"=>{"hostname"=>"somehost", "name"=>"somehost", "version"=>"6.2.4"}, "host"=>"somehost"}}