We just recently started trying to use logstash. The problem is that memory usage grows, at least within first 2 days, to the point that we can't use it.
I tried reducing Xms and Xmx from default of 1G to 512M. restarted. Made sure that these options are there in java call (with ps). After couple of hours I'm back at 1.5GB of mem used.
We use logstash 7.15.0-1 on Ubuntu Bionic. After 18hours of runtime, with -Xms512m -Xmx512m, it is using now (RSS) 1723004 kB:
=$ ps uw -p 56563
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 56563 7.7 0.4 26823120 1723004 ? SNsl Oct14 85:46 /usr/share/logstash/jdk/bin/java -Xms512m -Xmx512m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djruby.compile.invoked
Immediately after restart is was ~ 350MB, but it grew very fast to 1.6GB, and then gradually to ~ 1.7 now.
I know that generally speaking it's not that much of memory, but simple processing of loglines using 1.7GB is simply strange, plus - we have some servers with like 8GB of ram, and usage of 1.5GB there is simply not an option.
Our config:
input {
file {
"path" => "/cache/postgres_logs/*.csv"
"tags" => [ "PostgreSQL" ]
"sincedb_path" => "/cache/postgres_logs/sincedb_pgsql"
codec => multiline {
auto_flush_interval => 2
pattern => "^%{TIMESTAMP_ISO8601}"
what => previous
negate => true
}
}
file {
"path" => "/cache/pgbouncer_logs/pgbouncer-*.log"
"tags" => [ "pgBouncerOnPg" ]
"sincedb_path" => "/cache/pgbouncer_logs/sincedb_pgbouncer"
}
}
filter {
if "PostgreSQL" in [tags] {
csv {
# Columns list:
# https://www.postgresql.org/docs/12/runtime-config-logging.html#RUNTIME-CONFIG-LOGGING-CSVLOG
columns => [ "log_time", "user_name", "database_name", "process_id", "connection_from", "session_id", "session_line_num", "command_tag", "session_start_time", "virtual_transaction_id", "transaction_id", "error_severity", "sql_state_code", "sql_message", "detail", "hint", "internal_query", "internal_query_pos", "context", "query", "query_pos", "location", "application_name" ]
skip_empty_columns => true
convert => {
"process_id" => "integer"
"session_line_num" => "integer"
"transaction_id" => "integer"
"session_start_time" => "date_time"
"internal_query_pos" => "integer"
"query_pos" => "integer"
}
}
mutate {
gsub => [ "sql_message", "[\r\n\t ]+", " "]
add_field => {
"sourcetype" => "PostgreSQL"
"region" => "us-east-1"
"az" => "us-east-1b"
"host_fqdn" => "..."
"pgrole" => "..."
"project" => "..."
"cluster" => "94"
"environment" => "production"
}
}
date {
match => ["log_time", "YYYY-MM-dd HH:mm:ss.SSS z"]
}
grok {
match => ["sql_message", "duration: %{DATA:duration:float} ms +(statement|(parse|bind|execute) [^:]+): %{GREEDYDATA:statement}"]
break_on_match => false
tag_on_failure => []
}
# Add normalized versions of fields that can contain queries
# \x22 is " character
ruby {
code => "
%w{ sql_message query statement }.each do |k|
ov = event.get(k)
next unless ov
v = ov.downcase
v.gsub!( %r{\s+}, ' ')
v.gsub!( %r{\s*/\*.*?\*/\s*}, ' ' )
v.gsub!( %r{\x22?(...|cluster\d+_shard_\d+|cluster\d+)\x22?\.}, 'SOME_SHARD.' )
v.gsub!( %r{'[^']*'|\b\d+\b}, '?' )
v.gsub!( %r{ in \([\s,?]+\)}, ' in (...)' )
event.set( 'n_' + k, v )
end
"
}
# Add each element from marginalia as "marginalia_ORIGINAL_NAME" => ORIGINAL_VALUE
ruby {
code => "
if m = event.get('sql_message').match( %r{/\*(.*?)\*/[\s;]*$} )
m[1].
split(',').
map {|s| s.
split(':', 2)}.
each { |s|
event.set( 'marginalia_' + s[0], s[1] )
}
end
"
}
} else {
mutate {
add_field => {
"sourcetype" => "pgBouncerOnPg"
"region" => "us-east-1"
"az" => "us-east-1b"
"host_fqdn" => "..."
"pgrole" => "..."
"project" => "..."
"cluster" => "94"
"environment" => "production"
}
}
grok {
match => ["message", "^(?<log_ts>\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d \S+) \[%{INT:pid:int}\] %{WORD:log_level}"]
}
grok {
match => ["path", "-%{INT:pgbouncer_port:int}.log$"]
}
date {
match => ["log_ts", "YYYY-MM-dd HH:mm:ss.SSS z"]
}
}
}
output {
http {
http_method => "post"
url => "...."
headers => ['Authorization', '....']
}
}
Is there anything we could do about it?