Explosion of fields from nginx log parsing

[jgrevich (Justin Grevich)]

We exceeded the default number of total fields for one index and it appears to be related to how we're parsing log events from nginx (and perhaps others).

Does anyone know what might cause these types of fields to be created?

nginx.#cmd})).(#p
nginx.#cmd})).(#p.keyword
nginx.'1'
nginx.'1'.keyword
nginx.'ns'
nginx.'ns'.keyword
nginx./%20ns
nginx./%20ns.keyword

It appears that new fields are created based on the value of the nginx request data. Here is the relevant logstash code

  # Nginx
  if [source] == "/var/log/nginx/access.log" {
    kv {
      add_field => { "type" => "nginx_access" }
      target => "nginx"
    }
  }
  else if [source] == "/var/log/nginx/fancy_access.log" {
    drop { }
  }
  else if [source] == "/var/log/nginx/error.log" {
    if [message] =~ "^\[" {
      grok {
        add_field => {
          "[nginx][error][logtype]" => "module"
          "type" => "nginx_error"
        }
        match => { "message" => ["\[ N %{TIMESTAMP_ISO8601:[nginx][error][time]} %{NUMBER:[nginx][error][process_id]}/(?<source_code_filename>T.*):%{NUMBER:[nginx][error][source_code][line_number]} ]: %{GREEDYDATA:[nginx][error][message]}"] }
      }
      mutate {
        rename => { "source_code_filename" => "[nginx][error][source_code][filename]" }
      }
    }
    else if [message] =~ "^%{YEAR}" {
      grok {
        add_field => {
          "[nginx][error][logtype]" => "core"
          "type" => "nginx_error"
        }
        match => { "message" => ["%{DATA:[nginx][error][time]} \[%{DATA:[nginx][error][level]}\] %{NUMBER:[nginx][error][pid]}#%{NUMBER:[nginx][error][tid]}: (\*%{NUMBER:[nginx][error][connection_id]} )?%{GREEDYDATA:[nginx][error][message]}"] }
      }
      date {
        match => [ "[nginx][error][time]", "YYYY/MM/dd H:m:s" ]
        remove_field => "[nginx][error][time]"
      }
    }
    else if [message] =~ "^App" {
      grok {
        add_field => {
          "[nginx][error][logtype]" => "app"
          "type" => "nginx_error"
        }
        match => { "message" => ["App %{NUMBER:[nginx][error][app][process_id]} output: %{GREEDYDATA:[nginx][error][app][message]}"] }
      }
    }
  }