Elasticsearch filter giving "TypeError: no implicit conversion of nil into String"

Elastic Stack Logstash
1

I have log files from an app that logs requests and responses. I want to knit those together. I write the request to elasticsearch, then, when I process the response, I want to lookup the request that generated it. So for this input

---- 743-b0d37f28 -- 2018-03-13T05:34:48.536 -- POST
---- 743-b0d37f28 -- 2018-03-13T05:34:48.536 -- {A passel of JSON}

I run this config

input { stdin { } }
output {
    stdout { codec => "rubydebug" }
    elasticsearch {
      hosts => "localhost"
      index => "logstash-test-%{+YYYY.MM.dd}"
      document_id => "%{docId}"
    }
}

filter {
  dissect { mapping => { "message" => "---- %{docId} -- %{ts} -- %{msg}" } }

  if [msg] !~ "\APOST" {
    elasticsearch {
      hosts => ["localhost"]
      query => "_id: %{docId}"
      index => "logstash-test-%{+YYYY.MM.dd}"
      fields => [ "request" ]
    }
  }

  if [msg] =~ "\APOST" {
    mutate { add_field => { "[request][url]" => "Monsieur Spalanzani n'aime pas la musique" } }
  } else {
    mutate { add_field => { "[response][body]" => "Gluck, dar mir verblieb" } }
  }
}

That produces this output

{
          "host" => "<...>",
         "docId" => "743-b0d37f28",
           "msg" => "POST\r",
      "@version" => "1",
       "request" => {
        "url" => "Monsieur Spalanzani n'aime pas la musique"
    },
       "message" => "---- 743-b0d37f28 -- 2018-03-13T05:34:48.536 -- POST\r",
    "@timestamp" => 2018-03-13T18:20:51.633Z,
            "ts" => "2018-03-13T05:34:48.536"
}
{
          "host" => "<...>",
         "docId" => "743-b0d37f28",
           "msg" => "{A passel of JSON}\r",
          "tags" => [
        [0] "_elasticsearch_lookup_failure"
    ],
      "@version" => "1",
       "message" => "---- 743-b0d37f28 -- 2018-03-13T05:34:48.536 -- {A passel of JSON}\r",
    "@timestamp" => 2018-03-13T18:20:51.635Z,
      "response" => {
        "body" => "Gluck, dar mir verblieb"
    },
            "ts" => "2018-03-13T05:34:48.536"
}

The error message is

[2018-03-13T14:20:51,493][INFO ][logstash.pipeline        ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[...]
[2018-03-13T14:20:52,021][WARN ][logstash.filters.elasticsearch] Failed to query elasticsearch for previous event {:index=>"logstash-test-%{+YYYY.MM.dd}", :query=>"_id: 743-b0d37f28", :event=>#, :error=>#}

Looking at the document in Discover the _id was set.

{
  "_index": "logstash-test-2018.03.13",
  "_type": "doc",
  "_id": "743-b0d37f28",
  "_version": 6,
  "_score": null,
  "_source": {
    "host": "<...>",
    "docId": "743-b0d37f28",
    "msg": "{A passel of JSON}\r",
    "tags": [
      "_elasticsearch_lookup_failure"
    ],
    "@version": "1",
    "message": "---- 743-b0d37f28 -- 2018-03-13T05:34:48.536 -- {A passel of JSON}\r",
    "@timestamp": "2018-03-13T18:20:51.635Z",
    "response": {
      "body": "Gluck, dar mir verblieb"
    },
    "ts": "2018-03-13T05:34:48.536"
  },
  "fields": {
    "@timestamp": [
      "2018-03-13T18:20:51.635Z"
    ],
    "ts": [
      "2018-03-13T05:34:48.536Z"
    ]
  },
  "sort": [
    1520965251635
  ]
}
Where is the nil coming from?
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.