Some application logs are not getting indexed

Elastic Stack Logstash
1

Hi All,

Some of the application logs are not getting indexed. below are the error log about then in logstash logs.

below logs has error like,

....:response=>{"index"=>{"_index"=>"filebeat-7.14.0-2021.10", "_type"=>"_doc", "_id"=>"P_FC", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [response.body] of type [text] in document with id 'P_FK-JUWRUC'. Preview of field's value: '{email=abc@abc.com}'", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:1932"}}}}}

.....msg\":\"external request\",\"v\":1}"}], :response=>{"index"=>{"_index"=>"dev-api-000001", "_type"=>"_doc", "_id"=>"IeF", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [request.body] tried to parse field [body] as object, but found a concrete value"}}}}

(In above, not sure why its going to two diff index, it should only go to dev-api-000001 . ( pipeline config output is below)

complete logs -

[root@<logstash_server1> ~]# cat /var/log/logstash/logstash-plain.log |grep '<App_server2_IP>'| grep 'dev-api'| grep error -i

[2021-10-25T10:45:29,885][WARN ][logstash.outputs.elasticsearch][main][d2] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"filebeat-7.14.0-2021.10", :routing=>nil}, {"type"=>"dev-api_app_server2", "host"=>{"os"=>{"type"=>"linux", "version"=>"7.9 (Maipo)", "codename"=>"Maipo", "name"=>"Red Hat Enterprise Linux Server", "platform"=>"rhel", "kernel"=>"3.10.0-1160.45.1.el7.x86_64", "family"=>"redhat"}, "ip"=>["<App_server2_IP>", "fe80::250:56ff:fbbe:5990"], "mac"=>["00:52:50:be:52:96"], "containerized"=>false, "architecture"=>"x86_64", "name"=>"app_server2", "id"=>"b4a4", "hostname"=>"<App_server2>"}, "program"=>"dev-api", "message"=>"Oct 25 10:45:23 <App_server2> dev-api: {\"level\":\"info\",\"time\":1635147923598,\"pid\":105690,\"hostname\":\"<App_server2>\",\"response\":{\"uri\":\"[https://api-consumer-/con-api/me?fields=email\](https://api-consumer-/con-api/me?fields=email%5C)",\"method\":\"GET\",\"statusCode\":200,\"statusMessage\":\"OK\",\"headers\":{\"date\":\"Mon, 25 Oct 2021 07:45:23 GMT\",\"content-type\":\"application/json; charset=utf-8\",\"content-length\":\"51\",\"connection\":\"close\",\"strict-transport-security\":\"max-age=31536000; includeSubDomains\",\"x-request-id\":\"ee\",\"etag\":\"W/\\\"33-hjv8/5ws\\\"\",\"vary\":\"Accept-Encoding\"},\"body\":{\"email\":\"abc@abc.com\"}},\"msg\":\"external response\",\"v\":1}", "level"=>"info", "pid"=>105690, "agent"=>{"type"=>"filebeat", "version"=>"7.14.0", "ephemeral_id"=>"ef", "name"=>"app_server2", "id"=>"26", "hostname"=>"<App_server2>"}, "msg"=>"external response", "time"=>1635147923598, "logsource"=>"<App_server2>", "hostname"=>"<App_server2>", "log_type"=>"dev-api_app_server2", "tags"=>["beats_input_codec_plain_applied"], "ecs"=>{"version"=>"1.10.0"}, "v"=>1, "app_id"=>"node", "input"=>{"type"=>"log"}, "@timestamp"=>2021-10-25T07:45:23.598Z, "log"=>{"offset"=>6307, "file"=>{"path"=>"/var/log/dev-api/server.log"}}, "timestamp"=>"Oct 25 10:45:23", "response"=>{"statusMessage"=>"OK", "body"=>{"email"=>"abc@abc.com"}, "headers"=>{"connection"=>"close", "date"=>"Mon, 25 Oct 2021 07:45:23 GMT", "strict-transport-security"=>"max-age=31536000; includeSubDomains", "x-request-id"=>"ee", "content-type"=>"application/json; charset=utf-8", "content-length"=>"51", "etag"=>"W/\"3\"", "vary"=>"Accept-Encoding"}, "method"=>"GET", "statusCode"=>200, "uri"=>"https://api-consumer-/con-api/me?fields=email"}, "@version"=>"1", "json_message"=>"{\"level\":\"info\",\"time\":1635147923598,\"pid\":105690,\"hostname\":\"<App_server2>\",\"response\":{\"uri\":\"[https://api-consumer-/con-api/me?fields=email\](https://api-consumer-/con-api/me?fields=email%5C)",\"method\":\"GET\",\"statusCode\":200,\"statusMessage\":\"OK\",\"headers\":{\"date\":\"Mon, 25 Oct 2021 07:45:23 GMT\",\"content-type\":\"application/json; charset=utf-8\",\"content-length\":\"51\",\"connection\":\"close\",\"strict-transport-security\":\"max-age=31536000; includeSubDomains\",\"x-request-id\":\"ee\",\"etag\":\"W/\\\"33-hjv8/5ws\\\"\",\"vary\":\"Accept-Encoding\"},\"body\":{\"email\":\"abc@abc.com\"}},\"msg\":\"external response\",\"v\":1}"}], :response=>{"index"=>{"_index"=>"filebeat-7.14.0-2021.10", "_type"=>"_doc", "_id"=>"P_F", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [response.body] of type [text] in document with id 'P_F-K-J'. Preview of field's value: '{email=abc@abc.com}'", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:1932"}}}}}


[2021-10-25T12:15:57,257][WARN ][logstash.outputs.elasticsearch][main][482] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"dev-api", :routing=>nil}, {"type"=>"dev-api_app_server2", "host"=>{"os"=>{"type"=>"linux", "version"=>"7.9 (Maipo)", "codename"=>"Maipo", "name"=>"Red Hat Enterprise Linux Server", "platform"=>"rhel", "kernel"=>"3.10.0-1160.45.1.el7.x86_64", "family"=>"redhat"}, "hostname"=>"<App_server2>", "containerized"=>false, "mac"=>["00:52:50:be:52:96"], "architecture"=>"x86_64", "id"=>"b4", "name"=>"app_server2", "ip"=>["<App_server2_IP>", "fe80::250:56ff:fbbe:5990"]}, "program"=>"dev-api", "message"=>"Oct 25 12:15:55 <App_server2> dev-api: {\"level\":\"info\",\"time\":1635153355844,\"pid\":105690,\"hostname\":\"<App_server2>\",\"request\":{\"method\":\"POST\",\"uri\":\"[https://login./as/token.oauth2\](https://login./as/token.oauth2%5C)",\"headers\":{\"accept\":\"application/json\",\"authorization\":\"Basic ***\",\"content-type\":\"application/x-www-form-urlencoded\"},\"body\":\"grant_type=authorization_code&code=VU0vbP-My-6AW&redirect_uri=https%3A%2F%2Fdev.com%2Fauth%2Fcallback\"},\"msg\":\"external request\",\"v\":1}", "request"=>{"body"=>"grant_type=authorization_code&code=VU0vbP-My-69AAW&redirect_uri=https%3A%2F%2Fdev.com%2Fauth%2Fcallback", "headers"=>{"authorization"=>"Basic ***", "content-type"=>"application/x-www-form-urlencoded", "accept"=>"application/json"}, "method"=>"POST", "uri"=>"https://login./as/token.oauth2"}, "level"=>"info", "pid"=>105690, "agent"=>{"version"=>"7.14.0", "ephemeral_id"=>"e88a0591-8183-4f", "type"=>"filebeat", "name"=>"app_server2", "id"=>"250b578c-a719-4f6", "hostname"=>"<App_server2>"}, "msg"=>"external request", "time"=>1635153355844, "logsource"=>"<App_server2>", "hostname"=>"<App_server2>", "log_type"=>"dev-api_app_server2", "tags"=>["beats_input_codec_plain_applied"], "ecs"=>{"version"=>"1.10.0"}, "v"=>1, "app_id"=>"node", "input"=>{"type"=>"log"}, "@timestamp"=>2021-10-25T09:15:55.844Z, "log"=>{"offset"=>30936, "file"=>{"path"=>"/var/log/dev-api/server.log"}}, "timestamp"=>"Oct 25 12:15:55", "@version"=>"1", "json_message"=>"{\"level\":\"info\",\"time\":1635153355844,\"pid\":105690,\"hostname\":\"<App_server2>\",\"request\":{\"method\":\"POST\",\"uri\":\"[https://login./as/token.oauth2\](https://login./as/token.oauth2%5C)",\"headers\":{\"accept\":\"application/json\",\"authorization\":\"Basic ***\",\"content-type\":\"application/x-www-form-urlencoded\"},\"body\":\"grant_type=authorization_code&code=VU0vbP-My-69AW&redirect_uri=https%3A%2F%2Fdev.com%2Fauth%2Fcallback\"},\"msg\":\"external request\",\"v\":1}"}], :response=>{"index"=>{"_index"=>"dev-api-000001", "_type"=>"_doc", "_id"=>"IeF", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [request.body] tried to parse field [body] as object, but found a concrete value"}}}}


[2021-10-25T12:15:57,262][WARN ][logstash.outputs.elasticsearch][main][d2] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"filebeat-7.14.0-2021.10", :routing=>nil}, {"type"=>"dev-api_app_server2", "host"=>{"hostname"=>"<App_server2>", "os"=>{"type"=>"linux", "version"=>"7.9 (Maipo)", "codename"=>"Maipo", "name"=>"Red Hat Enterprise Linux Server", "platform"=>"rhel", "kernel"=>"3.10.0-1160.45.1.el7.x86_64", "family"=>"redhat"}, "containerized"=>false, "mac"=>["00:52:50:be:52:96"], "architecture"=>"x86_64", "name"=>"app_server2", "id"=>"b4", "ip"=>["<App_server2_IP>", "fe80::250:56ff:fbbe:5990"]}, "program"=>"dev-api", "message"=>"Oct 25 12:15:55 <App_server2> dev-api: {\"level\":\"info\",\"time\":1635153355917,\"pid\":105690,\"hostname\":\"<App_server2>\",\"response\":{\"uri\":\"[https://login./as/token.oauth2\](https://login./as/token.oauth2%5C)",\"method\":\"POST\",\"statusCode\":200,\"statusMessage\":\"OK\",\"headers\":{\"connection\":\"close\",\"date\":\"Mon, 25 Oct 2021 09:15:55 GMT\",\"x-frame-options\":\"SAMEORIGIN\",\"referrer-policy\":\"origin\",\"cache-control\":\"no-cache, no-store\",\"pragma\":\"no-cache\",\"expires\":\"Thu, 01 Jan 1970 00:00:00 GMT\",\"content-type\":\"application/json;charset=utf-8\",\"set-cookie\":[\"PF=gu;Path=/;Secure;HttpOnly\"]},\"body\":{\"access_token\":\"ey\",\"scope\":\"openid dev\",\"id_token\":\"eA\",\"token_type\":\"Bearer\",\"expires_in\":7775999}},\"msg\":\"external response\",\"v\":1}", "level"=>"info", "pid"=>105690, "agent"=>{"ephemeral_id"=>"e3-f", "type"=>"filebeat", "version"=>"7.14.0", "name"=>"app_server2", "id"=>"26", "hostname"=>"<App_server2>"}, "msg"=>"external response", "time"=>1635153355917, "logsource"=>"<App_server2>", "hostname"=>"<App_server2>", "log_type"=>"dev-api_app_server2", "tags"=>["beats_input_codec_plain_applied"], "ecs"=>{"version"=>"1.10.0"}, "v"=>1, "app_id"=>"node", "input"=>{"type"=>"log"}, "@timestamp"=>2021-10-25T09:15:55.917Z, "log"=>{"offset"=>31456, "file"=>{"path"=>"/var/log/dev-api/server.log"}}, "timestamp"=>"Oct 25 12:15:55", "response"=>{"statusMessage"=>"OK", "body"=>{"access_token"=>"ely", "scope"=>"openid dev", "token_type"=>"Bearer", "id_token"=>"eA", "expires_in"=>7775999}, "headers"=>{"connection"=>"close", "date"=>"Mon, 25 Oct 2021 09:15:55 GMT", "x-frame-options"=>"SAMEORIGIN", "cache-control"=>"no-cache, no-store", "referrer-policy"=>"origin", "expires"=>"Thu, 01 Jan 1970 00:00:00 GMT", "content-type"=>"application/json;charset=utf-8", "pragma"=>"no-cache", "set-cookie"=>["PF=gu;Path=/;Secure;HttpOnly"]}, "method"=>"POST", "statusCode"=>200, "uri"=>"https://login./as/token.oauth2"}, "@version"=>"1", "json_message"=>"{\"level\":\"info\",\"time\":1635153355917,\"pid\":105690,\"hostname\":\"<App_server2>\",\"response\":{\"uri\":\"[https://login./as/token.oauth2\](https://login./as/token.oauth2%5C)",\"method\":\"POST\",\"statusCode\":200,\"statusMessage\":\"OK\",\"headers\":{\"connection\":\"close\",\"date\":\"Mon, 25 Oct 2021 09:15:55 GMT\",\"x-frame-options\":\"SAMEORIGIN\",\"referrer-policy\":\"origin\",\"cache-control\":\"no-cache, no-store\",\"pragma\":\"no-cache\",\"expires\":\"Thu, 01 Jan 1970 00:00:00 GMT\",\"content-type\":\"application/json;charset=utf-8\",\"set-cookie\":[\"PF=gbu;Path=/;Secure;HttpOnly\"]},\"body\":{\"access_token\":\"ey\",\"scope\":\"openid dev\",\"id_token\":\"eA\",\"token_type\":\"Bearer\",\"expires_in\":7775999}},\"msg\":\"external response\",\"v\":1}"}], :response=>{"index"=>{"_index"=>"filebeat-7.14.0-2021.10", "_type"=>"_doc", "_id"=>"BejJ", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"failed to parse field [response.body] of type [text] in document with id 'BJ'. Preview of field's value: '{access_token=eA, token_type=Bearer, expires_in=7775999}'", "caused_by"=>{"type"=>"illegal_state_exception", "reason"=>"Can't get text on a START_OBJECT at 1:3521"}}}}}

These are the original log message from server log file. (searching with timestamp , found in above error log )

[root@<App_server2> ~]# cat /var/log/dev-api/server.log |grep 1635147923598

Oct 25 10:45:23 <App_server2> dev-api: {"level":"info","time":1635147923598,"pid":105690,"hostname":"<App_server2>","response":{"uri":"https://api-consumer-/con-api/me?fields=email","method":"GET","statusCode":200,"statusMessage":"OK","headers":{"date":"Mon, 25 Oct 2021 07:45:23 GMT","content-type":"application/json; charset=utf-8","content-length":"51","connection":"close","strict-transport-security":"max-age=31536000; includeSubDomains","x-request-id":"efe","etag":"W/\"33-hjv8/5s\"","vary":"Accept-Encoding"},"body":{"email":"abc@abc.com"}},"msg":"external response","v":1}
[root@<App_server2> ~]#

[root@<App_server2> ~]# cat /var/log/dev-api/server.log |grep 1635147927632

Oct 25 10:45:27 <App_server2> dev-api: {"level":"info","time":1635147927632,"pid":105690,"hostname":"<App_server2>","response":{"uri":"https://api-consumer-/con-api/me?fields=email","method":"GET","statusCode":200,"statusMessage":"OK","headers":{"date":"Mon, 25 Oct 2021 07:45:27 GMT","content-type":"application/json; charset=utf-8","content-length":"51","connection":"close","strict-transport-security":"max-age=31536000; includeSubDomains","x-request-id":"7a","etag":"W/\"33-hjv8/5ws\"","vary":"Accept-Encoding"},"body":{"email":"abc@abc.com"}},"msg":"external response","v":1}
[root@<App_server2> ~]#
2

If above error message are not indexed not sure why but able to see output of below search query by searching with above timestamp in dev-api-000001 index (except for 1635153355844 ).

GET /dev-api-000001/_search

{

  "query": {

    "match": {

      "time": "1635147923598"

    }

  }

}

{

  "took" : 2,

  "timed_out" : false,

  "_shards" : {

    "total" : 2,

    "successful" : 2,

    "skipped" : 0,

    "failed" : 0

  },

  "hits" : {

    "total" : {

      "value" : 1,

      "relation" : "eq"

    },

    "max_score" : 1.0,

    "hits" : [

      {

        "_index" : "dev-api-000001",

        "_type" : "_doc",

        "_id" : "Me",

        "_score" : 1.0,

        "_ignored" : [

          "message.keyword",

          "json_message.keyword"

        ],

        "_source" : {

          "type" : "dev-api_app_server2",

          "host" : {

            "os" : {

              "type" : "linux",

              "version" : "7.9 (Maipo)",

              "codename" : "Maipo",

              "name" : "Red Hat Enterprise Linux Server",

              "platform" : "rhel",

              "kernel" : "3.10.0-1160.45.1.el7.x86_64",

              "family" : "redhat"

            },

            "ip" : [

              "<App_server2_IP>",

              "fe80::250:56ff:fbbe:5994"

            ],

            "mac" : [

              "00:50:56:bb:60:94"

            ],

            "containerized" : false,

            "architecture" : "x86_64",

            "name" : "app_server2",

            "id" : "b4a4",

            "hostname" : "<App_Server_2>"

          },

          "program" : "dev-api",

          "message" : """Oct 25 10:45:23 <App_Server_2> dev-api: {"level":"info","time":1635147923598,"pid":105690,"hostname":"<App_Server_2>","response":{"uri":"https://apiman-con-/con-api/me?fields=email","method":"GET","statusCode":200,"statusMessage":"OK","headers":{"date":"Mon, 25 Oct 2021 07:45:23 GMT","content-type":"application/json; charset=utf-8","content-length":"51","connection":"close","strict-transport-security":"max-age=31536000; includeSubDomains","x-request-id":"e5d947a4fe","etag":"W/\"33-hjv8/5ws\"","vary":"Accept-Encoding"},"body":{"email":"<abc@abc.com>"}},"msg":"external response","v":1}""",

          "level" : "info",

          "pid" : 105690,

          "agent" : {

            "type" : "filebeat",

            "version" : "7.14.0",

            "ephemeral_id" : "e8f",

            "name" : "app_server2",

            "id" : "26",

            "hostname" : "<App_Server_2>"

          },

          "msg" : "external response",
          "time" : 1635147923598,
          "logsource" : "<App_Server_2>",
          "hostname" : "<App_Server_2>",
          "log_type" : "dev-api_app_server2",
          "tags" : [

            "beats_input_codec_plain_applied"

          ],

          "ecs" : {

            "version" : "1.10.0"

          },

          "v" : 1,

          "app_id" : "node",

          "input" : {

            "type" : "log"

          },

          "@timestamp" : "2021-10-25T07:45:23.598Z",

          "log" : {

            "offset" : 6307,

            "file" : {

              "path" : "/var/log/dev-api/server.log"

            }

          },

          "timestamp" : "Oct 25 10:45:23",

          "response" : {

            "statusMessage" : "OK",

            "body" : {

              "email" : "<abc@abc.com>"

            },

            "headers" : {

              "connection" : "close",

              "date" : "Mon, 25 Oct 2021 07:45:23 GMT",

              "strict-transport-security" : "max-age=31536000; includeSubDomains",

              "x-request-id" : "e5e",

              "content-type" : "application/json; charset=utf-8",

              "content-length" : "51",

              "etag" : "W/\"33-hjv8/5s\"",

              "vary" : "Accept-Encoding"

            },

            "method" : "GET",

            "statusCode" : 200,

            "uri" : "https://apiman-con-/con-api/me?fields=email"

          },

          "@version" : "1",

          "json_message" : """{"level":"info","time":1635147923598,"pid":105690,"hostname":"<App_Server_2>","response":{"uri":"https://apiman-con-/con-api/me?fields=email","method":"GET","statusCode":200,"statusMessage":"OK","headers":{"date":"Mon, 25 Oct 2021 07:45:23 GMT","content-type":"application/json; charset=utf-8","content-length":"51","connection":"close","strict-transport-security":"max-age=31536000; includeSubDomains","x-request-id":"e8d947a4fe","etag":"W/\"33-hjv8/5ws\"","vary":"Accept-Encoding"},"body":{"email":"<abc@abc.com>"}},"msg":"external response","v":1}"""

        }

      }

    ]

  }

}

Below is the mapping for request , response field.

  "request" : {
          "properties" : {
            "body" : {
              "properties" : {
                "client" : {
                  "properties" : {
                    "bypassApprovalPage" : {
                      "type" : "boolean"
                    },
                    "clientId" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "enabled" : {
                      "type" : "boolean"
                    },
                    "grantTypes" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "name" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                    "redirectUris" : {
                      "type" : "text",
                      "fields" : {
                        "keyword" : {
                          "type" : "keyword",
                          "ignore_above" : 256
                        }
                      }
                    },
                   .
                   .
                   .

      "response" : {
          "properties" : {
            "body" : {
              "properties" : {
                "access_token" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "api_version" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "app_credential_urls" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "app_url" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "client_id" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "client_secret" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "created_at" : {
                  "type" : "date",
                  "format" : "strict_date_optional_time"
                },
                "email" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },
                "expires_in" : {
                  "type" : "long"
                },
                "id" : {
                  "type" : "text",
                  "fields" : {
                    "keyword" : {
                      "type" : "keyword",
                      "ignore_above" : 256
                    }
                  }
                },

This is the pipeline configuration for dev-api application.

input {

  beats {

    port => 5044

  }

}

filter {

if [log_type] == "dev-api_server1" and [app_id] == "node"

  {

    grok { match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:json_message}"  } } json { source =>  "json_message" }

    date { match => ["time", "UNIX_MS"]

         }

    mutate {

             replace => {

               "[type]" => "dev-api_server1"

             }

           }

  }

if [log_type] == "dev-api_server2" and [app_id] == "node"

  {

    grok { match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:json_message}"  } } json { source =>  "json_message" }

    date { match => ["time", "UNIX_MS"]

         }

    mutate {

             replace => {

               "[type]" => "dev-api_server2"

             }

           }

  }

output {

  if [log_type] == "dev-api_server1" {

  elasticsearch {

    hosts => ['http://es_1:<es_port>', 'http://es_2:<es_port>', 'http://es_3:<es_port>']

    index => "dev-api"

    template_name => "dev-api"

    template_overwrite => "false"

        user => elastic

    password => "${es_pwd}"

      }

}

  if [log_type] == "dev-api_server2" {

  elasticsearch {

    hosts => ['http://es_1:<es_port>', 'http://es_2:<es_port', 'http://es_3:<es_port>']

    index => "dev-api"

    template_name => "dev-api"

    template_overwrite => "false"

        user => elastic

    password => "${es_pwd}"

      }

}

Thanks,

3

Can someone please reply.

It looks first error is because, email field is causing problem

4

HI @leandrojmp,

Could you please check and advise.

Thanks,

5

See this post and follow the links from it. [response][body] and [request][body] cannot be text in some documents and an object in others. It has to be one or the other. Even if you fix your problem with the index name this will not fix the mapping exceptions.

6

Please don't ping people that aren't already taking part in your topic like that :slight_smile: