If above error message are not indexed not sure why but able to see output of below search query by searching with above timestamp in dev-api-000001 index (except for 1635153355844 ).
GET /dev-api-000001/_search
{
"query": {
"match": {
"time": "1635147923598"
}
}
}
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 2,
"successful" : 2,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "dev-api-000001",
"_type" : "_doc",
"_id" : "Me",
"_score" : 1.0,
"_ignored" : [
"message.keyword",
"json_message.keyword"
],
"_source" : {
"type" : "dev-api_app_server2",
"host" : {
"os" : {
"type" : "linux",
"version" : "7.9 (Maipo)",
"codename" : "Maipo",
"name" : "Red Hat Enterprise Linux Server",
"platform" : "rhel",
"kernel" : "3.10.0-1160.45.1.el7.x86_64",
"family" : "redhat"
},
"ip" : [
"<App_server2_IP>",
"fe80::250:56ff:fbbe:5994"
],
"mac" : [
"00:50:56:bb:60:94"
],
"containerized" : false,
"architecture" : "x86_64",
"name" : "app_server2",
"id" : "b4a4",
"hostname" : "<App_Server_2>"
},
"program" : "dev-api",
"message" : """Oct 25 10:45:23 <App_Server_2> dev-api: {"level":"info","time":1635147923598,"pid":105690,"hostname":"<App_Server_2>","response":{"uri":"https://apiman-con-/con-api/me?fields=email","method":"GET","statusCode":200,"statusMessage":"OK","headers":{"date":"Mon, 25 Oct 2021 07:45:23 GMT","content-type":"application/json; charset=utf-8","content-length":"51","connection":"close","strict-transport-security":"max-age=31536000; includeSubDomains","x-request-id":"e5d947a4fe","etag":"W/\"33-hjv8/5ws\"","vary":"Accept-Encoding"},"body":{"email":"<abc@abc.com>"}},"msg":"external response","v":1}""",
"level" : "info",
"pid" : 105690,
"agent" : {
"type" : "filebeat",
"version" : "7.14.0",
"ephemeral_id" : "e8f",
"name" : "app_server2",
"id" : "26",
"hostname" : "<App_Server_2>"
},
"msg" : "external response",
"time" : 1635147923598,
"logsource" : "<App_Server_2>",
"hostname" : "<App_Server_2>",
"log_type" : "dev-api_app_server2",
"tags" : [
"beats_input_codec_plain_applied"
],
"ecs" : {
"version" : "1.10.0"
},
"v" : 1,
"app_id" : "node",
"input" : {
"type" : "log"
},
"@timestamp" : "2021-10-25T07:45:23.598Z",
"log" : {
"offset" : 6307,
"file" : {
"path" : "/var/log/dev-api/server.log"
}
},
"timestamp" : "Oct 25 10:45:23",
"response" : {
"statusMessage" : "OK",
"body" : {
"email" : "<abc@abc.com>"
},
"headers" : {
"connection" : "close",
"date" : "Mon, 25 Oct 2021 07:45:23 GMT",
"strict-transport-security" : "max-age=31536000; includeSubDomains",
"x-request-id" : "e5e",
"content-type" : "application/json; charset=utf-8",
"content-length" : "51",
"etag" : "W/\"33-hjv8/5s\"",
"vary" : "Accept-Encoding"
},
"method" : "GET",
"statusCode" : 200,
"uri" : "https://apiman-con-/con-api/me?fields=email"
},
"@version" : "1",
"json_message" : """{"level":"info","time":1635147923598,"pid":105690,"hostname":"<App_Server_2>","response":{"uri":"https://apiman-con-/con-api/me?fields=email","method":"GET","statusCode":200,"statusMessage":"OK","headers":{"date":"Mon, 25 Oct 2021 07:45:23 GMT","content-type":"application/json; charset=utf-8","content-length":"51","connection":"close","strict-transport-security":"max-age=31536000; includeSubDomains","x-request-id":"e8d947a4fe","etag":"W/\"33-hjv8/5ws\"","vary":"Accept-Encoding"},"body":{"email":"<abc@abc.com>"}},"msg":"external response","v":1}"""
}
}
]
}
}
Below is the mapping for request , response field.
"request" : {
"properties" : {
"body" : {
"properties" : {
"client" : {
"properties" : {
"bypassApprovalPage" : {
"type" : "boolean"
},
"clientId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"enabled" : {
"type" : "boolean"
},
"grantTypes" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"redirectUris" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
.
.
.
"response" : {
"properties" : {
"body" : {
"properties" : {
"access_token" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"api_version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"app_credential_urls" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"app_url" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"client_id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"client_secret" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"created_at" : {
"type" : "date",
"format" : "strict_date_optional_time"
},
"email" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"expires_in" : {
"type" : "long"
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
This is the pipeline configuration for dev-api application.
input {
beats {
port => 5044
}
}
filter {
if [log_type] == "dev-api_server1" and [app_id] == "node"
{
grok { match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:json_message}" } } json { source => "json_message" }
date { match => ["time", "UNIX_MS"]
}
mutate {
replace => {
"[type]" => "dev-api_server1"
}
}
}
if [log_type] == "dev-api_server2" and [app_id] == "node"
{
grok { match => { "message" => "%{SYSLOGBASE} %{GREEDYDATA:json_message}" } } json { source => "json_message" }
date { match => ["time", "UNIX_MS"]
}
mutate {
replace => {
"[type]" => "dev-api_server2"
}
}
}
output {
if [log_type] == "dev-api_server1" {
elasticsearch {
hosts => ['http://es_1:<es_port>', 'http://es_2:<es_port>', 'http://es_3:<es_port>']
index => "dev-api"
template_name => "dev-api"
template_overwrite => "false"
user => elastic
password => "${es_pwd}"
}
}
if [log_type] == "dev-api_server2" {
elasticsearch {
hosts => ['http://es_1:<es_port>', 'http://es_2:<es_port', 'http://es_3:<es_port>']
index => "dev-api"
template_name => "dev-api"
template_overwrite => "false"
user => elastic
password => "${es_pwd}"
}
}
Thanks,