Hello
i am using EFK stack in my GKE cluster and recently i observed below errors in my fluentd logs.
2021-04-15 14:39:25 +0000 [warn]: dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch" location=nil tag="kubernetes.var.log.containers.api-php-79b7f6df4d-twqww_deci-white-deer_api-php-14a3179e6994c404d2ac945bb202ddfe6f30586e9e74945d26a0682af9d27e85.log" time=2021-04-15 14:39:21.810943808 +0000 record={"message"=>"Response", "context"=>["{\"error\":{\"root_cause\":[{\"type\":\"index_not_found_exception\",\"reason\":\"no such index\",\"index_uuid\":\"_na_\",\"resource.type\":\"index_or_alias\",\"resource.id\":\"internal_image\",\"index\":\"internal_image\"}],\"type\":\"index_not_found_exception\",\"reason\":\"no such index\",\"index_uuid\":\"_na_\",\"resource.type\":\"index_or_alias\",\"resource.id\":\"internal_image\",\"index\":\"internal_image\"},\"status\":404}"], "level"=>300, "level_name"=>"WARNING", "channel"=>"app", "datetime"=>{"date"=>"2021-04-15 14:39:21.810749", "timezone_type"=>3, "timezone"=>"UTC"}, "extra"=>{"url"=>"/internal_images/163ba8e6-1870-4120-951e-ab662cfdb16e", "ip"=>"85.169.238.110", "http_method"=>"GET", "server"=>"", "referrer"=>"https://admin.preprod.origami.decathlon.net/", "token"=>{"username"=>"admin", "authenticated"=>true, "roles"=>["ROLE_ADMIN"]}}, "log"=>"{\"message\":\"Response\",\"context\":[\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"index_not_found_exception\\\",\\\"reason\\\":\\\"no such index\\\",\\\"index_uuid\\\":\\\"_na_\\\",\\\"resource.type\\\":\\\"index_or_alias\\\",\\\"resource.id\\\":\\\"internal_image\\\",\\\"index\\\":\\\"internal_image\\\"}],\\\"type\\\":\\\"index_not_found_exception\\\",\\\"reason\\\":\\\"no such index\\\",\\\"index_uuid\\\":\\\"_na_\\\",\\\"resource.type\\\":\\\"index_or_alias\\\",\\\"resource.id\\\":\\\"internal_image\\\",\\\"index\\\":\\\"internal_image\\\"},\\\"status\\\":404}\"],\"level\":300,\"level_name\":\"WARNING\",\"channel\":\"app\",\"datetime\":{\"date\":\"2021-04-15 14:39:21.810749\",\"timezone_type\":3,\"timezone\":\"UTC\"},\"extra\":{\"url\":\"/internal_images/163ba8e6-1870-4120-951e-ab662cfdb16e\",\"ip\":\"85.169.238.110\",\"http_method\":\"GET\",\"server\":\"\",\"referrer\":\"https://XXXXXX.COM/\",\"token\":{\"username\":\"admin\",\"authenticated\":true,\"roles\":[\"ROLE_ADMIN\"]}}}\n", "stream"=>"stderr", "docker"=>{"container_id"=>"14a3179e6994c404d2ac945bb202ddfe6f30586e9e74945d26a0682af9d27e85"}, "kubernetes"=>{"container_name"=>"api-php", "namespace_name"=>"deci-white-deer", "pod_name"=>"api-php-79b7f6df4d-twqww", "pod_id"=>"16c4d5ea-afbd-4366-b734-bb1dd2b9cd49", "labels"=>{"pod-template-hash"=>"79b7f6df4d", "release"=>"preproduction-deci-white-deer-api", "run"=>"api-php"}, "host"=>"gke-cluster-origami--origami-pool-n2d-8c126215-sqsg", "master_url"=>"https://10.52.0.1:443/api", "namespace_id"=>"b04167bd-4900-11ea-aeb3-4201ac10000a", "namespace_labels"=>{"apps"=>"origami", "status"=>"private"}}}
2021-04-15 17:35:28 +0000 [warn]: dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch" location=nil tag="kubernetes.var.log.containers.gke-metrics-agent-r78zw_kube-system_gke-metrics-agent-60ae233e3df7fe6696b057d8ca4b13346022cefd5baa120e44f241421eee6ed3.log" time=2021-04-15 17:35:22.134896046 +0000 record={"level"=>"error", "ts"=>1618508122.1345887, "caller"=>"collectors/summary_api.go:225", "msg"=>"SummaryCollector: Failed to export metrics", "error"=>"rpc error: code = DeadlineExceeded desc = context deadline exceeded", "stacktrace"=>"gke-internal/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors.(*SummaryAPICollector).collectMetric\n\t/go/src/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors/summary_api.go:225\ngke-internal/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors.(*SummaryAPICollector).scrapeAndExport\n\t/go/src/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors/summary_api.go:171\ngke-internal/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors.(*SummaryAPICollector).CollectMetrics\n\t/go/src/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors/summary_api.go:156", "log"=>"{\"level\":\"error\",\"ts\":1618508122.1345887,\"caller\":\"collectors/summary_api.go:225\",\"msg\":\"SummaryCollector: Failed to export metrics\",\"error\":\"rpc error: code = DeadlineExceeded desc = context deadline exceeded\",\"stacktrace\":\"gke-internal/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors.(*SummaryAPICollector).collectMetric\\n\\t/go/src/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors/summary_api.go:225\\ngke-internal/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors.(*SummaryAPICollector).scrapeAndExport\\n\\t/go/src/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors/summary_api.go:171\\ngke-internal/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors.(*SummaryAPICollector).CollectMetrics\\n\\t/go/src/gke-logmon/gke-metrics-agent/receivers/kube_node_receiver/collectors/summary_api.go:156\"}\n", "stream"=>"stderr", "docker"=>{"container_id"=>"60ae233e3df7fe6696b057d8ca4b13346022cefd5baa120e44f241421eee6ed3"}, "kubernetes"=>{"container_name"=>"gke-metrics-agent", "namespace_name"=>"kube-system", "pod_name"=>"gke-metrics-agent-r78zw", "pod_id"=>"2dbb69ff-5acd-4921-bc3b-1913a642e788", "labels"=>{"component"=>"gke-metrics-agent", "controller-revision-hash"=>"55b54b8965", "k8s-app"=>"gke-metrics-agent", "pod-template-generation"=>"3"}, "host"=>"gke-cluster-origami--origami-pool-n2d-8c126215-sqsg", "master_url"=>"https://10.52.0.1:443/api", "namespace_id"=>"580dcc11-555b-11e9-b800-4201ac100004", "namespace_labels"=>{"name"=>"kube-system", "purpose"=>"services", "certmanager_k8s_io/disable-validation"=>"true"}}}
2021-04-16 08:31:22 +0000 [warn]: dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch" location=nil tag="kubernetes.var.log.containers.api-php-79b7f6df4d-twqww_deci-white-deer_api-php-14a3179e6994c404d2ac945bb202ddfe6f30586e9e74945d26a0682af9d27e85.log" time=2021-04-16 08:31:16.999574382 +0000 record={"message"=>"Response", "context"=>["{\"error\":{\"root_cause\":[{\"type\":\"index_not_found_exception\",\"reason\":\"no such index\",\"index_uuid\":\"_na_\",\"resource.type\":\"index_or_alias\",\"resource.id\":\"internal_image\",\"index\":\"internal_image\"}],\"type\":\"index_not_found_exception\",\"reason\":\"no such index\",\"index_uuid\":\"_na_\",\"resource.type\":\"index_or_alias\",\"resource.id\":\"internal_image\",\"index\":\"internal_image\"},\"status\":404}"], "level"=>300, "level_name"=>"WARNING", "channel"=>"app", "datetime"=>{"date"=>"2021-04-16 08:31:16.999394", "timezone_type"=>3, "timezone"=>"UTC"}, "extra"=>{"url"=>"/internal_images/42dda104-069b-483e-bca8-a9c30bcac5de", "ip"=>"81.66.65.89", "http_method"=>"GET", "server"=>"", "referrer"=>"https://admin.preprod.origami.decathlon.net/", "token"=>{"username"=>"admin", "authenticated"=>true, "roles"=>["ROLE_ADMIN"]}}, "log"=>"{\"message\":\"Response\",\"context\":[\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"index_not_found_exception\\\",\\\"reason\\\":\\\"no such index\\\",\\\"index_uuid\\\":\\\"_na_\\\",\\\"resource.type\\\":\\\"index_or_alias\\\",\\\"resource.id\\\":\\\"internal_image\\\",\\\"index\\\":\\\"internal_image\\\"}],\\\"type\\\":\\\"index_not_found_exception\\\",\\\"reason\\\":\\\"no such index\\\",\\\"index_uuid\\\":\\\"_na_\\\",\\\"resource.type\\\":\\\"index_or_alias\\\",\\\"resource.id\\\":\\\"internal_image\\\",\\\"index\\\":\\\"internal_image\\\"},\\\"status\\\":404}\"],\"level\":300,\"level_name\":\"WARNING\",\"channel\":\"app\",\"datetime\":{\"date\":\"2021-04-16 08:31:16.999394\",\"timezone_type\":3,\"timezone\":\"UTC\"},\"extra\":{\"url\":\"/internal_images/42dda104-069b-483e-bca8-a9c30bcac5de\",\"ip\":\"81.66.65.89\",\"http_method\":\"GET\",\"server\":\"\",\"referrer\":\"https://XXXXXX.COM/\",\"token\":{\"username\":\"admin\",\"authenticated\":true,\"roles\":[\"ROLE_ADMIN\"]}}}\n", "stream"=>"stderr", "docker"=>{"container_id"=>"14a3179e6994c404d2ac945bb202ddfe6f30586e9e74945d26a0682af9d27e85"}, "kubernetes"=>{"container_name"=>"api-php", "namespace_name"=>"deci-white-deer", "pod_name"=>"api-php-79b7f6df4d-twqww", "pod_id"=>"16c4d5ea-afbd-4366-b734-bb1dd2b9cd49", "labels"=>{"pod-template-hash"=>"79b7f6df4d", "release"=>"preproduction-deci-white-deer-api", "run"=>"api-php"}, "host"=>"gke-cluster-origami--origami-pool-n2d-8c126215-sqsg", "master_url"=>"https://10.52.0.1:443/api", "namespace_id"=>"b04167bd-4900-11ea-aeb3-4201ac10000a", "namespace_labels"=>{"apps"=>"origami", "status"=>"private"}}}
I have an High memory usage on log-forwarder due to this error
. can somebody please help on this?
