Hello All,
i was trying to send k8s container logs to Elasticsearch through filebeat. we are getting more logs that expected and also it is trigger the below warning continuously from filebeat side and it trying write in old indices . can some one suggest on this. Thank you in advance.
**2023-05-18T12:29:12.620Z WARN [elasticsearch] elasticsearch/client.go:408 Cannot index event publisher.Event{Content:beat.Event{Timestamp:time.Time{wall:0x2d69e370, ext:63819632370, loc:(*time.Location)(nil)}, Meta:null, Fields:{"agent":{"ephemeral_id":"aab60380-cd1b-4e0f-99cc-72582a6fca1f","hostname":"filebeat-dr-ds-qckxz","id":"cafca62d-ba9d-44b9-b1fa-d0897337e9b6","name":"aks-proddefpool-26950791-vmss00000m","type":"filebeat","version":"7.10.1"},"app_environment":"aks","cluster":"cricketonline-eastus2-prod-app-aks","container":{"id":"ef1fe7c50970bdd4856cadabb6e2bc71a8aef871e95acaa1703d85e0fcada80d","image":{"name":"docker.io/spvest/azure-keyvault-controller:1.2.3"},"runtime":"containerd"},"datacenter":"eastus2","ecs":{"version":"1.6.0"},"environment":"prod","instance":"domain","kubernetes":{"container":{"image":"docker.io/spvest/azure-keyvault-controller:1.2.3","name":"controller"},"labels":{"app_kubernetes_io/component":"azure-key-vault-to-k8s-akv2k8s-controller","app_kubernetes_io/instance":"azure-key-vault-to-k8s","app_kubernetes_io/name":"akv2k8s","pod-template-hash":"657c4bb7fc"},"namespace":"extensions","node":{"name":"aks-proddefpool-26950791-vmss00000m"},"pod":{"name":"azure-key-vault-to-k8s-akv2k8s-controller-657c4bb7fc-fbxt2","uid":"fda3588f-4926-427b-9db2-f766bfd2daa2"},"replicaset":{"name":"azure-key-vault-to-k8s-akv2k8s-controller-657c4bb7fc"}},"log":{"file":{"path":"/var/log/containers/azure-key-vault-to-k8s-akv2k8s-controller-657c4bb7fc-fbxt2_extensions_controller-ef1fe7c50970bdd4856cadabb6e2bc71a8aef871e95acaa1703d85e0fcada80d.log"},"offset":26322670},"message":"E0514 03:39:30.761851 1 worker.go:92] Failed to process key cwdigital-prod/sb-send-kount-cred-sync. Reason: Failed to get secret for 'sb-send-kount-cred-sync' from Azure Key Vault 'prod-eastus2-kv-25366'","tags":["Microservice_Log"]}, Private:file.State{Id:"native::7249479-2049", PrevId:"", Finished:false, Fileinfo:(os.fileStat)(0xc000432270), Source:"/var/log/containers/azure-key-vault-to-k8s-akv2k8s-controller-657c4bb7fc-fbxt2_extensions_controller-ef1fe7c50970bdd4856cadabb6e2bc71a8aef871e95acaa1703d85e0fcada80d.log", Offset:26322920, Timestamp:time.Time{wall:0xc111a2ea2f84e407, ext:169250334, loc:(time.Location)(0x64d0ce0)}, TTL:-1, Type:"container", Meta:map[string]string(nil), FileStateOS:file.StateOS{Inode:0x6e9e47, Device:0x801}, IdentifierName:"native"}, TimeSeries:false}, Flags:0x1, Cache:publisher.EventCache{m:common.MapStr(nil)}} (status=403): {"type":"cluster_block_exception","reason":"index [indexname] blocked by: [FORBIDDEN/8/index write (api)];"}