Hi
I am using logstash running Kubernetes under ECK. I am ingesting both logs (filebeat) and metrics (metricbeat). Now i want ingest data from heartbeat. I get the following error:
[2023-11-14T13:22:23,598][INFO ][logstash.outputs.elasticsearch][main][80b6fcbcae1c789247b66f08ff48b2bde70d63d576edb68ec43e28cae9649bcd] Retrying failed action {:status=>403, :action=>["create", {:_id=>nil, :_index=>"testtype-http-straffe.skifte", :routing=>nil}, {"event"=>{"dataset"=>"http"}, "headers"=>{"http_accept"=>nil, "http_version"=>"HTTP/1.1", "accept_encoding"=>"gzip,deflate", "request_method"=>"POST", "x_forwarded_scheme"=>"https", "http_host"=>"logstash.elastic-app-logging.domdev.lan", "http_user_agent"=>"Manticore 0.9.1", "x_forwarded_port"=>"443", "x_forwarded_for"=>"10.209.254.65", "x_forwarded_host"=>"logstash.elastic-app-logging.domdev.lan", "x_real_ip"=>"10.209.254.65", "x_scheme"=>"https", "content_type"=>"application/json", "x_forwarded_proto"=>"https", "x_request_id"=>"1310b8c7d7f380d50851e23aaa5c29bc", "request_path"=>"/", "content_length"=>"340700"}, "summary"=>{"up"=>1, "down"=>0}, "@version"=>"1", "http"=>{"response"=>{"status_code"=>200, "body"=>{"hash"=>"caf313a2a2492aab037f25ea846572b4d2dd0dbf0c29936da7c13d9b4e005224", "bytes"=>243}, "mime_type"=>"application/json", "headers"=>{"X-Xss-Protection"=>"0", "Date"=>"Tue, 14 Nov 2023 13:11:19 GMT", "X-Frame-Options"=>"DENY", "Expires"=>"0", "Vary"=>["Origin", "Access-Control-Request-Method", "Access-Control-Request-Headers"], "Content-Type"=>"application/vnd.spring-boot.actuator.v3+json", "Cache-Control"=>"no-cache, no-store, max-age=0, must-revalidate", "X-Content-Type-Options"=>"nosniff", "Pragma"=>"no-cache"}}, "rtt"=>{"write_request"=>{"us"=>42}, "validate"=>{"us"=>780}, "response_header"=>{"us"=>603}, "total"=>{"us"=>930}, "content"=>{"us"=>176}}}, "agent"=>{"ephemeral_id"=>"b505ecd6-f6cd-4403-b948-4ad5009aa940", "type"=>"heartbeat", "id"=>"37825afd-a98a-4c5b-b514-af668a5bda5d", "version"=>"8.10.2", "name"=>"heartbeat-7d5985d586-2n6jf"}, "ecs"=>{"version"=>"8.0.0"}, "kubernetes"=>{"namespace_labels"=>{"field_cattle_io/projectId"=>"p-8gvfm", "kubernetes_io/metadata_name"=>"development", "argocd_argoproj_io/instance"=>"solution-development-development-wrapper"}, "container"=>{"name"=>"spring"}, "namespace_uid"=>"7d3a393e-a64c-4cf0-a75f-1b0c8df6e172", "labels"=>{"release"=>"cpr-abonnement", "pod-template-hash"=>"6658849bd6", "type"=>"application", "appX"=>"spring"}, "pod"=>{"uid"=>"ec339645-196a-48e7-bd1c-497297b0bdd0", "ip"=>"10.42.7.165", "name"=>"cpr-abonnement-spring-6658849bd6-q2qw6"}, "replicaset"=>{"name"=>"cpr-abonnement-spring-6658849bd6"}, "namespace"=>"development", "deployment"=>{"name"=>"cpr-abonnement-spring"}, "node"=>{"uid"=>"68adf4d6-a273-4383-af8a-d97233cb6cd7", "hostname"=>"rke-node-69", "name"=>"rke-node-69", "labels"=>{"node_longhorn_io/create-default-disk"=>"config", "beta_kubernetes_io/arch"=>"amd64", "kubernetes_io/arch"=>"amd64", "node_kubernetes_io/instance-type"=>"rke2", "kubernetes_io/hostname"=>"rke-node-69", "beta_kubernetes_io/instance-type"=>"rke2", "beta_kubernetes_io/os"=>"linux", "kubernetes_io/os"=>"linux"}}}, "container"=>{"runtime"=>"containerd", "id"=>"b1bd42718c08d4522eaed67e4c979c8f0e7e81025c2f2e36a4c82995c8b673a5", "image"=>{"name"=>"sosconreg.azurecr.io/cpr-abonnement-service:2023-11-14T08.14.01.254776358"}}, "@timestamp"=>2023-11-14T13:11:19.944Z, "monitor"=>{"id"=>"auto-http-0XB2965F489EF2DE6A-85eb9366d9e99796", "name"=>"", "check_group"=>"47433775-82ef-11ee-99ed-2af066366adf", "type"=>"http", "ip"=>"10.42.7.165", "duration"=>{"us"=>980}, "status"=>"up", "timespan"=>{"gte"=>"2023-11-14T13:11:19.945Z", "lt"=>"2023-11-14T13:11:29.945Z"}}, "tags"=>["beats_input_raw_event"], "tcp"=>{"rtt"=>{"connect"=>{"us"=>121}}}, "state"=>{"id"=>"default-18bcda05868-0", "ends"=>nil, "started_at"=>"2023-11-14T11:38:39.592827403Z", "flap_history"=>, "up"=>1113, "down"=>0, "checks"=>1113, "status"=>"up", "duration_ms"=>"5560352"}, "url"=>{"domain"=>"10.42.7.165", "path"=>"/actuator/health/readiness", "scheme"=>"http", "full"=>"http://10.42.7.165:8081/actuator/health/readiness", "port"=>8081}, "host"=>"10.42.2.4", "service"=>{"environment"=>"development"}, "data_stream"=>{"type"=>"testtype", "namespace"=>"straffe.skifte", "dataset"=>"http"}}], :error=>{"type"=>"security_exception", "reason"=>"action [indices:data/write/bulk[s]] is unauthorized for user [elasticsearch-logstash-application-logging-elasticsearch-elasticsearch-application-logging-logstash-user] with effective roles [eck_logstash_user_role] on indices [testtype-http-straffe.skifte], this action is granted by the index privileges [create_doc,create,delete,index,write,all]"}}
My logstash config looks like this:
{{- with $.Values.elastic }}
apiVersion: logstash.k8s.elastic.co/v1alpha1
kind: Logstash
metadata:
name: logstash-{{ .clusterName }}
namespace: {{ .namespace }}
spec:
version: {{ .version }}
count: 2
podTemplate:
spec:
containers:
- name: logstash
resources:
requests:
memory: 2Gi
cpu: 1
limits:
memory: 4Gi
elasticsearchRefs:
- name: elasticsearch-{{ .clusterName }}
clusterName: elastic
config:
pipeline:
workers: 4
services:
- name: http
service:
spec:
ports:
- port: 8080
name: "http"
protocol: TCP
targetPort: 8080
pipelines:
- pipeline.id: main
config.string: |
input {
http {
port => 8080
codec => json
ecs_compatibility => 'disabled'
}
}
output {
#stdout {
# codec => rubydebug
#}
elasticsearch {
hosts => [ "${ELASTIC_ES_HOSTS}" ]
user => "${ELASTIC_ES_USER}"
password => "${ELASTIC_ES_PASSWORD}"
ssl_certificate_authorities => "${ELASTIC_ES_SSL_CERTIFICATE_AUTHORITY}"
data_stream => "true"
}
}
{{- end }}
Everything works fine for all data where data_stream.type is "logs" or "metrics", where I have old index templates. But even though I create a new index template for "testtype", I get the error. If I change the data_stream.type in the above to metrics, it works?
Br
Casper
