Elastic agent send _nil id to Logstash

Elastic Stack Elastic Agent
1

Hi there!

we'reusing the logstash output to send data from the elastic agent but we're are facing a strange behaviour.

Our logstash pipeline

input {
  elastic_agent {
    port => 5044
    ssl => true
    ssl_certificate_authorities => ["/etc/logstash/certs/ca.crt"]
    ssl_certificate => "/etc/logstash/certs/logstash.crt"
    ssl_key => "/etc/logstash/certs/logstash.pkcs8.key"
    ssl_client_authentication => "required"
  }
}



output {
  elasticsearch {
    hosts => ["<masked>"]
    api_key => "<masked>"
    data_stream => true
    ssl => true
    cacert => "/etc/logstash/certs/http_ca.crt"
  }
}

Is not doing anything to the incoming events but the output fails due to 409 as below

[2024-08-30T08:17:20,314][WARN ][logstash.outputs.elasticsearch] Failed action {:status=>409, :action=>["create", {:_id=>nil, :_index=>"metrics-elastic_agent.metricbeat-default", :routing=>nil}, {"tags"=>["beats_input_raw_event"], "@timestamp"=>2024-08-30T08:17:09.149Z, "metricset"=>{"name"=>"stats", "period"=>60000}, "event"=>{"module"=>"beat", "duration"=>3772750, "dataset"=>"elastic_agent.metricbeat"}, "data_stream"=>{"namespace"=>"default", "dataset"=>"elastic_agent.metricbeat", "type"=>"metrics"}, "component"=>{"id"=>"elasticsearch/metrics-fleet-logstash-output", "binary"=>"metricbeat"}, "service"=>{"type"=>"beat", "name"=>"beat", "address"=>"http://unix/stats"}, "elastic_agent"=>{"snapshot"=>false, "id"=>"d81158a1-a287-47bb-b192-4ca373d60a05", "version"=>"8.15.0", "process"=>"metricbeat"}, "beat"=>{"id"=>"bddc0698-9a66-440c-b98a-c4c3c1a84d6d", "stats"=>{"beat"=>{"version"=>"8.15.0", "name"=>"cfelk20", "host"=>"cfelk20", "uuid"=>"bddc0698-9a66-440c-b98a-c4c3c1a84d6d", "type"=>"metricbeat"}, "system"=>{"load"=>{"norm"=>{"15"=>0.0525, "1"=>0.015, "5"=>0.0475}, "15"=>0.42, "1"=>0.12, "5"=>0.38}, "cpu"=>{"cores"=>8}}, "cpu"=>{"system"=>{"ticks"=>690, "time"=>{"ms"=>690}}, "user"=>{"ticks"=>8820, "time"=>{"ms"=>8820}}, "total"=>{"ticks"=>9510, "value"=>9510, "time"=>{"ms"=>9510}}}, "libbeat"=>{"output"=>{"events"=>{"toomany"=>0, "duplicates"=>0, "acked"=>43535, "batches"=>143, "failed"=>951, "dropped"=>0, "total"=>44486, "active"=>0}, "read"=>{"bytes"=>10480, "errors"=>0}, "write"=>{"bytes"=>5877463, "errors"=>2}, "type"=>"logstash"}, "pipeline"=>{"events"=>{"filtered"=>0, "retry"=>3874, "failed"=>0, "published"=>44473, "dropped"=>0, "active"=>938, "total"=>44473}, "clients"=>11, "queue"=>{"acked"=>0, "max_events"=>0}}, "config"=>{"running"=>11, "starts"=>11, "stops"=>0, "reloads"=>0}}, "cgroup"=>{"cpu"=>{"id"=>"elastic-agent.service", "stats"=>{"throttled"=>{"ns"=>0, "periods"=>0}, "periods"=>0}}, "memory"=>{"mem"=>{"usage"=>{"bytes"=>603074560.0}}, "id"=>"elastic-agent.service"}}, "handles"=>{"open"=>24, "limit"=>{"hard"=>524288, "soft"=>524288}}, "runtime"=>{"goroutines"=>101}, "uptime"=>{"ms"=>1561527}, "info"=>{"ephemeral_id"=>"1c5a6bc9-32a2-461d-bd9d-afcaabbc55bc", "version"=>"8.15.0", "name"=>"metricbeat", "uptime"=>{"ms"=>1561527.0}}, "memstats"=>{"rss"=>252702720, "gc_next"=>122960528, "memory"=>{"alloc"=>105220952, "total"=>2654801816}}}, "type"=>"metricbeat"}, "agent"=>{"ephemeral_id"=>"24f33ef2-89ca-4980-848c-683e04fddfab", "id"=>"d81158a1-a287-47bb-b192-4ca373d60a05", "name"=>"cfelk20", "version"=>"8.15.0", "type"=>"metricbeat"}, "ecs"=>{"version"=>"8.0.0"}, "@version"=>"1", "host"=>{"hostname"=>"cfelk20", "id"=>"27788f2a7458418288ff591cb17fd3db", "name"=>"cfelk20", "ip"=>["192.168.143.70"], "containerized"=>false, "architecture"=>"x86_64", "mac"=>["00-50-56-A0-23-EB"], "os"=>{"kernel"=>"5.15.0-94-generic", "version"=>"22.04.3 LTS (Jammy Jellyfish)", "name"=>"Ubuntu", "codename"=>"jammy", "platform"=>"ubuntu", "family"=>"debian", "type"=>"linux"}}}], :response=>{"create"=>{"status"=>409, "error"=>{"type"=>"version_conflict_engine_exception", "reason"=>"[DWLlrUJoMHXn9hrzAAABkaJcFB0][LMjuLkkXPvrprtzeJ0ANC89rEbpxqMT5GaiYBoYwPo4ECosXSVP4lfV_5KYo@2024-08-30T08:17:09.149Z]: version conflict, document already exists (current version [1])", "index_uuid"=>"XDwNk-aTSVOctgoN4FzuTw", "shard"=>"0", "index"=>".ds-metrics-elastic_agent.metricbeat-default-2024.08.30-000012"}}}}

Seems like the event doesn't have an _id set and gives the error.

Any suggestion?

Thanks a lot!