Kibana version:
7.9.1
Elasticsearch version:
7.9.1
APM Server version:
7.9.1
Original install method (e.g. download page, yum, deb, from source, etc.) and version:
Official docker images
Description of the problem including expected versus actual behavior. Please include screenshots (if relevant):
I'm trying to use Elastic APM as backend for distibuted tracing. I am using jaeger-agent to collect traces from laravel app. My previous tracing setup with docker-compose was working flawlessly (jaeger agent + collector + jaeger-queryUi), Now i modified my setup to use APM as a backend by sending traces from my app -> Jaeger-Agent -> APM server via gRPC. I followed this jaeger integration guide https://www.elastic.co/guide/en/apm/server/current/jaeger.html but when i start sending traces i'm geting these errors in APM logs:
2020-09-13T20:07:31.202Z WARN [elasticsearch] elasticsearch/client.go:407 Cannot index event publisher.Event{Content:beat.Event{Timestamp:time.Time{wall:0x148af8e0, ext:63735624444, loc:(*time.Location)(nil)}, Meta:{"pipeline":"apm"}, Fields:{"agent":{"name":"Jaeger","version":"unknown"},"ecs":{"version":"1.5.0"},"labels":{"Lounge":"GatewayService-JaegerProvider","peer.ipv4":"172.21.0.3","peer.port":"80","sampler.param":"true","sampler.type":"const"},"observer":{"ephemeral_id":"98acea66-a148-497b-9eaa-c360900a367a","hostname":"ee30089a825b","id":"f9fd9f44-dbc7-4cdd-9403-54576dba007a","type":"apm-server","version":"7.9.1","version_major":7},"processor":{"event":"transaction","name":"transaction"},"service":{"language":{"name":"unknown"},"name":"Gateway-Service"},"timestamp":{"us":1600027644344652},"trace":{"id":"163470aa489b76d9"},"transaction":{"duration":{"us":5532199},"id":"163470aa489b76d9","name":"GET: /csrfd","result":"Success","sampled":true,"type":"fpm"},"url":{"full":"http:///csrfd","original":"/csrfd","path":"/csrfd","scheme":"http"}}, Private:interface {}(nil), TimeSeries:false}, Flags:0x1, Cache:publisher.EventCache{m:common.MapStr(nil)}} (status=400): {"type":"mapper_parsing_exception","reason":"failed to parse","caused_by":{"type":"class_cast_exception","reason":"class org.elasticsearch.index.mapper.ScaledFloatFieldMapper cannot be cast to class org.elasticsearch.index.mapper.ObjectMapper (org.elasticsearch.index.mapper.ScaledFloatFieldMapper is in unnamed module of loader java.net.FactoryURLClassLoader @50a13c2f; org.elasticsearch.index.mapper.ObjectMapper is in unnamed module of loader 'app')"}}
When i switch to jaeger-collector instead of APM - it gets all the tracing correctly.
Here is my docker-compose.yml:
version: "3"
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.1
networks:
- author-book-network
ports:
- "9200:9200"
- "9300:9300"
restart: on-failure
environment:
- cluster.name=jaeger-cluster
- discovery.type=single-node
- http.host=0.0.0.0
- transport.host=127.0.0.1
- ES_JAVA_OPTS=-Xms1000m -Xmx1000m
- xpack.security.enabled=false
volumes:
- esdata:/usr/share/elasticsearch/data
kibana:
image: docker.elastic.co/kibana/kibana:7.9.1
ports:
- "5601:5601"
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
- SERVER_NAME=kibana
#- ES.TAGS-AS-FIELDS.ALL=true
#- SERVER_HOST=0.0.0.0
depends_on:
- elasticsearch
networks:
- author-book-network
apm-server:
image: docker.elastic.co/apm/apm-server:7.9.1
depends_on:
- elasticsearch
- kibana
cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"]
cap_drop: ["ALL"]
ports:
- 8200:8200
networks:
- author-book-network
command: >
apm-server -e
-E apm-server.jaeger.grpc.enabled=true
-E apm-server.jaeger.grpc.host=":14250"
-E apm-server.rum.enabled=true
-E setup.kibana.host=kibana:5601
-E setup.template.settings.index.number_of_replicas=0
-E apm-server.kibana.enabled=true
-E apm-server.kibana.host=kibana:5601
-E output.elasticsearch.hosts=["elasticsearch:9200"]
-E ELASTIC_APM_LOG_FILE=stderr
-E ELASTIC_APM_LOG_FILE=debug
# healthcheck:
# interval: 10s
# retries: 12
# test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/
# jaeger-collector:
# image: jaegertracing/jaeger-collector
# ports:
# #- "14269:14269"
# - "14250:14250"
# - "14268:14268"
# - "14267:14267"
# - "9411:9411"
# networks:
# - author-book-network
# restart: on-failure
# environment:
# - SPAN_STORAGE_TYPE=elasticsearch
# command: [
# "--es.server-urls=http://elasticsearch:9200",
# "--es.num-shards=1",
# "--es.num-replicas=0",
# "--log-level=error"
# ]
# depends_on:
# - elasticsearch
jaeger-agent:
image: jaegertracing/jaeger-agent
hostname: jaeger-agent
#command: ["--reporter.grpc.host-port=jaeger-collector:14250"]
command: ["--reporter.grpc.host-port=apm-server:14250"]
ports:
- "5775:5775/udp"
- "6831:6831/udp"
- "6832:6832/udp"
- "5778:5778"
networks:
- author-book-network
restart: on-failure
environment:
- SPAN_STORAGE_TYPE=elasticsearch
depends_on:
#- jaeger-collector
- apm-server
jaeger-query:
image: jaegertracing/jaeger-query
environment:
- SPAN_STORAGE_TYPE=elasticsearch
- no_proxy=localhost
ports:
- "16686:16686"
- "16687:16687"
networks:
- author-book-network
restart: on-failure
command: [
"--es.server-urls=http://elasticsearch:9200",
"--span-storage.type=elasticsearch",
"--log-level=debug"
]
depends_on:
- jaeger-agent
volumes:
esdata:
driver: local
networks:
author-book-network:
external: true
Steps to reproduce:
- Install Elasticsearch, Kibana and APM through docker-compose
- Use Jaeger-Agent to send tracing to APM
