APM agent remote configuration access forbidden / unauthorized

[muhammadkholidb]

Hi, I am working on Elastic APM on my project and saw an error on the logs when running with apm java agent:

2022-02-08 17:21:33,546 [elastic-apm-remote-config-poller] ERROR co.elastic.apm.agent.configuration.ApmServerConfigurationSource - Remote configuration is not available. Check the connection between APM Server and Kibana.

I used kibana_system user for APM Server to communicate with Kibana as specified in my docker-compose.yml configuration below. I used kibana_system user because it has privillege to read and view_index_metadata on index apm-* as described here.

Can somebody help me to fix this?

Is there something wrong with my Elastic stack configuration on my docker-compose.yml?

Kibana version: 7.17.0

Elasticsearch version: 7.17.0

APM Server version: 7.17.0

APM Agent language and version: Java agent version 1.28.4

Original install method (e.g. download page, yum, deb, from source, etc.) and version:
docker-compose.yml

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION}
    environment:
      - bootstrap.memory_lock=true
      - cluster.name=docker-cluster
      - cluster.routing.allocation.disk.threshold_enabled=false
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=true
      - xpack.security.authc.api_key.enabled=true
    ulimits:
      memlock:
        hard: -1
        soft: -1
    volumes:
      - es_data:/usr/share/elasticsearch/data
      - es_config:/usr/share/elasticsearch/config
    ports:
      - 8080:9200
    networks:
      - elastic
    healthcheck:
      interval: 20s
      retries: 10
      test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'

  kibana:
    image: docker.elastic.co/kibana/kibana:${VERSION}
    depends_on:
      elasticsearch:
        condition: service_healthy
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
      SERVER_PUBLICBASEURL: http://34.101.103.104:2511
      ELASTICSEARCH_USERNAME: ${KIBANA_SYSTEM_USERNAME}
      ELASTICSEARCH_PASSWORD: ${KIBANA_SYSTEM_PASSWORD}
    ports:
      - 2511:5601
    networks:
      - elastic
    healthcheck:
      interval: 10s
      retries: 20
      test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null -u ${KIBANA_SYSTEM_USERNAME}:${KIBANA_SYSTEM_PASSWORD} http://localhost:5601/api/status

  apm-server:
    image: docker.elastic.co/apm/apm-server:${VERSION}
    depends_on:
      elasticsearch:
        condition: service_healthy
      kibana:
        condition: service_healthy
    cap_add: [ "CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID" ]
    cap_drop: [ "ALL" ]
    ports:
      - 2512:8200
    networks:
      - elastic
    command: >
      apm-server -e
        -E apm-server.rum.enabled=true
        -E setup.kibana.host=kibana:5601
        -E setup.template.settings.index.number_of_replicas=0
        -E apm-server.kibana.enabled=true
        -E apm-server.kibana.host=kibana:5601
        -E apm-server.kibana.username=${KIBANA_SYSTEM_USERNAME}
        -E apm-server.kibana.password=${KIBANA_SYSTEM_PASSWORD}
        -E apm-server.auth.secret_token=${SECRET_TOKEN}
        -E output.elasticsearch.hosts=["elasticsearch:9200"]
        -E output.elasticsearch.username=${ELASTIC_USERNAME}
        -E output.elasticsearch.password=${ELASTIC_PASSWORD}
    healthcheck:
      interval: 10s
      retries: 12
      test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/

volumes:
  es_data:
    driver: local
  es_config:
    driver: local

networks:
  elastic:
    driver: bridge

Provide logs and/or server output (if relevant):
APM Server logs:

{"log.level":"error","@timestamp":"2022-02-08T10:21:33.524Z","log.logger":"request","log.origin":{"file.name":"middleware/log_middleware.go","file.line":60},"message":"service unavailable","service.name":"apm-server","url.original":"/config/v1/agents","http.request.method":"POST","user_agent.original":"apm-agent-java/1.28.4 (esdc-backend)","source.address":"103.105.35.64","http.request.body.bytes":465,"http.request.id":"113b2009-0e91-4666-9eba-8cbc8bbbdbb4","event.duration":64074195,"http.response.status_code":503,"error.message":"{\"statusCode\":403,\"error\":\"Forbidden\",\"message\":\"Forbidden\"}","ecs.version":"1.6.0"}

Kibana logs:

{"type":"log","@timestamp":"2022-02-08T10:21:33+00:00","tags":["warning","plugins","security","api-authorization"],"pid":7,"message":"User not authorized for \"/api/apm/settings/agent-configuration/search\": responding with 403"}
{"type":"response","@timestamp":"2022-02-08T10:21:33+00:00","tags":["access:apm"],"pid":7,"method":"post","statusCode":403,"req":{"url":"/api/apm/settings/agent-configuration/search","method":"post","headers":{"host":"kibana:5601","user-agent":"Elastic-apm-server/7.17.0 (linux; amd64; a1459eb3f24e7c054794a82f6c19b66975098866; 2022-01-28 08:52:26 +0000 UTC)","content-length":"68","accept":"application/json","content-type":"application/json","kbn-xsrf":"1","accept-encoding":"gzip"},"remoteAddress":"172.26.0.4","userAgent":"Elastic-apm-server/7.17.0 (linux; amd64; a1459eb3f24e7c054794a82f6c19b66975098866; 2022-01-28 08:52:26 +0000 UTC)"},"res":{"statusCode":403,"responseTime":62,"contentLength":60},"message":"POST /api/apm/settings/agent-configuration/search 403 62ms - 60.0B"}