Apm-server not getting connected to kibana. Using v 7.17.1 for all

[Amol_Nagotkar]

Hi all,

issue is cant see index here http://127.0.0.1:9200/_cat/indices

------------docker-compose file---------------
version: '3'

services:
elasticsearch:
container_name: elasticsearch
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
ports:
- "9200:9200"
environment:
- xpack.security.enabled=true
- discovery.type=single-node
- ES_JAVA_OPTS=-Xmx2g -Xms2g
volumes:
# - ./cert:/usr/share/elasticsearch/cert
- ./elasticsearch.yml:/usr/share/elasticsearch/elasticsearch.yml

kibana:
container_name: kibana
image: docker.elastic.co/kibana/kibana:7.17.1
ports:
- "5601:5601"
depends_on:
- elasticsearch
# privileged: true
volumes:
# - ./cert:/usr/share/kibana/cert
- ./kibana.yml:/usr/share/kibana/config/kibana.yml
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
- elasticsearch.username=kibana_system
- elasticsearch.password=password
# - SERVER_SSL_ENABLED=true
# - SERVER_SSL_CERTIFICATE=/usr/share/kibana/cert/elastic-certificates.crt
# - SERVER_SSL_KEY=/usr/share/kibana/cert/elastic-certificates.key

filebeat:
build:
context: ./filebeat
container_name: filebeat
command: filebeat -e -strict.perms=false
volumes:
# - ./cert:/usr/share/filebeat/cert
- /path:/app/logs/template
- /path:/app/logs/shortcode-generation
depends_on:
- elasticsearch
- kibana
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
apm-server:
image: docker.elastic.co/apm/apm-server:7.17.1
container_name: apm-server
environment:
- apm-server.host=0.0.0.0:8200
- output.elasticsearch.hosts=["http://elasticsearch:9200"]
- apm-server.instrumentation.enabled=true
- apm-server.instrumentation.environment=local
- apm-server.instrumentation.hosts=["http://localhost:8200"]
ports:
- "8200:8200"
depends_on:
- elasticsearch
volumes:
# # - ./cert:/usr/share/apm-server/cert
- ./apm-server.yml:/usr/share/apm-server/apm-server.yml
template:
network_mode: "host"
build:
context: .
dockerfile: Dockerfile
container_name: template
depends_on:
- elasticsearch
env_file:
- .env
volumes:
- /etc/hosts:/etc/hosts
- ./logs/template:/app/logs

volumes:
elasticsearch_data:
driver: local

----------filebeat.yml -----------------
name: "shortify-filebeat"
logging.metrics.enabled: true
xpack.security.enabled: true
xpack.monitoring.enabled: true
setup.ilm.enabled: true

filebeat.inputs:

• type: log
  scan_frequency: 1s
  enabled: true
  paths:

  • /app/logs/template/*.log
    fields:
    service: template
    fields_under_root: true
    json:
    keys_under_root: true
    overwrite_keys: true
    message_key: 'message'

• type: log
  scan_frequency: 1s
  enabled: true
  paths:

  • /app/logs/shortcode-generation/*.log
    fields:
    service: shortcode-generation
    fields_under_root: true
    json:
    keys_under_root: true
    overwrite_keys: true
    message_key: 'message'

output.elasticsearch:
hosts: ["elasticsearch:9200"]
index: "shortify"
username: "elastic"
password: "password"

setup.template:
name: 'shortify'
pattern: 'shortify-*'
enabled: true

--------------elasticsearch.yml--------------
xpack.security.enabled: true

discovery.type: single-node

network.host: 127.0.0.1

http.port: 9200

-----------kibana.yml--------------------
server.name: "kibana"

server.host: "0.0.0.0" # Bind to all network interfaces

elasticsearch.hosts: ["http://127.0.0.1:9200"]

elasticsearch.username: "kibana_system"

elasticsearch.password: "password"

xpack.security.encryptionKey: "a-z"

xpack.encryptedSavedObjects.encryptionKey: "a-z"

xpack.reporting.encryptionKey : "a-z"

xpack.security.session.idleTimeout: "1h"

xpack.security.session.lifespan: "30d"

elasticsearch.ssl.verificationMode: none

------------apm-server.yml---------------
apm-server:
host: "0.0.0.0:8200"

output.elasticsearch:
hosts: ["http://localhost:9200"]

username: "elastic"
password: "password"

instrumentation:

enabled: true
environment: "local"

hosts:

• http://localhost:8200

----------------i have created roles and assigned it to user also------------
curl --location 'http://localhost:9200/_xpack/security/role/filebeat_writer'
--header 'Content-Type: application/json'
--header 'Authorization: Basic ZWxhc3RpYzp2aXZhQDEyMw=='
--data '{
"cluster": ["manage_index_templates", "monitor"],
"indices": [
{
"names": [ "filebeat-","shortify-"],
"privileges": ["write","create_index"]
}
]
}'

curl --location 'http://localhost:9200/_xpack/security/user/filebeat_internal'
--header 'Content-Type: application/json'
--header 'Authorization: Basic ZWxhc3RpYzp2aXZhQDEyMw=='
--data-raw '{
"password" : "password",
"roles" : [ "filebeat_writer"],
"full_name" : "Internal Filebeat User"
}'

i am able to create role and assign it to user.
but not able to see indexes.

thanks in advance

[Musab_Dogan]

Let's diagnose the issue with the followings.

1. Update the filebeat.yml and use elastic user temporarily to see if it's an authentication issue.
2. Check filebeat logs to find some ERROR related to data flow.
3. Check the filebeat.paths to see if is there any data to send Elasticsearch. /app/logs/shortcode-generation/*.log and /app/logs/template/*.log

[Amol_Nagotkar]

I am able see index now.
using v 7.17.1 for all
Now i am getting error in apm-server. (apm-server not getting connected to kibana)

failed to obtain connection to Kibana: fail to get the Kibana version: HTTP GET request to http://localhost:5601/api/status fails: fail to execute the HTTP GET request: Get "http://localhost:5601/api/status\": dial tcp [::1]:5601: connect: cannot assign requested address.

{"log.level":"error","@timestamp":"2024-08-16T11:20:48.581Z","log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/output.go","file.line":154},"message":"Failed to connect to backoff(elasticsearch(http://localhost:9200)): Get \"http://localhost:9200\": dial tcp [::1]:9200: connect: cannot assign requested address","service.name":"apm-server","ecs.version":"1.6.0"} {"log.level":"info","@timestamp":"2024-08-16T11:20:48.581Z","log.logger":"publisher_pipeline_output","log.origin":{"file.name":"pipeline/output.go","file.line":145},"message":"Attempting to reconnect to backoff(elasticsearch(http://localhost:9200)) with 8 reconnect attempt(s)","service.name":"apm-server","ecs.version":"1.6.0"} {"log.level":"info","@timestamp":"2024-08-16T11:20:48.581Z","log.logger":"publisher","log.origin":{"file.name":"pipeline/retry.go","file.line":219},"message":"retryer: send unwait signal to consumer","service.name":"apm-server","ecs.version":"1.6.0"} {"log.level":"info","@timestamp":"2024-08-16T11:20:48.581Z","log.logger":"publisher","log.origin":{"file.name":"pipeline/retry.go","file.line":223},"message":" done","service.name":"apm-server","ecs.version":"1.6.0"} {"log.level":"error","@timestamp":"2024-08-16T11:20:48.582Z","log.logger":"esclientleg","log.origin":{"file.name":"transport/logging.go","file.line":37},"message":"Error dialing dial tcp 127.0.0.1:9200: connect: connection refused","service.name":"apm-server","network":"tcp","address":"localhost:9200","ecs.version":"1.6.0"}

----------kibana.yml ------------
server.name: "kibana"

server.host: "0.0.0.0" # Bind to all network interfaces
elasticsearch.hosts: ["http://127.0.0.1:9200"]

elasticsearch.username: "elastic"

elasticsearch.password: "password"

xpack.security.encryptionKey: "VtG9FgY8JQ8CZW4eZbL2rHw8u9wT1mXy"

xpack.encryptedSavedObjects.encryptionKey: "VtG9FgY8JQ8CZW4eZbL2rHw8u9wT1mXy"

xpack.reporting.encryptionKey : "VtG9FgY8JQ8CZW4eZbL2rHw8u9wT1mXy"

xpack.security.session.idleTimeout: "1h"

xpack.security.session.lifespan: "30d"

elasticsearch.ssl.verificationMode: none

-------------apm-server.yml-----------------
apm-server:
host: "localhost:8200"
kibana:
host: "http://localhost:5601" # Replace with your Kibana URL
username: "kibana_system" # If authentication is required
password: "password" # If authentication is required
rum:
enabled: true

output.elasticsearch:
hosts: ["http://localhost:9200"]
username: "elastic"
password: "password"

---------elasticsearch.yml----------
xpack.security.enabled: true

discovery.type: single-node
network.host: 127.0.0.1
http.port: 9200

thank you in advance.