Kibana version: 7.10.2
Elasticsearch version: 7.10.2
APM Server version: 7.10.2
APM Agent language and version: node.js 3.10.0
Original install method (e.g. download page, yum, deb, from source, etc.) and version: APT repository
Fresh install or upgraded from other version? Fresh Install
Is there anything special in your setup? 1 Load balancer (haproxy) in front of apm-server
Description of the problem including expected versus actual behavior. Please include screenshots (if relevant): node.js client is constantly receiving a ResponseError from apm-server. The communication is encrypted and the authentication is made with api keys
Steps to reproduce:
- fresh install of apm-server
- generation of api key from cli:
apm-server apikey create --ingest --agent-config --sourcemap --name - configuration of node.js elastic agent
Provide logs and/or server output (if relevant):
This is how I configured APM-server:
apm-server:
host: "********************:8200"
ssl:
enabled: true
certificate: '/etc/apm-server/ssl/********************.crt'
key: '/etc/apm-server/ssl/********************.key'
secret_token: ***************************
api_key:
enabled: true
elasticsearch:
hosts: ["********************:9200"]
protocol: "https"
username: "********************"
password: "********************"
ssl.enabled: true
ssl.verification_mode: none
ssl.certificate_authorities: ["/etc/apm-server/elasticsearch-ca.pem"]
rum:
enabled: true
event_rate:
limit: 600
lru_size: 1000
allow_origins : ['*']
kibana:
enabled: true
host: "********************:5601"
protocol: "https"
username: "********************"
password: "********************"
ssl.verification_mode: none
ssl.certificate_authorities: ["/etc/apm-server/elasticsearch-ca.pem"]
setup.template.enabled: true
setup.template.settings:
index:
number_of_shards: 1
number_of_replicas: 0
output.elasticsearch:
hosts: ["********************:9200"]
protocol: "https"
username: "********************"
password: "********************"
ssl.verification_mode: none
ssl.certificate_authorities: ["/etc/apm-server/elasticsearch-ca.pem"]
logging.level: debug
logging.to_files: true
logging.files:
path: /var/log/apm-server
name: apm-server
keepfiles: 7
elastic -> elasticsearch -> Transport.js -> onBody -> console.log
{
method: 'HEAD',
path: '/_template/template_logs',
body: null,
querystring: '',
headers: {
'user-agent': 'elasticsearch-js/7.10.0 (darwin 20.3.0-x64; Node.js v14.13.0)'
},
timeout: 20000
}
elastic -> elasticsearch -> Transport.js -> onBody -> console.log
{
body: true,
statusCode: 403,
headers: {
'content-type': 'application/json; charset=UTF-8',
'content-length': '281'
},
meta: {
context: null,
request: { params: [Object], options: {}, id: 4 },
name: 'elasticsearch-js',
connection: {
url: 'https://elastic01***********:9200/',
id: 'https://elastic01************:9200/',
headers: {},
deadCount: 0,
resurrectTimeout: 0,
_openRequests: 0,
status: 'alive',
roles: [Object]
},
attempts: 0,
aborted: false
}
}
elastic-apm-node -> lib -> agent.js -> send -> console.log(error)
{
exception: {
message: 'Response Error',
type: 'ResponseError',
module: '@elastic/elasticsearch',
stacktrace: [ [Object], [Object], [Object], [Object], [Object] ],
handled: undefined
},
culprit: 'onBody (node_modules/@elastic/elasticsearch/lib/Transport.js)',
id: '5db70a535a386390f0950cb8d0a2bdce',
parent_id: undefined,
trace_id: undefined,
timestamp: 1612965003106000,
context: { user: {}, tags: {}, custom: {} }
}
[
{
filename: 'node_modules/@elastic/elasticsearch/lib/Transport.js',
lineno: 324,
function: 'onBody',
library_frame: true,
abs_path: '/Users/************/Documents/Repositories/process-api-orders/node_modules/@elastic/elasticsearch/lib/Transport.js',
pre_context: [ '', '' ],
context_line: ' const error = new ResponseError(result)',
post_context: [
" this.emit('response', error, result)",
' callback(error, result)'
]
},
{
filename: 'node_modules/@elastic/elasticsearch/lib/Transport.js',
lineno: 240,
function: 'onEnd',
library_frame: true,
abs_path: '/Users/************/Documents/Repositories/process-api-orders/node_modules/@elastic/elasticsearch/lib/Transport.js',
pre_context: [
' unzip(Buffer.concat(payload), onBody)',
' } else {'
],
context_line: ' onBody(null, payload)',
post_context: [ ' }', ' }' ]
},
{
filename: 'events.js',
lineno: 326,
function: 'emit',
library_frame: true,
abs_path: 'events.js'
},
{
filename: '_stream_readable.js',
lineno: 1252,
function: 'endReadableNT',
library_frame: true,
abs_path: '_stream_readable.js'
},
{
filename: 'internal/process/task_queues.js',
lineno: 80,
function: 'processTicksAndRejections',
library_frame: true,
abs_path: 'internal/process/task_queues.js'
}
]
The apikey is created with elastic user (superuser), I've modified the source to see where i'm receiving the Forbidden error.
It seems something related to /_template/template_logs:
curl -k -i --head -H "Authorization: ApiKey ****************************" https://elastic01*****************:9200/_template/template_logs
HTTP/1.1 403 Forbidden
content-type: application/json; charset=UTF-8
content-length: 335