Filebeat is failing to connect to Elasticsearch only in dev, despite having the same config as prod. The ping to the ES host returns a 200, but Filebeat is then attempting to connect to an x-pack endpoint, even though AWS ES does not have x-pack. I tried disabling monitoring and it continues to try to connect to x-pack despite that the config is the same as prod (other than the index name), which does not try to reach the x-pack endpoint.
Here is the config:
filebeat.registry_flush: 10s
filebeat.inputs:
- type: docker
exclude_lines: '.*health.*|.*HealthChecker.*'
multiline:
pattern: '^(\[20|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])'
negate: true
match: after
ignore_older: 48h
clean_inactive: 72h
scan_frequency: 1m
combine_partial: true
processors:
- add_docker_metadata: ~
containers:
path: "/var/lib/docker/containers"
stream: "all"
ids:
- "*"
enabled: true
- type: log
multiline:
pattern: '^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+([0-3][0-9]|[1-9])'
negate: true
match: after
ignore_older: 48h
clean_inactive: 72h
scan_frequency: 1m
combine_partial: true
paths:
- /var/log/secure
enabled: true
#==================== Elasticsearch template setting ==========================
setup.template:
name: 'filebeat'
pattern: 'filebeat-dev-*'
fields: 'fields.yml'
setup.template.settings:
index.number_of_shards: 5
index.codec: best_compression
index.mapper.dynamic: false
_source:
excludes: [
"beat.version",
"docker.container.labels.com.docker.compose.config-hash",
"docker.container.labels.com.docker.compose.container-number",
"docker.container.labels.com.docker.compose.oneoff",
"docker.container.labels.com.docker.compose.version",
"docker.container.labels.license",
"docker.container.labels.maintainer",
"docker.container.labels.site",
"docker.container.labels.vendor",
"docker.container.labels.org.label-schema.schema-version",
"docker.container.labels.org.label-schema.url",
"docker.container.labels.org.label-schema.vcs-url",
"docker.container.labels.org.label-schema.vendor",
"docker.container.labels.org.label-schema.version",
"docker.container.labels.io.confluent.docker.build.number",
"docker.container.labels.io.confluent.docker.git.id",
"docker.container.labels.io.confluent.docker.value",
"docker.container.labels.io.k8s.description",
"docker.container.labels.io.k8s.display-name",
"docker.container.labels.io.openshift.expose-services",
"docker.container.labels.io.openshift.s2i.assemble-input-files",
"docker.container.labels.io.openshift.s2i.build.commit.author",
"docker.container.labels.io.openshift.s2i.build.commit.date",
"docker.container.labels.io.openshift.s2i.build.commit.id",
"docker.container.labels.io.openshift.s2i.build.commit.message",
"docker.container.labels.io.openshift.s2i.build.commit.ref",
"docker.container.labels.io.openshift.s2i.build.image",
"docker.container.labels.io.openshift.s2i.build.source-context-dir",
"docker.container.labels.io.openshift.s2i.build.source-location",
"docker.container.labels.io.openshift.s2i.scripts-url",
"docker.container.labels.io.openshift.tags",
"docker.container.labels.license",
"docker.container.labels.maintainer",
"docker.container.labels.name",
"docker.container.labels.org.label-schema.build-date",
"docker.container.labels.org.label-schema.license",
"docker.container.labels.org.label-schema.name",
"docker.container.labels.org.label-schema.schema-version",
"docker.container.labels.org.label-schema.url",
"docker.container.labels.org.label-schema.vcs-url",
"docker.container.labels.org.label-schema.vendor",
"docker.container.labels.org.label-schema.version",
"docker.container.labels.quay.expires-after",
"docker.container.labels.vendor",
"host.architecture",
"host.containerized",
"host.id",
"host.os.codename",
"host.os.family",
"host.os.platform",
"host.os.version",
"host.os.name",
"offset",
"log.flags",
"log.file.path"
]
#================================ Outputs =====================================
#setup.dashboards.enabled: true
setup.kibana.host: "https://elasticsearch_host_redacted.us-east-1.es.amazonaws.com:443/_plugin/kibana"
xpack.monitoring.enabled: false
output.elasticsearch:
{% if filebeat_output_elasticsearch_enabled %}
### Elasticsearch as output
# Array of hosts to connect to.
hosts: ["https://elasticsearch_host_redacted.us-east-1.es.amazonaws.com:443"]
# Number of workers per Elasticsearch host.
#worker: 1
# Optional index name. The default is "filebeat" and generates
#[filebeat-]YYYY.MM.DD keys.
index: "filebeat-dev-%{+yyyy.MM.dd}"
{% endif %}
#================================ Logging =====================================
{% if filebeat_enable_logging %}
logging.level: {{ filebeat_log_level }}
# Enable file rotation with default configuration
logging.files:
path: {{ filebeat_log_dir }}
name: {{ filebeat_log_filename }}
keepfiles: 7
permissions: 0644
{% endif %}