- We are using elasticsearch for centralized logging in our application.
- There are many components in our application which generate logs.
- All components running in kubernetes cluster. we are using filebeat as deamonset container on every node in the cluster.
- There is an nginx ingress controller which sit in front of all service running in our cluster. nginx access logs with all othe components logs are shipped by filebeat into elasticsearch as a string.
- I want to transform nginx access logs using filebeat. I tried many examples but none of them works as expected for me. Please help us to configure this setup.
Sample for existing document in eleasticsearch:
{
"_source": {
"@timestamp": "2018-08-22T08:55:18.697Z",
"message": "203.88.135.122 - [203.88.135.122] - - [22/Aug/2018:08:55:18 +0000] "GET /blue/rest/organizations/jenkins/pipelines/Backend/branches/customization-icaseboard-dev/runs/7/nodes/20/steps/25/log/?start=25682 HTTP/1.1" 200 0 "https://jenkins.orderhive.plus/blue/organizations/jenkins/Backend/detail/customization-icaseboard-dev/7/pipeline" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" 944 0.003 [default-jenkins-80] 100.96.7.27:8080 0 0.004 200 f5ad0866dcd0f057939be292dfe6a7ac",
}
Sample for expexted document :
{
"_source": {
"@timestamp": "2018-06-01T15:25:47.000Z",
"source": "/var/log/nginx/pawel-blog-access-1.log",
"input_type": "log",
"os_name": "Ubuntu",
"request": "/profitable-slack-bot-rails",
"agent": ""Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0"",
"message": "157.234.132.47 - - [01/Jun/2018:17:25:47 +0200] "GET /profitable-slack-bot-rails HTTP/1.1" 200 12104 "https://news.ycombinator.com/" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0"",
"device": "Other",
"clientip": "157.234.132.47",
"response": 200,
"type": "log",
"httpversion": "1.1",
"host": "pawel-blog",
"referrer": ""https://news.ycombinator.com/"",
"build": "",
"tags": [
"beats_input_codec_plain_applied",
"nginx-geoip"
],
"os": "Ubuntu",
"@version": "1",
"offset": 571753,
"geoip": {
"latitude": -26.2309,
"longitude": 28.0583,
"country_code3": "ZA",
"timezone": "Africa/Johannesburg",
"region_name": "Gauteng",
"ip": "157.234.132.47",
"postal_code": "2000",
"continent_code": "AF",
"city_name": "Johannesburg",
"country_name": "South Africa",
"region_code": "GT",
"country_code2": "ZA",
"location": {
"lon": 28.0583,
"lat": -26.2309
}
},
}
filebeat.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
data:
filebeat.yml: |-
filebeat.config:
prospectors:
# Mounted filebeat-prospectors configmap:
path: ${path.config}/prospectors.d/.yml
# Reload prospectors configs as they change:
reload.enabled: false
modules:
path: ${path.config}/modules.d/.yml
# Reload module configs as they change:
reload.enabled: false
filebeat.modules:
- module: nginx
filebeat.autodiscover:
providers:
- type: docker
condition:
contains:
docker.container.image: "nginx-ingress-controller"
config:
- module: nginx
access:
prospector:
type: docker
containers.stream: stdout
containers.ids:
- "${data.docker.container.id}"
processors:
- add_kubernetes_metadata:
in_cluster: true
error:
prospector:
type: docker
containers.stream: stderr
containers.ids:
- "${data.docker.container.id}"
processors:
- add_kubernetes_metadata:
in_cluster: true
cloud.id: ${ELASTIC_CLOUD_ID}
cloud.auth: ${ELASTIC_CLOUD_AUTH}
output.elasticsearch:
hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}']
username: ${ELASTICSEARCH_USERNAME}
password: ${ELASTICSEARCH_PASSWORD}
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-prospectors
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
data:
kubernetes.yml: |-
- type: docker
json.message_key: log
json.keys_under_root: true
containers.ids:
- ""
processors:
- add_kubernetes_metadata:
in_cluster: true
- drop_event:
when:
or:
- regexp:
logger_name: 'org.apache.curator.'
- equals:
kubernetes.container.name: filebeat
- add_cloud_metadata: