Hi Team,
I am planning to send custom path logs(example: /var/log/nginx/*logs) with cloud and kubernetes metadata.
sample App deployed:
kind: Deployment
metadata:
name: nginx-deployment-test
namespace: vasu
labels:
app: nginx
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.14.2
ports:
- containerPort: 80
volumeMounts:
- name: shared-logs
mountPath: /var/log/nginx
volumes:
- name: shared-logs
hostPath:
path: /var/log/nginx
---
apiVersion: v1
kind: Service
metadata:
name: my-nginx-service
namespace: vasu
spec:
selector:
app: nginx
ports:
- protocol: TCP
port: 8081
targetPort: 80```
`Filebeat complete config :`
```apiVersion: v1
kind: ServiceAccount
metadata:
name: teststack-shipper
namespace: vasu
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: teststack-shipper
rules:
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: teststack-shipper
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: teststack-shipper
subjects:
- kind: ServiceAccount
name: teststack-shipper
namespace: vasu
---
apiVersion: v1
data:
teststack-config.yml: |
#=========================== Filebeat prospectors =============================
fields_under_root: false
fields:
product: ${PRODUCT}
env: ${ENVIRONMENT}
group: ${GROUP}
region: ${AWS_REGION}
cluster: ${K8S_CLUSTER_NAME}
node_name: ${NODE_NAME}
max_procs: 1
filebeat.inputs:
- type: log
tail_files: true
fields_under_root: false
enabled: true
encoding: utf-8
max_bytes: 65536
fields:
name: nginx
paths:
- /var/log/nginx/*.log
processors:
- add_cloud_metadata: ~
#============================= Filebeat modules ===============================
path.data: /teststack-shipper/data
path.logs: /teststack-shipper/logs
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: false
# Period on which files under path should be checked for changes
#reload.period: 10s
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
output.http:
hosts: ${TESTSTACK_SHIPPER_HOST}
compression_level: ${TESTSTACK_SHIPPER_COMPRESSION_LEVEL}
worker: ${TESTSTACK_SHIPPER_WORKER}
batch_size: ${TESTSTACK_SHIPPER_BATCH_SIZE}
auth_token: ${TESTSTACK_SHIPPER_AUTH_TOKEN}
batch_publish: true
#output.logstash:
# hosts: ["10.99.111.60:5044"]
#worker: 4
# output.file:
# path: "/tmp/filebeat"
# filename: filebeat
#================================ Logging =====================================
# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
# logging.level: debug
# At debug level, you can selectively enable logging only for some components.
# To enable all selectors use ["*"]. Examples of other selectors are "beat",
# "publish", "service".
# logging.selectors: ["http"]
kind: ConfigMap
metadata:
name: teststack-shipper-config-node-b6566ggm6g
namespace: vasu
---
apiVersion: v1
data:
AWS_INFRA_SECRETS: stagfpteststack
AWS_REGION: us-east-1
ENVIRONMENT: staging
GROUP: k8s
TESTSTACK_SHIPPER_BATCH_SIZE: "40"
TESTSTACK_SHIPPER_COMPRESSION_LEVEL: "0"
TESTSTACK_SHIPPER_HOST: http://vpce-09ab.aws.com/staging-fp/bulk
TESTSTACK_SHIPPER_WORKER: "8"
PRODUCT: staging-fcp
kind: ConfigMap
metadata:
name: teststack-shipper-env
namespace: vasu
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
k8s-app: teststack-shipper
name: teststack-shipper
namespace: vasu
spec:
selector:
matchLabels:
name: teststack-shipper
template:
metadata:
labels:
name: teststack-shipper
spec:
containers:
- env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
envFrom:
- configMapRef:
name: teststack-shipper-env
image: 2134.dkr.ecr.us-east-1.amazonaws.com/teststack:6.7.2
imagePullPolicy: Always
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- sleep 60
name: teststack-shipper
resources:
limits:
memory: 2000Mi
requests:
cpu: 100m
ephemeral-storage: 10Gi
memory: 500Mi
securityContext:
runAsUser: 0
volumeMounts:
- mountPath: /var/log
name: varlog
readOnly: true
- mountPath: /var/lib/docker
name: varlibdocker
readOnly: true
- mountPath: /etc/teststack-shipper
name: teststack-shipper-config-node
hostNetwork: false
hostPID: false
priorityClassName: high-priority
serviceAccountName: teststack-shipper
terminationGracePeriodSeconds: 120
volumes:
- hostPath:
path: /var/log
name: varlog
- hostPath:
path: /var/lib/docker
name: varlibdocker
- configMap:
name: teststack-shipper-config-node-b6566ggm6g
name: teststack-shipper-config-node
updateStrategy:
rollingUpdate:
maxUnavailable: 10%
type: RollingUpdate ```
`Able to send add_cloud_metadata value , but unable to send add_kubernetes_metadata.`
`Please help on this.`