Hello dear community.
I have an issue that i can't get logs of some pods on our Kubernetes cluster, here is how we collect logs :
filebeat -> logstash -> elasticsearch
filebeat is deployed on each node on the cluster 1.
logstash and elasticsearch are deployed on cluster 2.
here is our config :
filebeat :
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
data:
filebeat.yml: |-
filebeat.config:
inputs:
reload.enabled: true
modules:
path: ${path.config}/modules.d/*.yml
# Reload module configs as they change:
reload.enabled: true
filebeat.autodiscover:
providers:
- type: kubernetes
hints.enabled: true
filebeat.modules:
- module: system
output.logstash:
hosts: ['${LOGSTASH_HOST}:${LOGSTASH_PORT}']
index: "ds-ci-cluster"
setup.template:
name: "ds-ci-cluster"
pattern: "ds-ci-cluster-*"
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-inputs
namespace: kube-system
labels:
k8s-app: filebeat
data:
kubernetes.yml: |-
- type: docker
containers.ids:
- "*"
processors:
- add_kubernetes_metadata:
in_cluster: true
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: filebeat-ds-ci
namespace: kube-system
labels:
k8s-app: filebeat
spec:
template:
metadata:
labels:
k8s-app: filebeat
spec:
tolerations:
- operator: Exists
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: OURIMAGESREGISTRY/filebeat-oss:6.8.1
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: LOGSTASH_HOST
value: "100.107.237.232"
- name: LOGSTASH_PORT
value: "30003"
securityContext:
runAsUser: 0
resources:
limits:
memory: 1500Mi
requests:
cpu: 100m
memory: 100Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: inputs
mountPath: /usr/share/filebeat/inputs.d
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlog
mountPath: /var/log
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlog
hostPath:
path: /var/log
- name: inputs
configMap:
defaultMode: 0600
name: filebeat-inputs
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
---
logstash:
---
apiVersion: v1
kind: ConfigMap
metadata:
namespace: logging
name: logstash-jvm-config
data:
jvm.options: |
## JVM configuration
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms31g
-Xmx31g
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
-XX:+UseParNewGC
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djruby.compile.invokedynamic=true
-Djruby.jit.threshold=0
-XX:+HeapDumpOnOutOfMemoryError
-Djava.security.egd=file:/dev/urandom
---
apiVersion: v1
kind: ConfigMap
metadata:
namespace: logging
name: logstash-conf
data:
filebeat.conf: |-
input {
beats {
port => 5044
}
}
filter {
if [fields][log_type] == "atlassian-confluence" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp:date} %{LOGLEVEL:loglevel} %{GREEDYDATA:logmessage}" }
}
date {
match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "@timestamp"
}
}
if [fields][log_type] == "jenkins" {
grok {
match => {
"source" => "/var/jenkins_home/jobs/%{DATA:job_name}/builds/%{DATA:build_number}/log"
}
}
}
if [fields][type] == "oil-audit" {
grok {
match => {
"message" => "%{DATA:compartment} %{GREEDYDATA}"
}
}
mutate {
gsub => ["message", "(\S+=)", ", \1"]
}
mutate {
gsub => ["message", "(\S+=\S+) (\S+) (\S+) (UTC)", "\1T\2"]
}
kv {
include_keys => ["CompartmentName", "CompartmentId", "TenantId", "EventId", "EventName", "EventType", "EventSource", "RequestAction", "UserName", "EventTime", "ResponseTime", "ResponseStatus", "RequestOrigin", "RequestAction", "RequestAgent","ResponsePayload"]
field_split => " , "
trim_key => " "
trim_value => " "
}
}
if [fields][log_type] == "atlassian-jira" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp:date} %{GREEDYDATA:logprefix} %{LOGLEVEL:loglevel} %{GREEDYDATA:logmessage}" }
}
date {
match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "@timestamp"
}
}
if [fields][log_type] == "atlassian-bitbucket" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:timestamp:date} %{LOGLEVEL:loglevel} %{GREEDYDATA:logmessage}" }
}
date {
match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "@timestamp"
}
}
if [fields][log_type] == "atlassian-bitbucket-access" {
grok {
match => { "message" => "^%{IP:clientip}(,%{IP:proxyip})?(,%{IP:serverip})? \| %{NOTSPACE:protocol} \| %{NOTSPACE:requestid} \| %{NOTSPACE:user} \| %{TIMESTAMP_ISO8601:timestamp} \| \"?%{DATA:action}\"? \| (%{QS:ref_url}|-) (%{QS:agent} )?\| (%{INT:response}|-) \| (%{INT:bytes_read}|-) \| (%{INT:bytes_written}|-) \| (%{DATA:labels}|-) \| (%{INT:response_time}|-) \| (%{NOTSPACE:sessionid}|-) \|" }
}
date {
match => [ "timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "@timestamp"
}
}
}
output {
elasticsearch {
hosts => "elasticsearch.logging:9200"
index => "%{[@metadata][beat]}-%{+YYYY.MM}"
sniffing => false
}
}
http.conf: |-
input {
http { port => 8080 }
}
filter {
grok {
match => ['[headers][request_path]', '/%{DATA:request_path}/']
}
mutate { remove_field => ["headers"] }
}
output {
elasticsearch {
hosts => "elasticsearch.logging:9200"
index => "%{request_path}-%{+YYYY.MM}"
}
stdout { codec => rubydebug {metadata => true} }
}
pipelines.yml: |-
- pipeline.id: filebeat
path.config: /usr/share/logstash/pipeline/filebeat.conf
pipeline.workers: 20
pipeline.batch.size: 2000
- pipeline.id: http
path.config: /usr/share/logstash/pipeline/http.conf
pipeline.workers: 8
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
namespace: logging
name: logstash
spec:
replicas: 1
template:
metadata:
labels:
app: logstash
spec:
containers:
- image: OURIMAGESREGISTRY/logstash-oss:6.6.1
name: logstash
ports:
- name: logstash
containerPort: 5044
protocol: TCP
- name: logstash-http
containerPort: 8080
protocol: TCP
- name: logstash-ep
containerPort: 9600
protocol: TCP
volumeMounts:
- name: pipelines
mountPath: /usr/share/logstash/config/pipelines.yml
subPath: pipelines.yml
readOnly: false
- name: logstash-conf
mountPath: /usr/share/logstash/pipeline
readOnly: false
- name: logstash-jvm-config
mountPath: /usr/share/logstash/config/jvm.options
subPath: jvm.options
volumes:
- name: logstash-jvm-config
configMap:
name: logstash-jvm-config
items:
- key: jvm.options
path: jvm.options
- name: logstash-conf
configMap:
name: logstash-conf
items:
- key: filebeat.conf
path: filebeat.conf
- key: http.conf
path: http.conf
- name: pipelines
configMap:
name: logstash-conf
items:
- key: pipelines.yml
path: pipelines.yml
---
apiVersion: v1
kind: Service
metadata:
namespace: logging
name: logstash
labels:
app: logstash
spec:
type: NodePort
selector:
app: logstash
ports:
- name: logstash
port: 5044
nodePort: 30003
protocol: TCP
- name: logstash-http
port: 8080
nodePort: 30004
protocol: TCP
- name: logstash-ep
port: 9600
nodePort: 30005
protocol: TCP
what's weird is that in same namespace some logs get collected from a pod and not collected from another one in the same namespace.
Sorry for the long post and i appreciate your help guys 
if you need more config/explanations please ping me.
Thanks in advance.