Logstash loki standalone setup with input file form /var/log/ dir

[jijesh_vu]

Im using logstash client with input file plugin to read the kubernetes logs from /var/log/** directory. I have also a log generator is running on a node, and grafana is unable to detect the log generator logs .Grafana is able to identify the datasource but containers is not listing. Below is my entire setup.

---
# Source: logstash/templates/poddisruptionbudget.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: "logstash-logstash-pdb"
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: "logstash-logstash"
---
# Source: logstash/templates/configmap-pipeline.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-logstash-pipeline
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    log.level: debug
    path.config: /usr/share/logstash/pipeline
  logstash.conf: |
    input {
      file {
        id => "varlog"
        path => ["/var/log/**"]
        type => "var log"
        start_position => "beginning"
      }
    }
    filter {
        if [kubernetes] {
          mutate {
            add_field => {
              "container_name" => "%{[kubernetes][container][name]}"
              "namespace" => "%{[kubernetes][namespace]}"
              "pod" => "%{[kubernetes][pod][name]}"
            }
            replace => { "host" => "%{[kubernetes][node][name]}"}
          }
        }
        mutate {
          remove_field => ["tags"]
        }
     }
    
       output {
         stdout { codec => rubydebug}
         loki {
           url => "http://loki-loki-distributed-distributor.loki-benchmark.svc.cluster.local:3100/loki/api/v1/push"
         }
       }
---
# Source: logstash/templates/service-headless.yaml
kind: Service
apiVersion: v1
metadata:
  name: "logstash-logstash-headless"
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
spec:
  clusterIP: None
  selector:
    app: "logstash-logstash"
  ports:
    - name: http
      port: 9600
---
# Source: logstash/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: logstash-logstash
  labels:
    app: "logstash-logstash"
    chart: "logstash"
    heritage: "Helm"
    release: "logstash"
spec:
  serviceName: logstash-logstash-headless
  selector:
    matchLabels:
      app: "logstash-logstash"
      release: "logstash"
  replicas: 1
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: "logstash-logstash"
      labels:
        app: "logstash-logstash"
        chart: "logstash"
        heritage: "Helm"
        release: "logstash"
      annotations:
        pipelinechecksum: e5576a55d691ae22c1da1204f1e548e8aa936dc6415af52eb65699f5a155bb8
    spec:
      securityContext:
        fsGroup: 1000
        runAsUser: 1000
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - "logstash-logstash"
            topologyKey: kubernetes.io/hostname
      terminationGracePeriodSeconds: 120
      volumes:
        - name: logstashpipeline
          configMap:
            name: logstash-logstash-pipeline
        - name: varlog
          hostPath:
            path: /var/log
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers    
      containers:
      - name: "logstash"
        securityContext:
          capabilities:
            drop:
            - ALL
          runAsNonRoot: true
          runAsUser: 1000
        image: "grafana/logstash-output-loki:1.0.1"
        imagePullPolicy: "IfNotPresent"
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: http
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: http
          initialDelaySeconds: 60
          periodSeconds: 10
          successThreshold: 3
          timeoutSeconds: 5
        ports:
        - name: http
          containerPort: 9600
        resources:
          limits:
            cpu: 1000m
            memory: 1536Mi
          requests:
            cpu: 100m
            memory: 1536Mi
        env:
          - name: LS_JAVA_OPTS
            value: "-Xmx1g -Xms1g"
          - name: XPACK_MONITORING_ENABLED
            value: "false"
        volumeMounts:
          - name: logstashpipeline
            mountPath: /usr/share/logstash/pipeline/logstash.conf
            subPath: logstash.conf
          - name: varlog
            mountPath: /var/log
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
            readOnly: true