Index Pattern

[Suleman_Ahmed]

hello! I am running ELK on my kubernetes cluster and unable to create index pattern. Any help will be highly appreciated. My logstash yaml is as under:-

apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-config
  namespace: kube-system
  labels:
    app: logstash
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    path.config: /usr/share/logstash/pipeline
  pipeline.conf: |
    input {
      beats {
        port => 5044
        type => "kube-logs"
      }
    }

    filter {
      grok {
        match => {
          "source" => "%{GREEDYDATA}/%{GREEDYDATA:app}-%{DATA}-%{DATA}_%{DATA:namespace}_%{GREEDYDATA}"
        }
        add_tag => ["app-extracted"]
      }

      if [app] == "nginx-ingress-controller" {
        grok {
          match => {
            "log" => "%{IP:real_ip} %{DATA:http_host} %{DATA:proxy_protocol_ip} %{DATA:remote_addr} - \\[%{DATA:forwarded_for}\\] - %{DATA:remote_user} \\[%{DATA:ingress_time}\\] \"%{WORD:verb} %{NOTSPACE:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:result} %{NUMBER:bytes} \"%{DATA:referer}\" \"%{DATA:agent}\" %{DATA:request_length} %{DATA:request_time} \\[%{DATA:upstream}\\] %{DATA:upstream_addr} %{NUMBER:upstream_length} %{NUMBER:upstream_time} %{NUMBER:upstream_result}%{SPACE}%{WORD:request_id}"
          }
          add_tag => ["ingress-access-log"]
        }

        if "ingress-access-log" in [tags] {
          mutate {
            replace => { "type" => "ingress-access" }
          }
        }
      }
    }

    output {
      elasticsearch {
        ilm_enabled => false
        hosts => ["localhost:9200"]
        user => 'elastic'
        password => 't@321'
        index => "logstash-beta-%{+YYYY.MM.dd}"
      }
      if "coreapiaccount-crm" in [message] {
        elasticsearch {
          ilm_enabled => false
          hosts => ["localhost:9200"]
          user => 'elastic'
          password => 't@321'
          index => "coreapiaccount-%{+YYYY.MM.dd}"
        }
      }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: logging-logstash
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: logstash 
  template:
    metadata:
      labels:
        app: logstash
    spec:
      containers:
      - name: logstash
        image: harbor.abc.com/kubernetes/logstash:8.10.2
        imagePullPolicy: Always
        volumeMounts:
        - mountPath: /config
          name: config
        - mountPath: /pipeline
          name: pipeline
      volumes:
      - name: config
        configMap:
          name: logstash-config
          items:
          - key: logstash.yml
            path: logstash.yml
      - name: pipeline
        configMap:
          name: logstash-config
          items:
          - key: pipeline.conf
            path: pipeline.conf
      securityContext:
        fsGroup: 101
---
apiVersion: v1
kind: Service
metadata:
  labels:
    kubernetes.io/name: logstash
  name: logstash-service
  namespace: kube-system
spec:
  ports:
  - protocol: TCP
    port: 5044
    targetPort: 5044
  selector:
    app: logstash

[sholzhauer]

Anything in your logstash logs?

Usually this has to do with permissions or a wrongly configured output (not datastream compatible when writing to datastreams).

The logs should help.

[Suleman_Ahmed]

Can see current logs, but cant see index pattern in kibanaafter deploying above yaml. Filebeat is already sending logs to logstash service.