Processor working on global level but not on individual

Elastic Stack Beats
,
1

Hi, I am trying to make the processor work on the individual input level, but it looks like it works only on condition that it's defined globally.

Not working

#https://github.com/kdryetyln/ELK-with-ECK-Operator/blob/main/filesformedium/filebeat.yaml
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: ${stack_name}
  namespace: ${namespace}
spec:
  type: filebeat
  version: ${version}
  elasticsearchRef:
    name: ${stack_name}
  kibanaRef:
    name: ${stack_name}
  config:
    output.elasticsearch.enabled: false
    output.logstash:
      hosts: ["${stack_name}-ls-beats:5044"]
#      index: filebeat
    filebeat.autodiscover:
        providers:
          - type: kubernetes
            node: $${NODE_NAME}
            hints:
              enabled: true
              default_config:
                type: container
                paths:
                  - /var/log/containers/*$${data.kubernetes.container.id}.log
    filebeat.inputs:
      - id: containers
        type: container
        paths:
          - /var/log/containers/*.log
        exclude_lines: []
        processors:
          - add_kubernetes_metadata:
              labels.dedot: true
              annotations.dedot: true
              in_cluster: true
              deployment: true
              host: $${NODE_NAME}
    #          matchers:
    #            - logs_path:
    #                logs_path: "/var/log/containers/"
          - drop_event.when.or:
            - equals.kubernetes.namespace: "logging"
            - equals.kubernetes.namespace: "nginx"
            - equals.kubernetes.namespace: "argocd"
          - drop_fields:
              fields: [
                "container.id",
                "container.runtime",
                "input.type",
    #            "kubernetes.container.image",
                "kubernetes.labels.pod-template-hash",
                "kubernetes.namespace_labels.kubernetes_io/metadata_name",
                "kubernetes.namespace_uid",
    #            "kubernetes.node.hostname",
                "kubernetes.node.labels.arch",
                "kubernetes.node.labels.beta_kubernetes_io/arch",
                "kubernetes.node.labels.beta_kubernetes_io/instance-type",
                "kubernetes.node.labels.beta_kubernetes_io/os",
                "kubernetes.node.labels.eks_amazonaws_com/capacityType",
                "kubernetes.node.labels.eks_amazonaws_com/nodegroup",
                "kubernetes.node.labels.eks_amazonaws_com/nodegroup-image",
                "kubernetes.node.labels.eks_amazonaws_com/sourceLaunchTemplateId",
                "kubernetes.node.labels.eks_amazonaws_com/sourceLaunchTemplateVersion",
                "kubernetes.node.labels.failure-domain_beta_kubernetes_io/region",
                "kubernetes.node.labels.failure-domain_beta_kubernetes_io/zone",
                "kubernetes.node.labels.instanceBillingType",
                "kubernetes.node.labels.k8s_io/cloud-provider-aws",
                "kubernetes.node.labels.kubernetes_io/arch",
                "kubernetes.node.labels.kubernetes_io/hostname",
                "kubernetes.node.labels.kubernetes_io/os",
                "kubernetes.node.labels.node_kubernetes_io/instance-type",
                "kubernetes.node.labels.topology_ebs_csi_aws_com/zone",
                "kubernetes.node.labels.topology_kubernetes_io/zone",
                "kubernetes.node.labels.usageType",
                "kubernetes.node.name",
                "kubernetes.node.uid",
                "kubernetes.pod.ip",
                "kubernetes.pod.uid",
                "kubernetes.replicaset.name",
                "kubernetes.labels.app_kubernetes_io/instance",
                "kubernetes.labels.app_kubernetes_io/managed-by",
                "kubernetes.labels.app_kubernetes_io/name",
                "kubernetes.labels.app_kubernetes_io/part-of",
                "log.file.path",
                "log.offset",
                "agent.id",
                "agent.ephemeral_id",
                "agent.hostname",
                "agent.type",
                "agent.version",
                "ecs.version",
                "host.architecture",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-model_vendor_id",
                "kubernetes.node.labels.feature_node_kubernetes_io/system-os_release_VERSION_ID",
                "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_revision",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512F",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XGETBV1",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512BW",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_FMA3",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_HYPERVISOR",
                "kubernetes.node.labels.feature_node_kubernetes_io/kernel-config_NO_HZ_IDLE",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-model_family",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_MPX",
                "kubernetes.node.labels.feature_node_kubernetes_io/pci-1d0f_present",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX",
                "kubernetes.node.labels.feature_node_kubernetes_io/system-os_release_VERSION_ID_major",
                "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_full",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-hardware_multithreading",
                "kubernetes.node.labels.feature_node_kubernetes_io/system-os_release_ID",
                "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_major",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_LAHF",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_FXSROPT",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_FXSR",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AESNI",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_MOVBE",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512DQ",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-model_id",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_SYSEE",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XSAVE",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512CD",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XSAVES",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_X87",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_OSXSAVE",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX2",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_CMPXCHG8",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512VL",
                "kubernetes.node.labels.feature_node_kubernetes_io/kernel-config_NO_HZ",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_ADX",
                "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_minor",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_SYSCALL",
                "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XSAVEC",

              ]
              ignore_missing: true
    #      - decode_json_fields:
    #          fields: [ "message" ]
    #          process_array: false
    #          max_depth: 1
    #          expand_keys: false
    #          #target: ""
    #          overwrite_keys: true
    #          add_error_key: true
  daemonSet:
    podTemplate:
      spec:
        dnsPolicy: ClusterFirstWithHostNet
        serviceAccountName: filebeat
        automountServiceAccountToken: true
        terminationGracePeriodSeconds: 10
        hostNetwork: true
        tolerations:
          - key: dedicated
            operator: Exists
            effect: NoSchedule
        priorityClassName: system-node-critical
        securityContext:
          runAsUser: 0
        containers:
        - name: filebeat
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          resources:
            limits:
              cpu: 500m
              memory: 2000Mi
            requests:
              cpu: 100m
              memory: 200Mi
          volumeMounts:
          - name: varlogcontainers
            mountPath: /var/log/containers
          - name: varlogpods
            mountPath: /var/log/pods
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
        volumes:
        - name: varlogcontainers
          hostPath:
            path: /var/log/containers
        - name: varlogpods
          hostPath:
            path: /var/log/pods
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers

Working

#https://github.com/kdryetyln/ELK-with-ECK-Operator/blob/main/filesformedium/filebeat.yaml
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: ${stack_name}
  namespace: ${namespace}
spec:
  type: filebeat
  version: ${version}
  elasticsearchRef:
    name: ${stack_name}
  kibanaRef:
    name: ${stack_name}
  config:
    output.elasticsearch.enabled: false
    output.logstash:
      hosts: ["${stack_name}-ls-beats:5044"]
#      index: filebeat
    filebeat.autodiscover:
        providers:
          - type: kubernetes
            node: $${NODE_NAME}
            hints:
              enabled: true
              default_config:
                type: container
                paths:
                  - /var/log/containers/*$${data.kubernetes.container.id}.log
    filebeat.inputs:
      - id: containers
        type: container
        paths:
          - /var/log/containers/*.log
        exclude_lines: []
    processors:
      - add_kubernetes_metadata:
          labels.dedot: true
          annotations.dedot: true
          in_cluster: true
          deployment: true
          host: $${NODE_NAME}
#          matchers:
#            - logs_path:
#                logs_path: "/var/log/containers/"
      - drop_event.when.or:
        - equals.kubernetes.namespace: "logging"
        - equals.kubernetes.namespace: "nginx"
        - equals.kubernetes.namespace: "argocd"
      - drop_fields:
          fields: [
            "container.id",
            "container.runtime",
            "input.type",
#            "kubernetes.container.image",
            "kubernetes.labels.pod-template-hash",
            "kubernetes.namespace_labels.kubernetes_io/metadata_name",
            "kubernetes.namespace_uid",
#            "kubernetes.node.hostname",
            "kubernetes.node.labels.arch",
            "kubernetes.node.labels.beta_kubernetes_io/arch",
            "kubernetes.node.labels.beta_kubernetes_io/instance-type",
            "kubernetes.node.labels.beta_kubernetes_io/os",
            "kubernetes.node.labels.eks_amazonaws_com/capacityType",
            "kubernetes.node.labels.eks_amazonaws_com/nodegroup",
            "kubernetes.node.labels.eks_amazonaws_com/nodegroup-image",
            "kubernetes.node.labels.eks_amazonaws_com/sourceLaunchTemplateId",
            "kubernetes.node.labels.eks_amazonaws_com/sourceLaunchTemplateVersion",
            "kubernetes.node.labels.failure-domain_beta_kubernetes_io/region",
            "kubernetes.node.labels.failure-domain_beta_kubernetes_io/zone",
            "kubernetes.node.labels.instanceBillingType",
            "kubernetes.node.labels.k8s_io/cloud-provider-aws",
            "kubernetes.node.labels.kubernetes_io/arch",
            "kubernetes.node.labels.kubernetes_io/hostname",
            "kubernetes.node.labels.kubernetes_io/os",
            "kubernetes.node.labels.node_kubernetes_io/instance-type",
            "kubernetes.node.labels.topology_ebs_csi_aws_com/zone",
            "kubernetes.node.labels.topology_kubernetes_io/zone",
            "kubernetes.node.labels.usageType",
            "kubernetes.node.name",
            "kubernetes.node.uid",
            "kubernetes.pod.ip",
            "kubernetes.pod.uid",
            "kubernetes.replicaset.name",
            "kubernetes.labels.app_kubernetes_io/instance",
            "kubernetes.labels.app_kubernetes_io/managed-by",
            "kubernetes.labels.app_kubernetes_io/name",
            "kubernetes.labels.app_kubernetes_io/part-of",
            "log.file.path",
            "log.offset",
            "agent.id",
            "agent.ephemeral_id",
            "agent.hostname",
            "agent.type",
            "agent.version",
            "ecs.version",
            "host.architecture",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-model_vendor_id",
            "kubernetes.node.labels.feature_node_kubernetes_io/system-os_release_VERSION_ID",
            "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_revision",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512F",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XGETBV1",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512BW",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_FMA3",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_HYPERVISOR",
            "kubernetes.node.labels.feature_node_kubernetes_io/kernel-config_NO_HZ_IDLE",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-model_family",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_MPX",
            "kubernetes.node.labels.feature_node_kubernetes_io/pci-1d0f_present",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX",
            "kubernetes.node.labels.feature_node_kubernetes_io/system-os_release_VERSION_ID_major",
            "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_full",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-hardware_multithreading",
            "kubernetes.node.labels.feature_node_kubernetes_io/system-os_release_ID",
            "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_major",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_LAHF",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_FXSROPT",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_FXSR",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AESNI",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_MOVBE",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512DQ",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-model_id",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_SYSEE",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XSAVE",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512CD",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XSAVES",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_X87",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_OSXSAVE",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX2",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_CMPXCHG8",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_AVX512VL",
            "kubernetes.node.labels.feature_node_kubernetes_io/kernel-config_NO_HZ",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_ADX",
            "kubernetes.node.labels.feature_node_kubernetes_io/kernel-version_minor",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_SYSCALL",
            "kubernetes.node.labels.feature_node_kubernetes_io/cpu-cpuid_XSAVEC",

          ]
          ignore_missing: true
    #      - decode_json_fields:
    #          fields: [ "message" ]
    #          process_array: false
    #          max_depth: 1
    #          expand_keys: false
    #          #target: ""
    #          overwrite_keys: true
    #          add_error_key: true
  daemonSet:
    podTemplate:
      spec:
        dnsPolicy: ClusterFirstWithHostNet
        serviceAccountName: filebeat
        automountServiceAccountToken: true
        terminationGracePeriodSeconds: 10
        hostNetwork: true
        tolerations:
          - key: dedicated
            operator: Exists
            effect: NoSchedule
        priorityClassName: system-node-critical
        securityContext:
          runAsUser: 0
        containers:
        - name: filebeat
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          resources:
            limits:
              cpu: 500m
              memory: 2000Mi
            requests:
              cpu: 100m
              memory: 200Mi
          volumeMounts:
          - name: varlogcontainers
            mountPath: /var/log/containers
          - name: varlogpods
            mountPath: /var/log/pods
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
        volumes:
        - name: varlogcontainers
          hostPath:
            path: /var/log/containers
        - name: varlogpods
          hostPath:
            path: /var/log/pods
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers

Am I missing something?

2

nvm, it looks like processing broke completely on unrelated issue. Closing this one off.