Filebeats processor active not right under specific input

Elastic Stack Beats
1

In the official doc, it says:
Processors are valid:

  • At the top-level in the configuration. The processor is applied to all data collected by Filebeat.
  • Under a specific input. The processor is applied to the data collected for that input.
    but I found it has differents action between processors at top-level from under specific input:

below are my config top-level :(I'm aware that metadata and timestamp cannot be dropped)

filebeat.yml
filebeat.config.inputs:
  enabled: true
  path: conf/*.yml
  reload.enabled: true
  reload.period: 10s
output.console:
  enabled: true
  codec.json:
    pretty: true
logging.level: debug

processors:
    - copy_fields:
        fields:
          - from: host.name
            to: srv_host
          - from: '@timestamp'
            to: logUploadKafkaTime
          - from: dur
            to: mydur
        fail_on_error: false
        ignore_missing: true
    - drop_fields:
        fields: ["agent","@timestamp","host","log","@metadata","ecs"]
        

conf/data.yml
- type: log
  paths:
    - ./data.log
  fields_under_root: true
  fields:
    '@log_type': data
    net_pub: 1
  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: false

input:

{
    "srv_id": "cdn-kes-ln-shenyang-ctum-edge274-db-txyun_8101",
	"srs_ver": "6.0.777",
	"ttime": "2020-12-10T20:56:38.793+08:00",
	"tp": "play_summary",
	"dr_pkt_dur": 0,
	"dr_v_fs": 0,
	"dr_a_fs": 0,
	"kB": 0,
	"avg_kbr": 0,
	"xfor_ip": "",
	"dest_ip": "",
	"UA": "python-requests/2.25.0",
	"Referer": "",
	"srs_md": false,
	"play_cnt": 1,
	"req_sid": "",
	"sum_vfc": 0,
	"sum_afc": 0,
	"avg_vfr": 0,
	"avg_afr": 0,
	"cli_id": "__default__",
	"url": "http://120.201.26.174/",
	"stime": "2020-12-10T20:56:38.793+08:00",
	"dur": 0,
	"sid": "",
	"osid": "",
	"app": "__defaultVhost__",
	"vht": "",
	"cli_ip": "120.92.93.201",
	"v_c": "Other",
	"tsc_type": "org",
	"w": 0,
	"h": 0,
	"a_c": "Other",
	"a_s_rate": -1,
	"a_ch_num": 0,
	"cid": 595,
	"con": "http-play",
	"block_cnt": 0,
	"block_dur": 0,
	"skip_dur": 0,
	"no_video_dur": 0,
	"speed_1.5x": 0,
	"speed_1.25x": 0,
	"speed_1x": 0,
	"speed_0.75x": 0,
	"block_100s_dur": 0,
	"block_100s_cnt": 0,
	"exit_code": 9100
}

output

{
  "@timestamp": "2020-12-15T10:11:11.591Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.10.1"
  },
  "srs_md": false,
  "tp": "play_summary",
  "sum_vfc": 0,
  "url": "http://120.201.26.174/",
  "no_video_dur": 0,
  "block_100s_dur": 0,
  "block_dur": 0,
  "skip_dur": 0,
  "dr_pkt_dur": 0,
  "v_c": "Other",
  "dur": 0,
  "a_ch_num": 0,
  "tsc_type": "org",
  "stime": "2020-12-10T20:56:38.793+08:00",
  "speed_0.75x": 0,
  "srv_host": "yangdan03.local",
  "app": "__defaultVhost__",
  "UA": "python-requests/2.25.0",
  "h": 0,
  "mydur": 0,
  "play_cnt": 1,
  "avg_kbr": 0,
  "dest_ip": "",
  "vht": "",
  "a_s_rate": -1,
  "cid": 595,
  "sid": "",
  "Referer": "",
  "srs_ver": "6.0.777",
  "xfor_ip": "",
  "kB": 0,
  "block_cnt": 0,
  "osid": "",
  "block_100s_cnt": 0,
  "@log_type": "srs",
  "net_pub": 1,
  "speed_1x": 0,
  "cli_ip": "120.92.93.201",
  "avg_vfr": 0,
  "sum_afc": 0,
  "w": 0,
  "srv_id": "cdn-kes-ln-shenyang-ctum-edge274-db-txyun_8101",
  "req_sid": "",
  "dr_v_fs": 0,
  "avg_afr": 0,
  "con": "http-play",
  "a_c": "Other",
  "cli_id": "__default__",
  "speed_1.5x": 0,
  "exit_code": 9100,
  "speed_1.25x": 0,
  "ttime": "2020-12-10T20:56:38.793+08:00",
  "dr_a_fs": 0
}

got err:

Fail to apply processor global{copy_fields=[{From:host.name To:srv_host} {From:@timestamp To:logUploadKafkaTime} {From:dur To:mydur}], drop_fields={"Fields":["agent","@timestamp","host","log","@metadata","ecs"],"IgnoreMissing":false}}: failed to drop field [@timestamp]: key not found

[1] but when move processors to data.yml:

filebeat.yml
filebeat.config.inputs:
  enabled: true
  path: conf/*.yml
  reload.enabled: true
  reload.period: 10s
output.console:
  enabled: true
  codec.json:
    pretty: true
logging.level: debug

        

conf/data.yml
- type: log
  paths:
    - ./srs.log
  fields_under_root: true
  fields:
    '@log_type': srs
    net_pub: 1
  json.keys_under_root: true
  json.overwrite_keys: true
  json.add_error_key: false
  processors:
    - copy_fields:
        fields:
          - from: host.name
            to: srv_host
          - from: '@timestamp'
            to: logUploadKafkaTime
          - from: dur
            to: mydur-srs
        fail_on_error: false
        ignore_missing: true
    - drop_fields:
        fields: ["agent","@timestamp","host","log","@metadata","ecs"]

the same input, but I got a different output

{
  "@timestamp": "2020-12-15T10:11:44.091Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.10.1"
  },
  "xfor_ip": "",
  "sid": "",
  "vht": "",
  "dur": 0,
  "srv_id": "cdn-kes-ln-shenyang-ctum-edge274-db-txyun_8101",
  "con": "http-play",
  "block_100s_dur": 0,
  "h": 0,
  "sum_vfc": 0,
  "avg_vfr": 0,
  "avg_afr": 0,
  "srs_md": false,
  "kB": 0,
  "osid": "",
  "no_video_dur": 0,
  "srs_ver": "6.0.777",
  "@log_type": "srs",
  "speed_1.5x": 0,
  "block_cnt": 0,
  "avg_kbr": 0,
  "tp": "play_summary",
  "cid": 595,
  "v_c": "Other",
  "speed_1.25x": 0,
  "a_ch_num": 0,
  "play_cnt": 1,
  "UA": "python-requests/2.25.0",
  "ttime": "2020-12-10T20:56:38.793+08:00",
  "w": 0,
  "mydur-srs": 0,
  "tsc_type": "org",
  "a_s_rate": -1,
  "block_dur": 0,
  "req_sid": "",
  "sum_afc": 0,
  "speed_1x": 0,
  "host": {
    "name": "yangdan03.local"
  },
  "dr_v_fs": 0,
  "dest_ip": "",
  "speed_0.75x": 0,
  "net_pub": 1,
  "a_c": "Other",
  "ecs": {
    "version": "1.6.0"
  },
  "exit_code": 9100,
  "Referer": "",
  "dr_pkt_dur": 0,
  "dr_a_fs": 0,
  "agent": {
    "id": "b1d84e01-ddc2-4e66-a189-4eb54b53eba1",
    "name": "yangdan03.local",
    "type": "filebeat",
    "version": "7.10.1",
    "hostname": "yangdan03.local",
    "ephemeral_id": "4d790536-60f2-418a-8c6f-eee1557c0809"
  },
  "skip_dur": 0,
  "block_100s_cnt": 0,
  "url": "http://120.201.26.174/",
  "app": "__defaultVhost__",
  "cli_id": "__default__",
  "stime": "2020-12-10T20:56:38.793+08:00",
  "cli_ip": "120.92.93.201"
}

I got

Fail to apply processor client{copy_fields=[{From:host.name To:srv_host} {From:@timestamp To:logUploadKafkaTime} {From:dur To:mydur-srs}], drop_fields={"Fields":["agent","@timestamp","host","log","@metadata","ecs"],"IgnoreMissing":false}}: failed to drop field [agent]: key not found; failed to drop field [@timestamp]: key not found; failed to drop field [host]: key not found; failed to drop field [ecs]: key not found

why? how can I fix it?

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.