In the official doc, it says:
Processors are valid:
- At the top-level in the configuration. The processor is applied to all data collected by Filebeat.
- Under a specific input. The processor is applied to the data collected for that input.
but I found it has differents action between processors at top-level from under specific input:
below are my config top-level :(I'm aware that metadata and timestamp cannot be dropped)
filebeat.yml
filebeat.config.inputs:
enabled: true
path: conf/*.yml
reload.enabled: true
reload.period: 10s
output.console:
enabled: true
codec.json:
pretty: true
logging.level: debug
processors:
- copy_fields:
fields:
- from: host.name
to: srv_host
- from: '@timestamp'
to: logUploadKafkaTime
- from: dur
to: mydur
fail_on_error: false
ignore_missing: true
- drop_fields:
fields: ["agent","@timestamp","host","log","@metadata","ecs"]
conf/data.yml
- type: log
paths:
- ./data.log
fields_under_root: true
fields:
'@log_type': data
net_pub: 1
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: false
input:
{
"srv_id": "cdn-kes-ln-shenyang-ctum-edge274-db-txyun_8101",
"srs_ver": "6.0.777",
"ttime": "2020-12-10T20:56:38.793+08:00",
"tp": "play_summary",
"dr_pkt_dur": 0,
"dr_v_fs": 0,
"dr_a_fs": 0,
"kB": 0,
"avg_kbr": 0,
"xfor_ip": "",
"dest_ip": "",
"UA": "python-requests/2.25.0",
"Referer": "",
"srs_md": false,
"play_cnt": 1,
"req_sid": "",
"sum_vfc": 0,
"sum_afc": 0,
"avg_vfr": 0,
"avg_afr": 0,
"cli_id": "__default__",
"url": "http://120.201.26.174/",
"stime": "2020-12-10T20:56:38.793+08:00",
"dur": 0,
"sid": "",
"osid": "",
"app": "__defaultVhost__",
"vht": "",
"cli_ip": "120.92.93.201",
"v_c": "Other",
"tsc_type": "org",
"w": 0,
"h": 0,
"a_c": "Other",
"a_s_rate": -1,
"a_ch_num": 0,
"cid": 595,
"con": "http-play",
"block_cnt": 0,
"block_dur": 0,
"skip_dur": 0,
"no_video_dur": 0,
"speed_1.5x": 0,
"speed_1.25x": 0,
"speed_1x": 0,
"speed_0.75x": 0,
"block_100s_dur": 0,
"block_100s_cnt": 0,
"exit_code": 9100
}
output
{
"@timestamp": "2020-12-15T10:11:11.591Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.10.1"
},
"srs_md": false,
"tp": "play_summary",
"sum_vfc": 0,
"url": "http://120.201.26.174/",
"no_video_dur": 0,
"block_100s_dur": 0,
"block_dur": 0,
"skip_dur": 0,
"dr_pkt_dur": 0,
"v_c": "Other",
"dur": 0,
"a_ch_num": 0,
"tsc_type": "org",
"stime": "2020-12-10T20:56:38.793+08:00",
"speed_0.75x": 0,
"srv_host": "yangdan03.local",
"app": "__defaultVhost__",
"UA": "python-requests/2.25.0",
"h": 0,
"mydur": 0,
"play_cnt": 1,
"avg_kbr": 0,
"dest_ip": "",
"vht": "",
"a_s_rate": -1,
"cid": 595,
"sid": "",
"Referer": "",
"srs_ver": "6.0.777",
"xfor_ip": "",
"kB": 0,
"block_cnt": 0,
"osid": "",
"block_100s_cnt": 0,
"@log_type": "srs",
"net_pub": 1,
"speed_1x": 0,
"cli_ip": "120.92.93.201",
"avg_vfr": 0,
"sum_afc": 0,
"w": 0,
"srv_id": "cdn-kes-ln-shenyang-ctum-edge274-db-txyun_8101",
"req_sid": "",
"dr_v_fs": 0,
"avg_afr": 0,
"con": "http-play",
"a_c": "Other",
"cli_id": "__default__",
"speed_1.5x": 0,
"exit_code": 9100,
"speed_1.25x": 0,
"ttime": "2020-12-10T20:56:38.793+08:00",
"dr_a_fs": 0
}
got err:
Fail to apply processor global{copy_fields=[{From:host.name To:srv_host} {From:@timestamp To:logUploadKafkaTime} {From:dur To:mydur}], drop_fields={"Fields":["agent","@timestamp","host","log","@metadata","ecs"],"IgnoreMissing":false}}: failed to drop field [@timestamp]: key not found
[1] but when move processors to data.yml:
filebeat.yml
filebeat.config.inputs:
enabled: true
path: conf/*.yml
reload.enabled: true
reload.period: 10s
output.console:
enabled: true
codec.json:
pretty: true
logging.level: debug
conf/data.yml
- type: log
paths:
- ./srs.log
fields_under_root: true
fields:
'@log_type': srs
net_pub: 1
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: false
processors:
- copy_fields:
fields:
- from: host.name
to: srv_host
- from: '@timestamp'
to: logUploadKafkaTime
- from: dur
to: mydur-srs
fail_on_error: false
ignore_missing: true
- drop_fields:
fields: ["agent","@timestamp","host","log","@metadata","ecs"]
the same input, but I got a different output
{
"@timestamp": "2020-12-15T10:11:44.091Z",
"@metadata": {
"beat": "filebeat",
"type": "_doc",
"version": "7.10.1"
},
"xfor_ip": "",
"sid": "",
"vht": "",
"dur": 0,
"srv_id": "cdn-kes-ln-shenyang-ctum-edge274-db-txyun_8101",
"con": "http-play",
"block_100s_dur": 0,
"h": 0,
"sum_vfc": 0,
"avg_vfr": 0,
"avg_afr": 0,
"srs_md": false,
"kB": 0,
"osid": "",
"no_video_dur": 0,
"srs_ver": "6.0.777",
"@log_type": "srs",
"speed_1.5x": 0,
"block_cnt": 0,
"avg_kbr": 0,
"tp": "play_summary",
"cid": 595,
"v_c": "Other",
"speed_1.25x": 0,
"a_ch_num": 0,
"play_cnt": 1,
"UA": "python-requests/2.25.0",
"ttime": "2020-12-10T20:56:38.793+08:00",
"w": 0,
"mydur-srs": 0,
"tsc_type": "org",
"a_s_rate": -1,
"block_dur": 0,
"req_sid": "",
"sum_afc": 0,
"speed_1x": 0,
"host": {
"name": "yangdan03.local"
},
"dr_v_fs": 0,
"dest_ip": "",
"speed_0.75x": 0,
"net_pub": 1,
"a_c": "Other",
"ecs": {
"version": "1.6.0"
},
"exit_code": 9100,
"Referer": "",
"dr_pkt_dur": 0,
"dr_a_fs": 0,
"agent": {
"id": "b1d84e01-ddc2-4e66-a189-4eb54b53eba1",
"name": "yangdan03.local",
"type": "filebeat",
"version": "7.10.1",
"hostname": "yangdan03.local",
"ephemeral_id": "4d790536-60f2-418a-8c6f-eee1557c0809"
},
"skip_dur": 0,
"block_100s_cnt": 0,
"url": "http://120.201.26.174/",
"app": "__defaultVhost__",
"cli_id": "__default__",
"stime": "2020-12-10T20:56:38.793+08:00",
"cli_ip": "120.92.93.201"
}
I got
Fail to apply processor client{copy_fields=[{From:host.name To:srv_host} {From:@timestamp To:logUploadKafkaTime} {From:dur To:mydur-srs}], drop_fields={"Fields":["agent","@timestamp","host","log","@metadata","ecs"],"IgnoreMissing":false}}: failed to drop field [agent]: key not found; failed to drop field [@timestamp]: key not found; failed to drop field [host]: key not found; failed to drop field [ecs]: key not found
why? how can I fix it?