I collect log file and output to es with filebeat, But I get many unused fileds,like this:
The many of field which is no value, How to delete it
The filelds is no value，I don’t want to send the fields to es
Hi @kiddingl
Could you tell me exactly where in Kibana you're seeing this? Also which inputs/modules you're using?
It's quite hard to tell where in Kibana you took those screenshots.
Also, which version of Kibana and Elasticsearch are you running?
[root@app8 filebeat_bak2]# cat filebeat.yml
output.elasticsearch:
hosts: ["app5:9200"]
username: "elastic"
password: "g8UiNqgnfwqW"
filebeat.config.inputs:
path: /opt/envs/filebeat/*.yml
enabled: true
processors:
- drop_fields:
fields: ["log","host","input","agent","aws*"]
ignore_missing: false
[root@app8 filebeat_bak2]# cat /opt/envs/filebeat/starx_ner_new_log.yml
#filebeat.inputs:
- type: log #默认log，从日志文件读取每一行。stdin，从标准输入读取
enabled: true
paths:
- /opt/logs/starx_ner_new/*.bak # 这里也可用通配符采集多个文件如F:/test/*.log
multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
[root@app8 filebeat_bak2]#
{
"_index": "filebeat-7.8.0-2022.04.21-000001",
"_type": "_doc",
"_id": "SkkITIABSPmCCNxb9qvF",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2022-04-21T12:11:36.331Z",
"message": "2022-04-01 13:51:24.814 | ERROR | EngineCore.Models.Inserter:insert_core:74 - An error has been caught in function 'insert_core', process 'Process-46' (1970), thread 'Thread-24398' (139908274444032):\nTraceback (most recent call last):\n\n File \"/usr/local/lib/python3.7/threading.py\", line 885, in _bootstrap\n self._bootstrap_inner()\n │ └ <function Thread._bootstrap_inner at 0x7f3f27715598>\n └ <Timer(Thread-24398, started 139908274444032)>\n File \"/usr/local/lib/python3.7/threading.py\", line 917, in _bootstrap_inner\n self.run()\n │ └ <function Timer.run at 0x7f3f27715f28>\n └ <Timer(Thread-24398, started 139908274444032)>\n File \"/usr/local/lib/python3.7/threading.py\", line 1166, in run\n self.function(*self.args, **self.kwargs)\n │ │ │ │ │ └ {}\n │ │ │ │ └ <Timer(Thread-24398, started 139908274444032)>\n │ │ │ └ (<EventProxy object, typeid 'Event' at 0x7f3f1e1a11d0>,)\n │ │ └ <Timer(Thread-24398, started 139908274444032)>\n │ └ <bound method DorisInserter.insert_start of <EngineCore.Models.Inserter.DorisInserter object at 0x7f3f23e7bc50>>\n └ <Timer(Thread-24398, started 139908274444032)>\n\n File \"/opt/workspace/starx-ner-new/EngineCore/Models/Inserter.py\", line 51, in insert_start\n self.insert_core(insert_data)\n │ │ └ [({'EEP': [], 'PPN': [], 'Religion': [], 'degree': [], 'mac': [], 'OfficerID': [], 'email': [], 'HK_id': [], 'carnum': [], 'i...\n │ └ <function DorisInserter.insert_core at 0x7f3f21b19488>\n └ <EngineCore.Models.Inserter.DorisInserter object at 0x7f3f23e7bc50>\n\n> File \"/opt/workspace/starx-ner-new/EngineCore/Models/Inserter.py\", line 74, in insert_core\n doris_session.insert(insert_data_list)\n │ │ └ [({'EEP': [], 'PPN': [], 'Religion': [], 'degree': [], 'mac': [], 'OfficerID': [], 'email': [], 'HK_id': [], 'carnum': [], 'i...\n │ └ <function DorisSession.insert at 0x7f3f21b82620>\n └ <EngineCore.Models.Doris.Doris.DorisSession object at 0x7f3f224daeb8>\n\n File \"/opt/workspace/starx-ner-new/EngineCore/Models/Doris/Doris.py\", line 37, in insert\n merge_data.update(self.etccdd_obj.get_custom_data())\n │ │ └ <EngineCore.Models.Doris.Doris.DorisSession object at 0x7f3f224daeb8>\n │ └ <method 'update' of 'dict' objects>\n └ {'date': <class 'EngineCore.RegularCore.EntityTypeLibs.BaseType.DateType'>, 'credit': <class 'EngineCore.RegularCore.EntityTy...\n\nAttributeError: 'DorisSession' object has no attribute 'etccdd_obj'",
"ecs": {
"version": "1.5.0"
}
},
"fields": {
"cef.extensions.flexDate1": [],
"netflow.flow_end_microseconds": [],
"netflow.system_init_time_milliseconds": [],
"netflow.flow_end_nanoseconds": [],
"misp.observed_data.last_observed": [],
"netflow.max_flow_end_microseconds": [],
"file.mtime": [],
"aws.cloudtrail.user_identity.session_context.creation_date": [],
"netflow.min_flow_start_seconds": [],
"misp.intrusion_set.first_seen": [],
"file.created": [],
"misp.threat_indicator.valid_from": [],
"process.parent.start": [],
"azure.auditlogs.properties.activity_datetime": [],
"crowdstrike.event.ProcessStartTime": [],
"zeek.ocsp.update.this": [],
"crowdstrike.event.IncidentStartTime": [],
"netflow.observation_time_microseconds": [],
"event.start": [],
"cef.extensions.agentReceiptTime": [],
"cef.extensions.oldFileModificationTime": [],
"checkpoint.subs_exp": [],
"event.end": [],
"netflow.max_flow_end_milliseconds": [],
"netflow.min_flow_start_nanoseconds": [],
"zeek.smb_files.times.changed": [],
"crowdstrike.event.StartTimestamp": [],
"netflow.flow_start_nanoseconds": [],
"netflow.flow_start_seconds": [],
"crowdstrike.event.ProcessEndTime": [],
"zeek.x509.certificate.valid.until": [],
"misp.observed_data.first_observed": [],
"netflow.exporter.timestamp": [],
"netflow.monitoring_interval_start_milli_seconds": [],
"cef.extensions.oldFileCreateTime": [],
"event.ingested": [],
"@timestamp": [
"2022-04-21T12:11:36.331Z"
],
"zeek.ocsp.update.next": [],
"crowdstrike.event.UTCTimestamp": [],
"tls.server.not_before": [],
"cef.extensions.startTime": [],
"netflow.min_flow_start_milliseconds": [],
"azure.signinlogs.properties.created_at": [],
"cef.extensions.endTime": [],
"suricata.eve.tls.notbefore": [],
"zeek.kerberos.valid.from": [],
"cef.extensions.fileCreateTime": [],
"misp.threat_indicator.valid_until": [],
"crowdstrike.event.EndTimestamp": [],
"misp.campaign.last_seen": [],
"cef.extensions.deviceReceiptTime": [],
"netflow.observation_time_seconds": [],
"crowdstrike.metadata.eventCreationTime": [],
"cef.extensions.fileModificationTime": [],
"tls.client.not_before": [],
"zeek.smb_files.times.created": [],
"zeek.smtp.date": [],
"netflow.collection_time_milliseconds": [],
"zeek.pe.compile_time": [],
"netflow.max_flow_end_seconds": [],
"tls.client.not_after": [],
"netflow.flow_start_milliseconds": [],
"event.created": [],
"package.installed": [],
"zeek.kerberos.valid.until": [],
"suricata.eve.flow.end": [],
"netflow.observation_time_milliseconds": [],
"netflow.flow_start_microseconds": [],
"tls.server.not_after": [],
"netflow.flow_end_seconds": [],
"process.start": [],
"suricata.eve.tls.notafter": [],
"zeek.snmp.up_since": [],
"azure.enqueued_time": [],
"netflow.max_flow_end_nanoseconds": [],
"misp.intrusion_set.last_seen": [],
"netflow.min_flow_start_microseconds": [],
"netflow.observation_time_nanoseconds": [],
"cef.extensions.managerReceiptTime": [],
"file.accessed": [],
"netflow.flow_end_milliseconds": [],
"misp.campaign.first_seen": [],
"netflow.min_export_seconds": [],
"suricata.eve.flow.start": [],
"suricata.eve.timestamp": [
"2022-04-21T12:11:36.331Z"
],
"cef.extensions.deviceCustomDate1": [],
"cef.extensions.deviceCustomDate2": [],
"netflow.monitoring_interval_end_milli_seconds": [],
"file.ctime": [],
"crowdstrike.event.IncidentEndTime": [],
"zeek.smb_files.times.accessed": [],
"zeek.ocsp.revoke.time": [],
"zeek.x509.certificate.valid.from": [],
"netflow.max_export_seconds": [],
"zeek.smb_files.times.modified": [],
"kafka.block_timestamp": [],
"misp.report.published": []
},
"sort": [
1650543096331
]
}
why the fields includes so many unused field, how to not collect it
I found a solution ，I disable default template in filebat
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.