I have investigated for a month and still cannot find a solution to structure the json data captured by Wireshark. I am using FileBeat in Elastic to import json data. However, all the json data are packed to the "message" fields.
What I need to do is that I have to search the fields in that json (e.g. ip_ip_dst_host, tcp_tcp_dstport, timestamp, tcp_tcp_payload)
The below is the result showing in Elastic:
I know FileBeat has some default fields, but for me, they are useless. I only take care of the fields in "message"
And below is the FileBeat configuraton:
filebeat.inputs:
- type: log
enabled: true
paths:
- C:\wireshark\*.json
json.keys_under_root: true
json.add_error_key: true
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
host: "http://localhost:5601"
output.elasticsearch:
# Array of hosts to connect to.
hosts: ["http://localhost:9200"]
username: "xxx"
password: "xxx"
pipeline: message_process
#index: "testserver01-%{+yyyy-MM-dd}"
indices:
- index: "packet-%{[agent.version]}-%{+yyyy.MM.dd}"
template.enabled: true
setup.template.name: "packet"
setup.template.pattern: packet-*"
processors:
- add_host_metadata:
when.not.contains.tags: forwarded
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
#- drop_event:
# when:
# equals:
# index._type: "pcap_file"
- decode_json_fields:
fields: ["message"]
process_array: true
max_depth: 6
target: ""
overwrite_keys: true
add_error_key: true
when:
regexp:
message: '^\{'
My json file is like that:
{"timestamp":"1639101979659","layers":{"frame":{"frame_frame_interface_id":"0","frame_frame_encap_type":"1","frame_frame_time":"2021-12-10T02:06:19.659398000Z","frame_frame_offset_shift":"0.000000000","frame_frame_time_epoch":"1639101979.659398000","frame_frame_time_delta":"0.000000000","frame_frame_time_delta_displayed":"0.000000000","frame_frame_time_relative":"0.000000000","frame_frame_number":"1","frame_frame_len":"112","frame_frame_cap_len":"112","frame_frame_marked":false,"frame_frame_ignored":false,"frame_frame_protocols":"eth:ethertype:ip:tcp:tls"},"eth":{"eth_eth_dst":"ee:ee:ee:ee:ee:ee","eth_eth_dst_resolved":"ee:ee:ee:ee:ee:ee","eth_eth_dst_oui":"15658734","eth_eth_addr":"ee:ee:ee:ee:ee:ee","eth_eth_addr_resolved":"ee:ee:ee:ee:ee:ee","eth_eth_addr_oui":"15658734","eth_eth_dst_lg":true,"eth_eth_lg":true,"eth_eth_dst_ig":false,"eth_eth_ig":false,"eth_eth_src_oui":"12503387","eth_eth_addr_oui":"12503387","eth_eth_src_lg":true,"eth_eth_lg":true,"eth_eth_src_ig":false,"eth_eth_ig":false,"eth_eth_type":"0x0800"},"ip":{"ip_ip_version":"4","ip_ip_hdr_len":"20","ip_ip_dsfield":"0x00","ip_ip_dsfield_dscp":"0","ip_ip_dsfield_ecn":"0","ip_ip_len":"98","ip_ip_id":"0xa688","ip_ip_flags":"0x40","ip_ip_flags_rb":false,"ip_ip_flags_df":true,"ip_ip_flags_mf":false,"ip_ip_frag_offset":"0","ip_ip_ttl":"64","ip_ip_proto":"6","ip_ip_checksum":"0x65e1","ip_ip_checksum_status":"2","ip_ip_src":"10.200.25.98","ip_ip_addr":["10.200.25.98","10.2.0.1"],"ip_ip_src_host":"10.200.25.98","ip_ip_host":["10.200.25.98","10.2.0.1"],"ip_ip_dst":"10.2.0.1","ip_ip_dst_host":"10.2.0.1"},"tcp":{"tcp_tcp_srcport":"51344","tcp_tcp_dstport":"443","tcp_tcp_port":["51344","443"],"tcp_tcp_stream":"0","tcp_tcp_completeness":"0","tcp_tcp_len":"46","tcp_tcp_seq":"1","tcp_tcp_seq_raw":"278114923","tcp_tcp_nxtseq":"47","tcp_tcp_ack":"1","tcp_tcp_ack_raw":"212180543","tcp_tcp_hdr_len":"32","tcp_tcp_flags":"0x0018","tcp_tcp_flags_res":false,"tcp_tcp_flags_ns":false,"tcp_tcp_flags_cwr":false,"tcp_tcp_flags_ecn":false,"tcp_tcp_flags_urg":false,"tcp_tcp_flags_ack":true,"tcp_tcp_flags_push":true,"tcp_tcp_flags_reset":false,"tcp_tcp_flags_syn":false,"tcp_tcp_flags_fin":false,"tcp_tcp_flags_str":"┬╖┬╖┬╖┬╖┬╖┬╖┬╖AP┬╖┬╖┬╖","tcp_tcp_window_size_value":"1424","tcp_tcp_window_size":"1424","tcp_tcp_window_size_scalefactor":"-1","tcp_tcp_checksum":"0x2e81","tcp_tcp_checksum_status":"2","tcp_tcp_urgent_pointer":"0","tcp_options_nop":["01","01"],"tcp_tcp_option_kind":["1","1"],"tcp_tcp_option_kind":"8","tcp_tcp_option_len":"10","tcp_tcp_options_timestamp_tsval":"3018575007","tcp_tcp_options_timestamp_tsecr":"3018573006","text":"Timestamps","tcp_tcp_time_relative":"0.000000000","tcp_tcp_time_delta":"0.000000000","tcp_tcp_analysis":null,"tcp_tcp_analysis_bytes_in_flight":"46","tcp_tcp_analysis_push_bytes_sent":"46","tcp_tcp_payload":"17:03:03:00:29:00:00:00:00:00:5b:ed:22:56:ab:29:f8:ad:4d:f2:5a:65:f0:17:15:46:3c:ae:3e:30:8e:ff:16:46:46:42:9a:56:e5:6b:85:53:46:6a:f9:ee"},"tls":{"tls_tls_record":null,"tls_tls_record_content_type":"23","tls_tls_record_version":"0x0303","tls_tls_record_length":"41","tls_tls_app_data":"00:00:00:00:00:5b:ed:22:56:ab:29:f8:ad:4d:f2:5a:65:f0:17:15:46:3c:ae:3e:30:8e:ff:16:46:46:42:9a:56:e5:6b:85:53:46:6a:f9:ee","tls_tls_app_data_proto":"http-over-tls"}}}
{"timestamp":"1639101979662","layers":{"frame":{"frame_frame_interface_id":"0","frame_frame_encap_type":"1","frame_frame_time":"2021-12-10T02:06:19.662332000Z","frame_frame_offset_shift":"0.000000000","frame_frame_time_epoch":"1639101979.662332000","frame_frame_time_delta":"0.002934000","frame_frame_time_delta_displayed":"0.002934000","frame_frame_time_relative":"0.002934000","frame_frame_number":"2","frame_frame_len":"132","frame_frame_cap_len":"132","frame_frame_marked":false,"frame_frame_ignored":false,"frame_frame_protocols":"eth:ethertype:ip:tcp:tls"},"eth":{"eth_eth_dst_oui":"12503387","eth_eth_addr_oui":"12503387","eth_eth_dst_lg":true,"eth_eth_lg":true,"eth_eth_dst_ig":false,"eth_eth_ig":false,"eth_eth_src":"ee:ee:ee:ee:ee:ee","eth_eth_src_resolved":"ee:ee:ee:ee:ee:ee","eth_eth_src_oui":"15658734","eth_eth_addr":"ee:ee:ee:ee:ee:ee","eth_eth_addr_resolved":"ee:ee:ee:ee:ee:ee","eth_eth_addr_oui":"15658734","eth_eth_src_lg":true,"eth_eth_lg":true,"eth_eth_src_ig":false,"eth_eth_ig":false,"eth_eth_type":"0x0800"},"ip":{"ip_ip_version":"4","ip_ip_hdr_len":"20","ip_ip_dsfield":"0x00","ip_ip_dsfield_dscp":"0","ip_ip_dsfield_ecn":"0","ip_ip_len":"118","ip_ip_id":"0xa750","ip_ip_flags":"0x40","ip_ip_flags_rb":false,"ip_ip_flags_df":true,"ip_ip_flags_mf":false,"ip_ip_frag_offset":"0","ip_ip_ttl":"64","ip_ip_proto":"6","ip_ip_checksum":"0x6505","ip_ip_checksum_status":"2","ip_ip_src":"10.2.0.1","ip_ip_addr":["10.2.0.1","10.200.25.98"],"ip_ip_src_host":"10.2.0.1","ip_ip_host":["10.2.0.1","10.200.25.98"],"ip_ip_dst":"10.200.25.98","ip_ip_dst_host":"10.200.25.98"},"tcp":{"tcp_tcp_srcport":"443","tcp_tcp_dstport":"51344","tcp_tcp_port":["443","51344"],"tcp_tcp_stream":"0","tcp_tcp_completeness":"0","tcp_tcp_len":"66","tcp_tcp_seq":"1","tcp_tcp_seq_raw":"212180543","tcp_tcp_nxtseq":"67","tcp_tcp_ack":"47","tcp_tcp_ack_raw":"278114969","tcp_tcp_hdr_len":"32","tcp_tcp_flags":"0x0018","tcp_tcp_flags_res":false,"tcp_tcp_flags_ns":false,"tcp_tcp_flags_cwr":false,"tcp_tcp_flags_ecn":false,"tcp_tcp_flags_urg":false,"tcp_tcp_flags_ack":true,"tcp_tcp_flags_push":true,"tcp_tcp_flags_reset":false,"tcp_tcp_flags_syn":false,"tcp_tcp_flags_fin":false,"tcp_tcp_flags_str":"┬╖┬╖┬╖┬╖┬╖┬╖┬╖AP┬╖┬╖┬╖","tcp_tcp_window_size_value":"1432","tcp_tcp_window_size":"1432","tcp_tcp_window_size_scalefactor":"-1","tcp_tcp_checksum":"0x2e95","tcp_tcp_checksum_status":"2","tcp_tcp_urgent_pointer":"0","tcp_options_nop":["01","01"],"tcp_tcp_option_kind":["1","1"],"tcp_tcp_option_kind":"8","tcp_tcp_option_len":"10","tcp_tcp_options_timestamp_tsval":"3018575010","tcp_tcp_options_timestamp_tsecr":"3018575007","text":"Timestamps","tcp_tcp_time_relative":"0.002934000","tcp_tcp_time_delta":"0.002934000","tcp_tcp_analysis":null,"tcp_tcp_analysis_acks_frame":"1","tcp_tcp_analysis_ack_rtt":"0.002934000","tcp_tcp_analysis_bytes_in_flight":"66","tcp_tcp_analysis_push_bytes_sent":"66","tcp_tcp_payload":"17:03:03:00:3d:00:00:00:00:00:8a:0b:87:f3:5d:2a:90:58:80:23:e8:75:b4:44:be:41:70:62:66:ae:d7:95:d2:de:3b:b2:5e:35:63:7e:b6:27:a9:37:45:bc:4a:b7:a1:4c:47:fa:37:58:14:83:e8:9b:ce:b7:5f:2a:8a:ce:6d:41"},"tls":{"tls_tls_record":null,"tls_tls_record_content_type":"23","tls_tls_record_version":"0x0303","tls_tls_record_length":"61","tls_tls_app_data":"00:00:00:00:00:8a:0b:87:f3:5d:2a:90:58:80:23:e8:75:b4:44:be:41:70:62:66:ae:d7:95:d2:de:3b:b2:5e:35:63:7e:b6:27:a9:37:45:bc:4a:b7:a1:4c:47:fa:37:58:14:83:e8:9b:ce:b7:5f:2a:8a:ce:6d:41","tls_tls_app_data_proto":"http-over-tls"}}}
