fixed it ! it was that the logs was partly json partly not . i talked to the developer to make all logs json.
now i have 2 problems
as you can see it doesn't parse only one field
"body": {
"data": {
"bar_codes": [
{
"bar_code_information": [
{
"score": 0,
"bar_code": "NoAction"
}
]
},
{
"bar_code_information": [
{
"bar_code": "ProductInsertion",
"score": 0
}
],
"frame_id": 1
}
],
"message": "OK"
},
is there a way to make filebeat to parse this data also as fileds?
the second issue i having duplicate entries , it's one time parsed as json and second time as as string
{
"_index": "filebeat-7.11.1-2021.03.16",
"_type": "_doc",
"_id": "GcbwOXgBJMHPPsfR4NM1",
"_version": 1,
"_score": 0,
"_source": {
"@timestamp": "2021-03-16T07:28:58.577Z",
"fields": {
"env": "dev"
},
"container": {
"id": "ae0c9e7a707cb7df5da7c972404dc748072bf9ae47caaea4550969a206a1ca68",
"image": {
"name": "tracx/grpc-proxy:2.0.15.9fc4db8"
},
"name": "grpc_proxy_pr",
"labels": {
"com_docker_compose_project_working_dir": "/home/avraham/grpc-proxy",
"co_elastic_logs/json_overwrite_keys": "True",
"com_docker_compose_oneoff": "False",
"co_elastic_logs/json_expand_keys": "True",
"com_docker_compose_container-number": "1",
"com_docker_compose_service": "grpc_proxy_pr",
"com_docker_compose_version": "1.26.2",
"co_elastic_logs/json_keys_under_root": "True",
"com_docker_compose_config-hash": "a0bc966fca142f4dced4b88c4a0a403a3b3f5932b4bc78d6260286c249163202",
"com_docker_compose_project": "grpc-proxy",
"com_docker_compose_project_config_files": "docker-compose.yml",
"co_elastic_logs/json_add_error_key": "True"
}
},
"stream": "stdout",
"message": "{\"@timestamp\":\"2021-03-16T07:28:58.577Z\",\"body\":{\"actionid\":\"2274\",\"cameraid\":\"3\",\"cartid\":\"5e14b2e301c4a4130000004e\",\"data\":{\"images\":[\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737797.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737847.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737897.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737948.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737997.jpg\"]},\"event\":\"/gpu_server.pipeline/send_data\",\"eventtype\":\"pipeline\",\"handle\":16839657125260286000,\"journeyid\":\"123456789\"},\"ecs.version\":\"1.6.0\",\"log-event-type\":\"aic-event\",\"log.level\":\"info\",\"message\":\"\"}",
"input": {
"type": "docker"
},
"ecs": {
"version": "1.6.0"
},
"host": {
"architecture": "x86_64",
"os": {
"name": "Ubuntu",
"kernel": "4.15.0-112-generic",
"codename": "bionic",
"platform": "ubuntu",
"version": "18.04.4 LTS (Bionic Beaver)",
"family": "debian"
},
"id": "fa5d375897824427adaa775265d7e126",
"containerized": false,
"name": "opdev",
"ip": [
"192.168.4.2",
"fe80::b67a:f1ff:fe33:5fee",
"16.1.15.2",
"fe80::e84c:b6ff:fe3e:a48d",
"192.168.5.51",
"192.168.122.1",
"172.21.0.1",
"172.19.0.1",
"172.23.0.1",
"172.18.0.1",
"fe80::42:a6ff:fec9:dc10",
"172.17.0.1",
"fe80::42:dff:fef0:940d",
"172.31.0.1",
"fe80::3cc8:fcff:fefa:dce4",
"fe80::602d:fdff:febe:8da8",
"fe80::6019:b1ff:feb8:8345",
"fe80::8874:fbff:feaf:fb74",
"fe80::3866:51ff:fec7:5a89",
"fe80::3c27:cff:fe7f:8b25",
"fe80::40ac:4cff:feac:2e9d",
"fe80::d086:91ff:feb3:bd40",
"fe80::e883:f6ff:fe15:911e",
"fe80::c4e6:2bff:fec2:dab1",
"fe80::10c1:66ff:fecf:ff53",
"fe80::a837:59ff:fe2a:be40",
"fe80::64ef:3fff:fe38:adbd",
"fe80::34c9:7dff:fe27:7a35",
"fe80::605b:54ff:fe36:3457",
"fe80::9c4c:80ff:fe92:7dce",
"fe80::18a7:96ff:fe80:de2d",
"fe80::c87:1dff:fe9a:aa0d",
"fe80::bc32:92ff:fe92:faab",
"fe80::e8e2:aeff:fef4:defb",
"fe80::98b9:feff:fe49:dcff",
"192.168.80.1",
"fe80::42:8eff:fe9f:14ac",
"fe80::8ba:9fff:fe9f:9cc5",
"fe80::143a:8dff:fe69:2675",
"fe80::e0db:20ff:fe2a:6fe",
"fe80::a821:5bff:fe59:5364",
"fe80::b002:8eff:fe66:11a0",
"fe80::947b:1ff:fe53:cf19",
"fe80::8ea:e2ff:fe6b:4a46",
"fe80::58d3:7eff:fed1:b04d",
"fe80::7897:7bff:fe08:546d",
"fe80::5c0c:c6ff:fe56:59f2",
"fe80::588f:a4ff:fe7a:9571",
"fe80::7cf2:6bff:fe11:7561",
"fe80::5c4d:a8ff:feb7:8317",
"fe80::20bd:51ff:fe96:f74d",
"fe80::906b:6dff:fea1:8b8d",
"fe80::dc01:44ff:fea6:a57e",
"fe80::ecf1:d2ff:fe87:f1d2",
"fe80::a844:11ff:fe3d:33fd",
"fe80::a82d:8bff:fe0f:f78c",
"fe80::8099:77ff:fe07:f65b"
],
"mac": [
"b4:7a:f1:33:5f:ee",
"b4:7a:f1:33:5f:ef",
"b4:7a:f1:33:5f:f0",
"b4:7a:f1:33:5f:f1",
"ea:4c:b6:3e:a4:8d",
"4e:b2:e3:e2:ff:35",
"52:54:00:22:5e:cf",
"52:54:00:22:5e:cf",
"02:42:f5:97:23:39",
"02:42:c5:69:06:7a",
"02:42:23:62:63:6e",
"02:42:a6:c9:dc:10",
"02:42:0d:f0:94:0d",
"02:42:6e:bb:3d:ee",
"3e:c8:fc:fa:dc:e4",
"62:2d:fd:be:8d:a8",
"62:19:b1:b8:83:45",
"8a:74:fb:af:fb:74",
"3a:66:51:c7:5a:89",
"3e:27:0c:7f:8b:25",
"42:ac:4c:ac:2e:9d",
"d2:86:91:b3:bd:40",
"ea:83:f6:15:91:1e",
"c6:e6:2b:c2:da:b1",
"12:c1:66:cf:ff:53",
"aa:37:59:2a:be:40",
"66:ef:3f:38:ad:bd",
"36:c9:7d:27:7a:35",
"62:5b:54:36:34:57",
"9e:4c:80:92:7d:ce",
"1a:a7:96:80:de:2d",
"0e:87:1d:9a:aa:0d",
"be:32:92:92:fa:ab",
"ea:e2:ae:f4:de:fb",
"9a:b9:fe:49:dc:ff",
"02:42:8e:9f:14:ac",
"0a:ba:9f:9f:9c:c5",
"16:3a:8d:69:26:75",
"e2:db:20:2a:06:fe",
"aa:21:5b:59:53:64",
"b2:02:8e:66:11:a0",
"96:7b:01:53:cf:19",
"0a:ea:e2:6b:4a:46",
"5a:d3:7e:d1:b0:4d",
"7a:97:7b:08:54:6d",
"5e:0c:c6:56:59:f2",
"5a:8f:a4:7a:95:71",
"7e:f2:6b:11:75:61",
"5e:4d:a8:b7:83:17",
"22:bd:51:96:f7:4d",
"92:6b:6d:a1:8b:8d",
"de:01:44:a6:a5:7e",
"ee:f1:d2:87:f1:d2",
"aa:44:11:3d:33:fd",
"aa:2d:8b:0f:f7:8c",
"82:99:77:07:f6:5b"
],
"hostname": "opdev"
},
"agent": {
"ephemeral_id": "7e8dcd0e-d8e6-449d-9e50-e03f9537792b",
"id": "548383b1-c613-433d-ace3-ce750bd3af07",
"name": "opdev",
"type": "filebeat",
"version": "7.11.1",
"hostname": "opdev"
},
"log": {
"offset": 123863,
"file": {
"path": "/var/lib/docker/containers/ae0c9e7a707cb7df5da7c972404dc748072bf9ae47caaea4550969a206a1ca68/ae0c9e7a707cb7df5da7c972404dc748072bf9ae47caaea4550969a206a1ca68-json.log"
}
},
"tags": [
"opdev",
"haifa_office"
]
},
"fields": {
"@timestamp": [
"2021-03-16T07:28:58.577Z"
]
},
"highlight": {
"agent.name": [
"@kibana-highlighted-field@opdev@/kibana-highlighted-field@"
],
"message": [
"{\"@timestamp\":\"2021-03-16T07:28:58.577Z\",\"body\":{\"actionid\":\"2274\",\"cameraid\":\"3\",\"cartid\":\"5e14b2e301c4a4130000004e\",\"data\":{\"images\":[\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737797.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737847.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737897.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737948.jpg\",\"/usr/data/cart-data/pipeline/5e14b2e301c4a4130000004e/123456789/2274/3/1615879737997.jpg\"]},\"@kibana-highlighted-field@event@/kibana-highlighted-field@\":\"/gpu_server.pipeline/send_data\",\"eventtype\":\"pipeline\",\"handle\":16839657125260286000,\"journeyid\":\"123456789\"},\"ecs.version\":\"1.6.0\",\"log-@kibana-highlighted-field@event@/kibana-highlighted-field@-type\":\"@kibana-highlighted-field@aic@/kibana-highlighted-field@-@kibana-highlighted-field@event@/kibana-highlighted-field@\",\"log.level\":\"info\",\"message\":\"\"}"
]
},
"sort": [
0,
1615879738577
]
}
as you can see i have everything duplicate now . is there a way to automatically exclude this container from the docker scanning
which uses
container.id
- *
maybe add some more label to the specific container?
