[Packetbeat] packet loss for mysql queries

Elastic Stack Beats
1
  • Version: packetbeat-8.10.4、packetbeat-7.10.2
  • Operating System: ubuntu20.04

#18471
#20890

Three years have passed, and the problem of packet loss in MySQL queries remains unresolved.

sql:

use dba_backup;
select * from a;

packetbeat debug logs
{"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":170},"message":"decode packet data","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":385},"message":"IPv4 packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":426},"message":"TCP packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"tcp","log.origin":{"file.name":"tcp/tcp.go","file.line":206},"message":"Connection src[192.168.90.61:54546] dst[192.168.99.7:7306] doesn't exist, creating new","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"tcp","log.origin":{"file.name":"tcp/tcp.go","file.line":114},"message":"tcp flow id: 0x0","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"tcp","log.origin":{"file.name":"tcp/tcp.go","file.line":127},"message":"pkt.start_seq=2062002113 pkt.last_seq=2062002133 stream.last_seq=0 (len=20)","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":275},"message":"MySQL parser called. parseState = Start","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.828+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":291},"message":"MySQL Header: Packet length 16, Seq 0, Type=3 isClient=true","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":382},"message":"Message complete. remaining=0","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":592},"message":"mysqlMessageParser returned ok=true complete=true","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":230},"message":"processed","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":170},"message":"decode packet data","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":385},"message":"IPv4 packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":426},"message":"TCP packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"tcp","log.origin":{"file.name":"tcp/tcp.go","file.line":114},"message":"tcp flow id: 0x0","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"tcp","log.origin":{"file.name":"tcp/tcp.go","file.line":127},"message":"pkt.start_seq=174114368 pkt.last_seq=174114561 stream.last_seq=0 (len=193)","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":275},"message":"MySQL parser called. parseState = Start","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":291},"message":"MySQL Header: Packet length 1, Seq 1, Type=3 isClient=false","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":321},"message":"Query response. Number of fields 3","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":451},"message":"db=dba_backup, table=a","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":451},"message":"db=dba_backup, table=a","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":451},"message":"db=dba_backup, table=a","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"mysqldetailed","log.origin":{"file.name":"mysql/mysql.go","file.line":592},"message":"mysqlMessageParser returned ok=true complete=false","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":230},"message":"processed","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":170},"message":"decode packet data","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":385},"message":"IPv4 packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":426},"message":"TCP packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":497},"message":"Ignore empty non-FIN packet","service.name":"packetbeat","ecs.version":"1.6.0"} {"log.level":"debug","@timestamp":"2023-10-27T01:44:04.829+0800","log.logger":"decoder","log.origin":{"file.name":"decoder/decoder.go","file.line":230},"message":"processed","service.name":"packetbeat","ecs.version":"1.6.0"}

2

I deployed packetbeat on the application side machine to capture MySQL traffic, and used the MySQL client to execute SQL statements. Except for the select statement, everything else can be captured normally. Is it due to my missing configuration or the reason?

packetbeat.interfaces.device: any
packetbeat.interfaces.poll_default_route: 1m
packetbeat.interfaces.internal_networks:
  - private
packetbeat.interfaces.type: af_packet
packetbeat.interfaces.metrics_interval: 10s

packetbeat.flows:
  enabled: false
  timeout: 30s
  period: 10s

packetbeat.protocols:
- type: mysql
  ports: [7306]
packetbeat.procs.enabled: false
output.kafka:
  enabled: true
  hosts: ["192.168.90.61:9092"]
  topic: "packetbeat"
3

The problem has been found. When using the MySQL client to execute SQL statements, the select cannot be obtained or the information obtained is inaccurate. There is no problem using other clients such as pymysql

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.