MySQL主从延迟根因诊断法全面详解

主库写入 → Binlog生产 → 网络传输 → Relay Log写入 → SQL重放
    ↓         ↓           ↓           ↓           ↓
  应用层    主库IO     网络层     从库IO     从库SQL

先量化 → 再定位 → 后优化
  ↓        ↓       ↓
真实延迟  瓶颈层次  针对性方案

-- 基础延迟查看
SHOW SLAVE STATUS\G

-- 关键字段解读
Seconds_Behind_Master: 0              -- 从库落后主库秒数（可能不准确）
Relay_Master_Log_File: mysql-bin.001  -- 当前正在重放的主库binlog文件
Exec_Master_Log_Pos: 1234567          -- 已执行到的位置
Read_Master_Log_Pos: 1234567          -- 已读取到的位置

-- 计算真实延迟（推荐）
SELECT 
    TIMESTAMPDIFF(SECOND, 
        FROM_UNIXTIME(@@global.sql_slave_skip_counter), 
        NOW()
    ) AS real_delay_seconds;

-- 方法1：基于binlog位置计算
SELECT 
    (Master_Log_File_Position - Exec_Master_Log_Pos) / 
    (主库binlog生成速度) AS estimated_delay_seconds;

-- 方法2：基于心跳表（最准确）
-- 主库定期插入时间戳
INSERT INTO heartbeat_table (ts) VALUES (NOW());

-- 从库查询延迟
SELECT TIMESTAMPDIFF(SECOND, ts, NOW()) AS real_delay 
FROM heartbeat_table ORDER BY id DESC LIMIT 1;

延迟高？
   ↓
IO线程延迟？ ←─ Relay_Log_Space_Increase 快？
   ↓ 是              ↓ 是
网络/主库问题       从库IO问题
   ↓ 否              ↓ 否
SQL线程延迟？ ←─ Seconds_Behind_Master 增长？
   ↓ 是
SQL执行问题
   ↓ 否
其他问题

-- 查看线程状态
SHOW SLAVE STATUS\G

-- 关键判断：
-- 1. IO线程延迟特征
Relay_Master_Log_File != Master_Log_File  -- 读取落后
Read_Master_Log_Pos - Exec_Master_Log_Pos 很大  -- 堆积多

-- 2. SQL线程延迟特征
Relay_Master_Log_File = Master_Log_File  -- 读取跟上
但 Seconds_Behind_Master 很大  -- 执行慢

# 1. 基础ping测试
ping -c 10 主库IP
# 正常：< 1ms（同机房），< 10ms（同城）
# 2. 带宽测试
iperf3 -c 主库IP -t 30
# 正常：> 100Mbps
# 3. 丢包率测试
ping -c 100 主库IP | grep packet
# 正常：丢包率 < 0.1%
# 4. TCP连接质量
netstat -s | grep retrans
# 正常：重传率 < 0.01%

# 1. 启用binlog压缩（MySQL 8.0+）
SET GLOBAL binlog_transaction_compression = ON;
SET GLOBAL binlog_transaction_compression_level_zstd = 3;

# 2. 调整TCP参数
echo "net.ipv4.tcp_window_scaling = 1" >> /etc/sysctl.conf
echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
sysctl -p

# 3. 使用专线/内网
# 避免公网传输，使用VPC内网或专线

-- 查看relay log堆积情况
SHOW SLAVE STATUS\G

-- 关键指标
Relay_Log_Space: 536870912  -- relay log总大小（字节）
Relay_Log_File: relay-bin.000123  -- 当前relay log文件
Relay_Log_Pos: 1234567  -- 当前位置

-- 计算堆积量
SELECT 
    (Read_Master_Log_Pos - Exec_Master_Log_Pos) / 1024 / 1024 AS堆积_MB;

# 1. iostat监控
iostat -x 1 10
# 关键指标
# %util: 磁盘利用率（>80%表示瓶颈）
# await: IO等待时间（<10ms正常）
# svctm: 服务时间（<5ms正常）
# 2. fio测试
fio --filename=/var/lib/mysql/test.io --direct=1 --rw=write \
    --bs=16k --size=1G --numjobs=1 --runtime=60 --group_reporting
# 3. 查看relay log写入速度
watch -n 1 'ls -lh /var/lib/mysql/relay-bin.* | tail -5'

-- 1. 调整relay log相关参数
SET GLOBAL relay_log_recovery = ON;  -- 崩溃恢复更快
SET GLOBAL relay_log_purge = ON;     -- 及时清理

-- 2. 优化磁盘IO
[mysqld]
# relay log优化
relay_log_info_repository = TABLE
relay_log_recovery = ON
sync_relay_log = 10000  # 每10000个事件同步一次

# InnoDB优化
innodb_flush_log_at_trx_commit = 2  # 从库可设为2
innodb_flush_method = O_DIRECT
innodb_io_capacity = 2000
innodb_io_capacity_max = 4000

-- 3. 使用更快的存储
# SSD/NVMe替代HDD
# RAID 10配置

-- 1. 查看当前SQL线程状态
SHOW PROCESSLIST;
-- 找到"Slave_SQL_Running_State"字段

-- 2. 开启慢查询日志（从库）
SET GLOBAL slow_query_log = ON;
SET GLOBAL long_query_time = 1;  -- 1秒以上记录
SET GLOBAL log_slow_slave_statements = ON;  -- 记录复制的慢SQL

-- 3. 分析慢查询日志
mysqldumpslow -s t /var/log/mysql/slow.log | head -20

-- 4. 实时监控SQL线程
SELECT * FROM performance_schema.replication_applier_status_by_worker;

-- 1. 查看当前执行的事务
SELECT * FROM information_schema.innodb_trx 
WHERE TIME_TO_SEC(TIMEDIFF(NOW(), trx_started)) > 60;

-- 2. 查看长时间运行的SQL
SELECT 
    id, user, host, db, command, time, state, info
FROM information_schema.processlist 
WHERE command != 'Sleep' AND time > 60
ORDER BY time DESC;

-- 3. 分析binlog中的大事务
mysqlbinlog --base64-output=DECODE-ROWS \
    --start-position=1234567 \
    /var/lib/mysql/mysql-bin.000001 | \
    grep -A 100 "BEGIN" | head -200

-- 1. 查看InnoDB锁等待
SELECT 
    r.trx_id AS waiting_trx_id,
    r.trx_mysql_thread_id AS waiting_thread,
    r.trx_query AS waiting_query,
    b.trx_id AS blocking_trx_id,
    b.trx_mysql_thread_id AS blocking_thread,
    b.trx_query AS blocking_query
FROM information_schema.innodb_lock_waits w
INNER JOIN information_schema.innodb_trx b ON b.trx_id = w.blocking_trx_id
INNER JOIN information_schema.innodb_trx r ON r.trx_id = w.requesting_trx_id;

-- 2. 查看元数据锁（DDL阻塞）
SELECT * FROM performance_schema.metadata_locks 
WHERE OWNER_THREAD_ID != CONNECTION_ID();

-- 1. 启用并行复制（MySQL 5.7+）
SET GLOBAL slave_parallel_type = 'LOGICAL_CLOCK';
SET GLOBAL slave_parallel_workers = 8;  -- 根据CPU核心数设置
SET GLOBAL slave_preserve_commit_order = ON;

-- 2. 优化SQL执行
-- 主库优化：添加索引、拆分大事务、避免DDL高峰
-- 从库优化：调整buffer pool、优化查询缓存

-- 3. 调整复制参数
[mysqld]
# 并行复制
slave_parallel_type = LOGICAL_CLOCK
slave_parallel_workers = 8
slave_preserve_commit_order = ON

# SQL线程优化
slave_transaction_retries = 10
slave_net_timeout = 60

# Buffer优化
innodb_buffer_pool_size = 12G  # 物理内存的70-80%
innodb_log_file_size = 2G

-- 单个事务包含大量操作
BEGIN;
-- 插入10万条记录
INSERT INTO orders SELECT * FROM temp_orders;
COMMIT;

-- 影响：从库必须顺序执行，无法并行

-- 1. 拆分大事务
DELIMITER $$
CREATE PROCEDURE split_large_transaction()
BEGIN
    DECLARE i INT DEFAULT 0;
    WHILE i < 100 DO
        START TRANSACTION;
        INSERT INTO orders 
        SELECT * FROM temp_orders 
        LIMIT 1000 OFFSET i*1000;
        COMMIT;
        SET i = i + 1;
    END WHILE;
END$$
DELIMITER ;

-- 2. 使用批量插入优化
INSERT INTO orders (col1, col2) VALUES 
    (val1, val2),
    (val3, val4),
    ...;

# 使用pt-online-schema-change
pt-online-schema-change \
    --alter "ADD COLUMN new_col INT" \
    --execute \
    D=your_db,t=your_table,h=localhost

# 使用gh-ost
gh-ost \
    --user="user" \
    --password="pass" \
    --host=localhost \
    --database="your_db" \
    --table="your_table" \
    --alter="ADD COLUMN new_col INT" \
    --execute

-- 主从配置对比脚本
SELECT 
    '主库' AS server,
    @@innodb_buffer_pool_size AS buffer_pool,
    @@innodb_log_file_size AS log_file_size,
    @@max_connections AS max_connections
UNION ALL
SELECT 
    '从库',
    @@innodb_buffer_pool_size,
    @@innodb_log_file_size,
    @@max_connections;

-- 1. 复制状态查看
SHOW SLAVE STATUS\G
SHOW MASTER STATUS\G
SHOW PROCESSLIST\G

-- 2. 性能模式监控
SELECT * FROM performance_schema.replication_connection_status;
SELECT * FROM performance_schema.replication_applier_status;
SELECT * FROM performance_schema.replication_applier_status_by_worker;

-- 3. InnoDB状态
SHOW ENGINE INNODB STATUS\G

-- 4. 锁信息
SELECT * FROM information_schema.innodb_locks;
SELECT * FROM information_schema.innodb_lock_waits;

# prometheus.yml
- job_name: 'mysql'
  static_configs:
    - targets: ['主库IP:9104', '从库IP:9104']

# 安装PMM
docker run -d \
  -p 443:443 \
  -v pmm-data:/srv \
  --name pmm-server \
  --restart always \
  percona/pmm-server:2

# 安装客户端
pmm-admin config --server-insecure-tls --server-url=https://admin:admin@pmm-server
pmm-admin add mysql --username=root --password=your_password

#!/bin/bash
# mysql_replication_diagnose.sh

echo "=== MySQL主从延迟诊断报告 ==="
echo "时间: $(date '+%Y-%m-%d %H:%M:%S')"

# 1. 基础信息
mysql -e "SHOW SLAVE STATUS\G" | grep -E "Slave_IO_Running|Slave_SQL_Running|Seconds_Behind_Master|Relay_Master_Log_File|Exec_Master_Log_Pos"

# 2. 延迟计算
DELAY=$(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Seconds_Behind_Master" | awk '{print $2}')
echo "当前延迟: ${DELAY}s"

# 3. 线程状态
echo "--- 线程状态 ---"
mysql -e "SHOW PROCESSLIST\G" | grep -A 5 "Slave"

# 4. 磁盘IO
echo "--- 磁盘IO状态 ---"
iostat -x 1 3 | tail -20

# 5. 网络延迟
echo "--- 网络延迟 ---"
ping -c 5 $(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Master_Host" | awk '{print $2}')

echo "=== 诊断完成 ==="

# my.cnf 生产环境标准配置
[mysqld]
# 基础配置
server-id = 101  # 主库
log-bin = mysql-bin
binlog_format = ROW
binlog_row_image = FULL
# 复制优化
sync_binlog = 1000  # 主库可适当降低
innodb_flush_log_at_trx_commit = 1  # 主库保持1
# 并行复制（从库）
slave_parallel_type = LOGICAL_CLOCK
slave_parallel_workers = 8
slave_preserve_commit_order = ON
# Relay Log优化
relay_log_info_repository = TABLE
relay_log_recovery = ON
sync_relay_log = 10000
# Buffer优化
innodb_buffer_pool_size = 12G
innodb_log_file_size = 2G
innodb_flush_method = O_DIRECT
# 监控
performance_schema = ON
log_slow_slave_statements = ON
long_query_time = 1

# alert_rules.yml
groups:
  - name: mysql_replication
    rules:
      - alert: MySQLReplicationDelay
        expr: mysql_slave_seconds_behind_master > 10
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "MySQL主从延迟过高"
          description: "从库 {{ $labels.instance }} 延迟 {{ $value }} 秒"
      - alert: MySQLReplicationStopped
        expr: mysql_slave_sql_running == 0 or mysql_slave_io_running == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "MySQL复制停止"
          description: "从库 {{ $labels.instance }} 复制线程停止"
      - alert: MySQLRelayLogAccumulation
        expr: rate(mysql_slave_relay_log_pos[5m]) < 10000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "MySQL Relay Log堆积"
          description: "从库 {{ $labels.instance }} Relay Log堆积"

#!/bin/bash
# auto_heal_replication.sh
DELAY_THRESHOLD=60
MAX_RETRY=3
while true; do
    # 获取当前延迟
    DELAY=$(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Seconds_Behind_Master" | awk '{print $2}')
    # 检查延迟是否超标
    if [ "$DELAY" -gt "$DELAY_THRESHOLD" ]; then
        echo "$(date): 检测到延迟 $DELAY 秒，开始自动修复"
        # 检查SQL线程状态
        SQL_RUNNING=$(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Slave_SQL_Running" | awk '{print $2}')
        if [ "$SQL_RUNNING" == "No" ]; then
            echo "SQL线程停止，尝试重启"
            mysql -e "STOP SLAVE; START SLAVE;"
        fi
        # 检查是否有错误
        LAST_ERROR=$(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Last_SQL_Error" | awk '{$1=$2=""; print $0}')
        if [ -n "$LAST_ERROR" ]; then
            echo "检测到错误: $LAST_ERROR"
            # 跳过错误（谨慎使用）
            mysql -e "STOP SLAVE; SET GLOBAL sql_slave_skip_counter = 1; START SLAVE;"
        fi
    fi
    sleep 60
done

发现延迟告警
    ↓
确认延迟真实性（心跳表验证）
    ↓
定位瓶颈层次（IO/SQL/网络）
    ↓
针对性处理
    ├─ 网络问题 → 联系网络团队
    ├─ IO问题 → 优化磁盘、调整参数
    └─ SQL问题 → 优化查询、启用并行
    ↓
验证修复效果
    ↓
记录故障报告
    ↓
优化预防措施

-- 建立性能基线表
CREATE TABLE replication_baseline (
    id INT AUTO_INCREMENT PRIMARY KEY,
    record_time DATETIME,
    avg_delay_seconds DECIMAL(10,2),
    max_delay_seconds DECIMAL(10,2),
    io_thread_status VARCHAR(20),
    sql_thread_status VARCHAR(20),
    relay_log_size_mb DECIMAL(10,2),
    network_latency_ms DECIMAL(10,2),
    INDEX idx_record_time (record_time)
);

-- 定时记录基线数据
DELIMITER $$
CREATE EVENT record_replication_baseline
ON SCHEDULE EVERY 1 HOUR
DO
BEGIN
    INSERT INTO replication_baseline 
    SELECT 
        NOW(),
        AVG(Seconds_Behind_Master),
        MAX(Seconds_Behind_Master),
        Slave_IO_Running,
        Slave_SQL_Running,
        Relay_Log_Space / 1024 / 1024,
        -- 网络延迟需要外部脚本获取
        0
    FROM information_schema.slave_status;
END$$
DELIMITER ;

# 1. 基础状态
mysql -e "SHOW SLAVE STATUS\G" | grep -E "Running|Behind|Position"
# 2. 真实延迟（心跳表）
mysql -e "SELECT TIMESTAMPDIFF(SECOND, ts, NOW()) AS delay FROM heartbeat ORDER BY id DESC LIMIT 1;"
# 3. Relay Log堆积
mysql -e "SHOW SLAVE STATUS\G" | awk '/Read_Master_Log_Pos|Exec_Master_Log_Pos/{print $2}' | awk 'NR==1{a=$1} NR==2{print "堆积: " (a-$1)/1024/1024 " MB"}'
# 4. 线程状态
mysql -e "SHOW PROCESSLIST\G" | grep -A 3 "Slave"
# 5. 网络延迟
ping -c 5 $(mysql -N -e "SHOW SLAVE STATUS\G" | grep Master_Host | awk '{print $2}')
# 6. 磁盘IO
iostat -x 1 5 | grep -A 5 "Device"

# MySQL 8.0 主从复制生产配置
[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock
[mysqld]
# 基础配置
user = mysql
pid-file = /var/run/mysqld/mysqld.pid
socket = /var/run/mysqld/mysqld.sock
datadir = /var/lib/mysql
tmpdir = /tmp
# 网络配置
bind-address = 0.0.0.0
port = 3306
max_connections = 500
# 复制配置（主库）
server-id = 101
log-bin = mysql-bin
binlog_format = ROW
binlog_row_image = FULL
sync_binlog = 1000
binlog_group_commit_sync_delay = 100
binlog_transaction_compression = ON
binlog_transaction_compression_level_zstd = 3
# 复制配置（从库）
# server-id = 102
# relay_log = mysql-relay-bin
# read_only = ON
# super_read_only = ON
# 并行复制（从库）
slave_parallel_type = LOGICAL_CLOCK
slave_parallel_workers = 8
slave_preserve_commit_order = ON
relay_log_info_repository = TABLE
relay_log_recovery = ON
sync_relay_log = 10000
log_slow_slave_statements = ON
# InnoDB配置
innodb_buffer_pool_size = 12G
innodb_log_file_size = 2G
innodb_flush_method = O_DIRECT
innodb_flush_log_at_trx_commit = 1  # 从库可设为2
innodb_io_capacity = 2000
innodb_io_capacity_max = 4000
innodb_file_per_table = ON
# 性能优化
performance_schema = ON
thread_cache_size = 100
table_open_cache = 4000
query_cache_type = 0
query_cache_size = 0
# 监控配置
slow_query_log = ON
slow_query_log_file = /var/log/mysql/slow.log
long_query_time = 1
log_queries_not_using_indexes = ON
# 安全配置
skip_name_resolve = ON
sql_mode = STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION
[mysqldump]
quick
quote-names
max_allowed_packet = 64M
[mysql]
no-auto-rehash
[isamchk]
key_buffer_size = 16M