Linux下Docker Compose里运行Redis内存数据库故障诊断脚本

chmod +x redis_compose_diag.sh

sudo redis_compose_diag.sh

sudo ./redis_compose_diag.sh myapp_redis_1

#!/bin/bash
# =============================================================================
# Redis 意外中断系统化排查脚本 (Docker Compose 专属增强版)
# 适配：docker-compose 部署Redis容器、分段可独立执行、缓存可复制容器ID/挂载路径
# 哲学诊断逻辑：本体论(容器/进程/宿主机分层)→奥卡姆剃刀(优先OOM/磁盘故障)→整体论(容器+宿主机联动)
# 运行权限：root / sudo （读取dmesg、系统内核参数、docker完整信息）
# 使用方法：
#  1. 完整执行：sudo bash redis_compose_diag.sh [容器名称/容器ID]
#  2. 分段排查：复制带【单段独立执行】标记区块，直接粘贴终端单独运行
# 缓存文件：/tmp/redis_diag_xxxx/redis_copy_cache.txt 存放所有可复制ID、路径、端口
# =============================================================================
# 全局颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
WHITE='\033[1;37m'
NC='\033[0m'
# ======================【单段独立执行】全局工具函数 + 初始化缓存目录 ======================
# 分割线打印
divider() {
    echo -e "\n${CYAN}============================================================${NC}"
    echo -e "${CYAN}  $1${NC}"
    echo -e "${CYAN}============================================================${NC}\n"
}
# 命令执行并保存日志
collect_log() {
    local cmd="$1"
    local outfile="$2"
    echo "执行命令：$cmd" >> "$DIAG_SUMMARY"
    eval "$cmd" > "$outfile" 2>&1
}
# 打印高亮可复制参数，写入缓存文件
print_copy_info() {
    local label="$1"
    local value="$2"
    echo -e "${WHITE}【可复制 $label 】${RED}$value${NC}"
    echo "$label=$value" >> "$COPY_CACHE"
}
# 输出预填充变量、一键复制执行的排查命令
print_quick_cmd() {
    echo -e "${CYAN}[一键复制执行命令]${NC} $1"
    echo "快捷命令：$1" >> "$COPY_CACHE"
}
# 初始化全局缓存变量（全程复用）
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
DIAG_DIR="/tmp/redis_diag_${TIMESTAMP}"
DIAG_SUMMARY="${DIAG_DIR}/diagnose_summary.log"
COPY_CACHE="${DIAG_DIR}/redis_copy_cache.txt"
mkdir -p "$DIAG_DIR"
> "$DIAG_SUMMARY"
> "$COPY_CACHE"
echo "诊断采集时间：$(date)" >> "$DIAG_SUMMARY"
# ==========================================================================================
# ======================【单段独立执行】入参处理、自动匹配Redis容器 ======================
divider "全局参数初始化 & 自动识别Redis容器"
# 入参容器名/ID
CONTAINER="$1"
if [ -z "$CONTAINER" ]; then
    # 自动筛选全部包含redis的容器（运行+停止）
    CONTAINER=$(docker ps -a --format '{{.Names}}' | grep -i redis | head -1)
    if [ -z "$CONTAINER" ]; then
        echo -e "${RED}[致命错误] 未找到任何redis容器，请手动传入容器名/ID执行脚本${NC}"
        echo "示例：sudo bash $0 redis_1"
        exit 1
    fi
    echo -e "${YELLOW}[自动匹配容器] ${CONTAINER}${NC}"
fi
# 全局核心缓存变量（一次性提取，全程复用）
CONTAINER_ID=$(docker inspect -f '{{.Id}}' "$CONTAINER" 2>/dev/null | cut -c1-12)
COMPOSE_PROJECT=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' "$CONTAINER")
COMPOSE_SERVICE=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}}' "$CONTAINER")
RUNNING_STATUS=$(docker inspect -f '{{.State.Running}}' "$CONTAINER")
EXIT_CODE=$(docker inspect -f '{{.State.ExitCode}}' "$CONTAINER")
OOM_KILLED_FLAG=$(docker inspect -f '{{.State.OOMKilled}}' "$CONTAINER")
# 提取宿主机数据挂载目录
DATA_MOUNT=$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Source}}{{end}}{{end}}' "$CONTAINER")
CONF_MOUNT=$(docker inspect -f '{{range .Mounts}}{{if match .Destination "/etc/redis"}}{{.Source}}{{end}}{{end}}' "$CONTAINER")
# 全部关键信息高亮打印，存入缓存文件
print_copy_info "容器全名" "$CONTAINER"
print_copy_info "容器短ID" "$CONTAINER_ID"
print_copy_info "Compose项目名" "$COMPOSE_PROJECT"
print_copy_info "Compose服务名" "$COMPOSE_SERVICE"
print_copy_info "容器运行状态" "$RUNNING_STATUS"
print_copy_info "容器退出码" "$EXIT_CODE"
print_copy_info "容器OOM杀死标记" "$OOM_KILLED_FLAG"
print_copy_info "宿主机Redis数据卷目录" "$DATA_MOUNT"
print_copy_info "宿主机Redis配置挂载目录" "$CONF_MOUNT"
print_copy_info "诊断文件存放目录" "$DIAG_DIR"
# 快捷查看全部缓存参数
print_quick_cmd "cat ${COPY_CACHE}"
# ==========================================================================================
# ======================【单段独立执行】1.容器基础状态 + 退出码解析 ======================
divider "1. 容器基础状态、退出码、资源限制"
# 打印容器完整基础信息
echo "容器完整清单："
docker ps -a --filter "name=${CONTAINER}" --format "table {{.Names}}\t{{.ID}}\t{{.Status}}\t{{.Image}}\t{{.CreatedAt}}"
echo -e "\n容器退出码：${EXIT_CODE}"
# 退出码故障解析
case $EXIT_CODE in
    0)   echo -e "${GREEN}正常主动退出 (0)${NC}" ;;
    137) echo -e "${RED}SIGKILL 强制杀死，大概率OOM内存溢出(137)${NC}" ;;
    139) echo -e "${RED}段错误SIGSEGV，Redis程序崩溃(139)${NC}" ;;
    143) echo -e "${YELLOW}SIGTERM正常优雅终止(143)${NC}" ;;
    255) echo -e "${RED}容器启动加载配置失败(255)${NC}" ;;
    *)   echo -e "${YELLOW}未知异常退出码，结合日志进一步排查${NC}" ;;
esac
# 打印容器内存/CPU资源限制
echo -e "\n容器内存限制："
MEM_LIMIT=$(docker inspect -f '{{.HostConfig.Memory}}' "$CONTAINER")
if [ "$MEM_LIMIT" -eq 0 ]; then
    echo "未设置容器内存限制"
else
    MEM_MB=$((MEM_LIMIT / 1024 / 1024))
    echo "限制内存：${MEM_MB} MB"
fi
echo -e "\n容器CPU限制："
CPU_QUOTA=$(docker inspect -f '{{.HostConfig.CpuQuota}}' "$CONTAINER")
CPU_PERIOD=$(docker inspect -f '{{.HostConfig.CpuPeriod}}' "$CONTAINER")
CPU_BIND=$(docker inspect -f '{{.HostConfig.CpusetCpus}}' "$CONTAINER")
if [ "$CPU_QUOTA" -eq 0 ] && [ -z "$CPU_BIND" ]; then
    echo "无CPU配额限制"
else
    [ -n "$CPU_BIND" ] && echo "CPU核心绑定：$CPU_BIND"
    [ "$CPU_QUOTA" -ne 0 ] && echo "CPU配额：$((CPU_QUOTA / CPU_PERIOD)) 核"
fi
# 快捷排查命令
print_quick_cmd "docker inspect ${CONTAINER} > ${DIAG_DIR}/container_full_info.json"
print_quick_cmd "cd ./${COMPOSE_PROJECT} && cat docker-compose.yml | grep -A15 ${COMPOSE_SERVICE}"
collect_log "docker inspect ${CONTAINER}" "${DIAG_DIR}/container_full_info.json"
# ==========================================================================================
# ======================【单段独立执行】2.容器日志采集、崩溃关键字过滤 ======================
divider "2. Redis容器运行日志（最近200行+全量保存）"
echo "===== 最近200行实时日志 ====="
docker logs --tail 200 "$CONTAINER" 2>&1
# 导出完整日志文件
collect_log "docker logs ${CONTAINER}" "${DIAG_DIR}/redis_container_full.log"
echo -e "\n${YELLOW}日志关键故障关键字过滤结果${NC}"
grep -i "oom\|fork cannot allocate\|misconf\|aof error\|rdb fail\|io error\|permission denied\|crash\|exit" "${DIAG_DIR}/redis_container_full.log"
# 日志故障解读
cat <<LOG_TIP
${YELLOW}日志故障对应根因：${NC}
1. fork: Cannot allocate memory → 宿主机vm.overcommit_memory !=1 / 内存不足
2. MISCONF Redis save snapshots → 磁盘满/数据卷只读/权限不足
3. AOF error / I/O error → 挂载目录磁盘损坏、inode耗尽
4. out of memory → Redis内存打满触发淘汰，容器内存限制过小
LOG_TIP
print_quick_cmd "less ${DIAG_DIR}/redis_container_full.log"
print_quick_cmd "docker logs -f ${CONTAINER} # 实时跟踪日志"
# ==========================================================================================
# ======================【单段独立执行】3.OOM双层校验（容器标记+宿主机dmesg） ======================
divider "3. OOM双重校验：容器OOM标记 + 宿主机内核OOM日志"
echo "容器内置OOMKilled标记：${OOM_KILLED_FLAG}"
if [ "$OOM_KILLED_FLAG" = "true" ]; then
    echo -e "${RED}严重警告：Docker容器内存超限，被内核直接杀死！${NC}"
fi
# 宿主机dmesg全局OOM记录
echo -e "\n宿主机dmesg近20条OOM记录："
collect_log "dmesg | grep -i 'oom\|out of memory\|killed process' | tail -20" "${DIAG_DIR}/host_oom_dmesg.log"
cat "${DIAG_DIR}/host_oom_dmesg.log"
# 系统messages日志OOM检索
if [ -r /var/log/messages ]; then
    echo -e "\n/var/log/messages OOM记录："
    collect_log "grep -i oom-killer /var/log/messages | tail -10" "${DIAG_DIR}/syslog_oom.log"
    cat "${DIAG_DIR}/syslog_oom.log"
fi
# 宿主机内存全局快照
echo -e "\n宿主机当前内存占用："
collect_log "free -h" "${DIAG_DIR}/host_memory.log"
cat "${DIAG_DIR}/host_memory.log"
# 全部容器内存占用排行
echo -e "\n全部容器内存占用TOP5："
collect_log "docker stats --no-stream --format 'table {{.Names}}\t{{.MemUsage}}\t{{.MemPerc}}'" "${DIAG_DIR}/docker_mem_rank.log"
cat "${DIAG_DIR}/docker_mem_rank.log"
print_quick_cmd "dmesg -A10 'Killed process' # 查看被杀进程完整上下文"
print_quick_cmd "watch -n1 free -h # 实时监控宿主机内存波动"
# ==========================================================================================
# ======================【单段独立执行】4.宿主机磁盘、inode、挂载权限检查 ======================
divider "4. 宿主机磁盘、inode、Redis数据卷权限校验"
echo "全局磁盘分区使用率："
collect_log "df -h" "${DIAG_DIR}/host_disk.log"
cat "${DIAG_DIR}/host_disk.log"
echo -e "\n全局inode占用："
collect_log "df -i" "${DIAG_DIR}/host_inode.log"
cat "${DIAG_DIR}/host_inode.log"
# 专项检查Redis数据卷
if [ -n "$DATA_MOUNT" ] && [ -d "$DATA_MOUNT" ]; then
    echo -e "\nRedis数据卷目录[$DATA_MOUNT]详情："
    collect_log "ls -lh $DATA_MOUNT" "${DIAG_DIR}/data_volume_file.log"
    collect_log "du -sh $DATA_MOUNT" "${DIAG_DIR}/data_volume_size.log"
    cat "${DIAG_DIR}/data_volume_file.log"
    cat "${DIAG_DIR}/data_volume_size.log"
    # 校验RDB/AOF文件完整性
    print_quick_cmd "docker exec -it ${CONTAINER} redis-check-rdb /data/dump.rdb"
    print_quick_cmd "docker exec -it ${CONTAINER} redis-check-aof /data/appendonly.aof"
fi
# 检查文件系统是否只读
echo -e "\n检测只读挂载分区："
collect_log "mount | grep -E ',ro|ro,' | grep -v tmpfs,sysfs,proc" "${DIAG_DIR}/ro_mount.log"
cat "${DIAG_DIR}/ro_mount.log"
print_quick_cmd "touch ${DATA_MOUNT}/test_write.tmp && rm -f ${DATA_MOUNT}/test_write.tmp # 测试数据卷写入权限"
# ==========================================================================================
# ======================【单段独立执行】5.运行中容器Redis内部指标采集 ======================
divider "5. Redis内部运行指标（仅容器running状态可采集）"
if [ "$RUNNING_STATUS" = "true" ]; then
    echo -e "${GREEN}容器正常运行，采集Redis全量监控指标${NC}"
    REDIS_EXEC="docker exec ${CONTAINER} redis-cli"
    # 采集核心指标
    collect_log "$REDIS_EXEC INFO ALL" "${DIAG_DIR}/redis_info_all.txt"
    collect_log "$REDIS_EXEC CONFIG GET maxmemory save appendonly dir" "${DIAG_DIR}/redis_core_config.txt"
    collect_log "$REDIS_EXEC SLOWLOG GET 30" "${DIAG_DIR}/redis_slowlog.txt"
    collect_log "$REDIS_EXEC CLIENT LIST" "${DIAG_DIR}/redis_client_list.txt"
    # 打印核心故障指标摘要
    echo -e "\n关键运行指标摘要："
    grep -E 'connected_clients|evicted_keys|used_memory_rss_human|rdb_last_bgsave_status|aof_last_write_status' "${DIAG_DIR}/redis_info_all.txt"
    # 端口映射与连通测试
    echo -e "\n容器端口映射："
    docker port "$CONTAINER"
    REDIS_LOCAL_PORT=$(docker port "$CONTAINER" 6379/tcp | awk -F: '{print $2}')
    if [ -n "$REDIS_LOCAL_PORT" ]; then
        print_copy_info "宿主机Redis映射端口" "$REDIS_LOCAL_PORT"
        echo -e "\n本地PING连通测试："
        timeout 2 bash -c "echo PING | nc 127.0.0.1 $REDIS_LOCAL_PORT" && echo -e "${GREEN}Redis正常响应${NC}" || echo -e "${RED}Redis主线程阻塞无响应${NC}"
    fi
    # 深度调试阻塞命令
    print_quick_cmd "docker exec -it ${CONTAINER} strace -p 1 -s 200 # 追踪Redis主线程阻塞"
    print_quick_cmd "docker exec -it ${CONTAINER} gdb -p 1 -batch -ex 'thread apply all bt' # 打印线程堆栈死锁排查"
else
    echo -e "${YELLOW}容器已停止，无法执行redis-cli，从宿主机挂载目录读取配置${NC}"
    if [ -n "$CONF_MOUNT" ] && [ -f "${CONF_MOUNT}/redis.conf" ]; then
        echo -e "\n提取宿主机挂载Redis配置："
        collect_log "grep -v '^#' ${CONF_MOUNT}/redis.conf | grep -E 'maxmemory|appendonly|save|dir'" "${DIAG_DIR}/local_redis_config.log"
        cat "${DIAG_DIR}/local_redis_config.log"
    fi
fi
# ==========================================================================================
# ======================【单段独立执行】6.宿主机内核关键参数校验 ======================
divider "6. 宿主机内核高危参数校验（Redis通用隐患）"
# vm overcommit_memory
echo "vm.overcommit_memory 参数："
collect_log "cat /proc/sys/vm/overcommit_memory" "${DIAG_DIR}/overcommit.log"
OVERCOMMIT=$(cat /proc/sys/vm/overcommit_memory)
if [ "$OVERCOMMIT" != "1" ]; then
    echo -e "${RED}警告：推荐设置vm.overcommit_memory=1，当前值${OVERCOMMIT}，极易fork持久化失败${NC}"
    print_quick_cmd "sysctl -w vm.overcommit_memory=1 # 临时修复"
fi
# THP透明大页
echo -e "\n透明大页THP状态："
if [ -f /sys/kernel/mm/transparent_hugepage/enabled ]; then
    collect_log "cat /sys/kernel/mm/transparent_hugepage/enabled" "${DIAG_DIR}/thp.log"
    THP_STATUS=$(cat /sys/kernel/mm/transparent_hugepage/enabled)
    if [[ "$THP_STATUS" == *"[always]"* ]]; then
        echo -e "${RED}警告：THP=always，会造成Redis内存碎片、主线程卡顿${NC}"
        print_quick_cmd "echo never > /sys/kernel/mm/transparent_hugepage/enabled # 临时关闭THP"
    fi
fi
# 系统文件句柄上限
echo -e "\n系统全局文件句柄上限："
collect_log "sysctl fs.file-max" "${DIAG_DIR}/file_max.log"
cat "${DIAG_DIR}/file_max.log"
# 系统时间同步检查
echo -e "\n系统时间状态："
timedatectl status 2>/dev/null || date
collect_log "timedatectl status 2>/dev/null || date" "${DIAG_DIR}/host_time.log"
print_quick_cmd "sysctl -p /etc/sysctl.conf # 永久生效内核参数"
# ==========================================================================================
# ======================【单段独立执行】7.故障总结、归档打包、恢复指引 ======================
divider "7. 故障根因优先级总结 & 完整诊断包打包"
cat <<SUMMARY
${GREEN}故障排查优先级（从高到低）${NC}
1. 容器退出码137 + OOMKilled=true → 容器内存限制过小，调大mem_limit
2. dmesg存在OOM记录 → 宿主机整体内存不足，扩容/迁移其他容器
3. 日志fork Cannot allocate memory → 修改vm.overcommit_memory=1、关闭THP
4. 磁盘100% / inode耗尽 / 只读分区 → 清理AOF/RDB、扩容存储
5. MISCONF持久化报错 → 数据卷权限、目录写入异常
6. 慢查询堆积、客户端连接爆满 → Redis主线程阻塞，优化大key/长命令
${YELLOW}快速恢复操作指引${NC}
1. OOM场景：修改docker-compose.yml mem_limit，调整内核overcommit+THP后重启compose
2. 磁盘满：清理过期dump.rdb、appendonly.aof，配置auto-aof-rewrite
3. 容器启动失败：检查挂载目录权限、redis.conf语法、端口占用
SUMMARY
# 打包全部诊断文件
tar -czf "${DIAG_DIR}.tar.gz" -C /tmp "redis_diag_${TIMESTAMP}" 2>/dev/null
echo -e "\n${GREEN}✅ 全部诊断信息打包完成${NC}"
print_copy_info "完整故障压缩包路径" "${DIAG_DIR}.tar.gz"
print_quick_cmd "ls ${DIAG_DIR} # 查看所有采集日志、指标文件"
print_quick_cmd "cat ${COPY_CACHE} # 快速复制容器ID、挂载目录、端口等参数"
exit 0