基于Nginx+Keepalived实现高可用(主从切换)配置指南

sestatus | grep "current mode"
# 输出应为：current mode: permissive

# 安装依赖
sudo dnf groupinstall "Development Tools" -y
sudo dnf install pcre-devel zlib-devel openssl-devel wget tar gcc make -y

# 下载并编译 Nginx（以 1.24.0 为例）
cd /tmp
wget https://nginx.org/download/nginx-1.24.0.tar.gz
tar -zxvf nginx-1.24.0.tar.gz
cd nginx-1.24.0

./configure \
  --prefix=/usr/local/nginx \
  --sbin-path=/usr/local/nginx/sbin/nginx \
  --conf-path=/usr/local/nginx/conf/nginx.conf \
  --pid-path=/usr/local/nginx/logs/nginx.pid \
  --with-http_ssl_module \
  --with-http_v2_module \
  --with-http_realip_module \
  --with-http_stub_status_module \
  --with-stream \
  --with-stream_ssl_module \
  --with-compat

make && sudo make install

/usr/local/nginx/sbin/nginx -V 2>&1 | grep -E "(SSL|HTTP_V2|STREAM)"
# 应输出包含 ssl, http_v2, stream 字样

sudo dnf install keepalived -y
# 启用开机自启（但先不启动）
sudo systemctl enable keepalived

# 全局块
user  nginx;
worker_processes  auto;
worker_rlimit_nofile 65535;
events {
    use epoll;
    worker_connections  65535;
    multi_accept on;
}
# HTTP 块：处理 Web 流量
http {
    include       mime.types;
    default_type  application/octet-stream;
    # 日志格式增强：记录 upstream 响应时间、状态、地址
    log_format main '$remote_addr - $remote_user [$time_local] '
                      '"$request" $status $body_bytes_sent '
                      '"$http_referer" "$http_user_agent" '
                      'rt=$request_time uct="$upstream_connect_time" '
                      'uht="$upstream_header_time" urt="$upstream_response_time" '
                      'upstream_addr=$upstream_addr';
    access_log  /usr/local/nginx/logs/access.log  main;
    error_log   /usr/local/nginx/logs/error.log  warn;
    sendfile        on;
    tcp_nopush      on;
    tcp_nodelay     on;
    keepalive_timeout  65;
    types_hash_max_size 2048;
    # 启用健康检查状态页（供 Keepalived 脚本调用）
    server {
        listen 127.0.0.1:8081;
        location /healthz {
            return 200 "OK\n";
            add_header Content-Type text/plain;
        }
        # 检查 upstream 是否可达（curl 后端 Java 服务）
        location /backend-health {
            proxy_pass http://192.168.56.20:8080/actuator/health;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_read_timeout 3;
            proxy_connect_timeout 3;
        }
    }
    # 主业务 Server 块
    server {
        listen       80;
        server_name  _;
        # 静态资源缓存
        location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg)$ {
            expires 1y;
            add_header Cache-Control "public, immutable";
        }
        # 反向代理到 Java 后端
        location / {
            proxy_pass http://192.168.56.20:8080;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            # 超时设置（防止长连接阻塞）
            proxy_connect_timeout 5;
            proxy_send_timeout 30;
            proxy_read_timeout 30;
            # 传递真实客户端 IP（配合 real_ip 模块）
            set_real_ip_from 192.168.56.0/24;
            real_ip_header X-Forwarded-For;
        }
    }
}

# 创建 nginx 用户（若不存在）
sudo useradd -r -s /sbin/nologin nginx

# 创建日志目录
sudo mkdir -p /usr/local/nginx/logs

# 启动 Nginx
sudo /usr/local/nginx/sbin/nginx

# 验证配置语法 & 端口监听
sudo /usr/local/nginx/sbin/nginx -t
sudo ss -tlnp | grep ':80\|:8081'

# 测试健康接口
curl http://127.0.0.1:8081/healthz    # 应返回 OK
curl http://127.0.0.1:8081/backend-health  # 应返回 {"status":"UP"}

! Configuration File for keepalived
global_defs {
    # 路由器标识（每台机器唯一，用于日志区分）
    router_id NGINX_MASTER_01
    # 通知邮箱（可选，生产环境建议对接企业微信/钉钉）
    # notification_email {
    #    admin@example.com
    # }
    # smtp_server 127.0.0.1
    # smtp_connect_timeout 30
}
# 自定义健康检查脚本（关键！）
vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2                    # 每2秒执行一次
    weight 2                      # 成功时权重+2，失败时-2
    fall 2                        # 连续2次失败才判定为 down
    rise 2                        # 连续2次成功才判定为 up
}
# VRRP 实例定义
vrrp_instance VI_1 {
    state MASTER                    # 当前节点角色：MASTER
    interface eth0                  # VIP 绑定的业务网卡（非心跳网卡！）
    virtual_router_id 51            # VRID，主备必须一致（1-255）
    priority 100                    # 优先级，越高越可能成为 Master（Backup 设为 90）
    advert_int 1                    # VRRP 通告间隔（秒）
    authentication {
        auth_type PASS
        auth_pass 1111              # 认证密码（主备必须一致）
    }
    # 虚拟 IP（VIP）配置
    virtual_ipaddress {
        192.168.56.100/24 dev eth0 label eth0:1  # VIP + 子网掩码 + 绑定网卡别名
    }
    # 跟踪脚本：当 chk_nginx 失败时，降低本节点优先级，触发切换
    track_script {
        chk_nginx
    }
    # 防脑裂：当检测到与 Backup 心跳中断时，执行强制降级
    # （需配合独立心跳网卡 eth1）
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

! Configuration File for keepalived
global_defs {
    router_id NGINX_BACKUP_01
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight 2
    fall 2
    rise 2
}
vrrp_instance VI_1 {
    state BACKUP                    # 角色为 BACKUP
    interface eth0
    virtual_router_id 51            # 必须与 Master 一致！
    priority 90                       # 低于 Master，确保初始为 Backup
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111              # 密码必须一致！
    }
    virtual_ipaddress {
        192.168.56.100/24 dev eth0 label eth0:1
    }
    track_script {
        chk_nginx
    }
    # 关键：启用非抢占模式！
    # 一旦 Backup 升为 Master，即使原 Master 恢复也不抢回 VIP
    nopreempt
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

#!/bin/bash
# /etc/keepalived/check_nginx.sh
# 功能：双重健康检查
# 1. 检查 nginx master 进程是否存在
# 2. 检查 nginx 能否通过 /backend-health 接口连通 Java 后端

NGINX_PID_FILE="/usr/local/nginx/logs/nginx.pid"
NGINX_HEALTH_URL="http://127.0.0.1:8081/backend-health"
TIMEOUT=3

# 检查 nginx 进程
if ! kill -0 $(cat $NGINX_PID_FILE 2>/dev/null) 2>/dev/null; then
    echo "[ERROR] nginx process not running"
    exit 1
fi

# 检查 nginx 健康接口（端到端）
if ! curl -s --max-time $TIMEOUT -f $NGINX_HEALTH_URL >/dev/null 2>&1; then
    echo "[ERROR] nginx /backend-health check failed"
    exit 1
fi

# 检查 nginx 自身健康（快速兜底）
if ! curl -s --max-time $TIMEOUT -f http://127.0.0.1:8081/healthz >/dev/null 2>&1; then
    echo "[ERROR] nginx /healthz check failed"
    exit 1
fi

echo "[OK] All health checks passed"
exit 0

sudo chmod +x /etc/keepalived/check_nginx.sh
sudo chown root:root /etc/keepalived/check_nginx.sh

#!/bin/bash
# /etc/keepalived/notify.sh
# 参数：$1 = master|backup|fault

ROLE=$1
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
VIP="192.168.56.100"
NODE_IP=$(hostname -I | awk '{print $1}')

LOG_FILE="/var/log/keepalived-state.log"

echo "[$TIMESTAMP] KEEPALIVED $ROLE on $(hostname) ($NODE_IP). VIP: $VIP" >> $LOG_FILE

case "$ROLE" in
    "master")
        echo "[$TIMESTAMP] $(hostname) is now MASTER. Enabling VIP..." >> $LOG_FILE
        # 可在此处添加：启动特定服务、更新 DNS、发送企业微信消息等
        ;;
    "backup")
        echo "[$TIMESTAMP] $(hostname) is now BACKUP. VIP released." >> $LOG_FILE
        ;;
    "fault")
        echo "[$TIMESTAMP] $(hostname) entered FAULT state! Check network & nginx." >> $LOG_FILE
        ;;
esac

sudo tail -f /var/log/keepalived-state.log

# 编辑 /etc/sysctl.conf
echo "
# Keepalived VIP ARP 优化
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.eth0.arp_ignore = 1
net.ipv4.conf.eth0.arp_announce = 2
" | sudo tee -a /etc/sysctl.conf

# 生效配置
sudo sysctl -p

<dependencies>
    <!-- Web -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <!-- Actuator（健康检查核心） -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>
    <!-- 可选：暴露所有端点（生产环境请按需开放） -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-devtools</artifactId>
        <scope>runtime</scope>
        <optional>true</optional>
    </dependency>
</dependencies>

management:
  endpoints:
    web:
      exposure:
        include: health,info,metrics,prometheus  # 至少暴露 health
  endpoint:
    health:
      show-details: when_authorized  # 生产环境建议设为 never 或 when_authorized
      probes:
        enabled: true
  server:
    port: 8081  # Actuator 独立端口（与业务端口 8080 分离）
# 业务服务器配置
server:
  port: 8080
  address: 0.0.0.0
spring:
  application:
    name: demo-java-service

package com.example.demo.health;

import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;

import java.util.concurrent.atomic.AtomicBoolean;

@Component
public class DatabaseHealthIndicator implements HealthIndicator {

    private final AtomicBoolean dbAvailable = new AtomicBoolean(true);

    @Override
    public Health health() {
        if (dbAvailable.get()) {
            return Health.up()
                    .withDetail("database", "connected")
                    .withDetail("version", "PostgreSQL 14.5")
                    .build();
        } else {
            return Health.down()
                    .withDetail("error", "Database connection failed")
                    .build();
        }
    }
}

# 打包并运行（假设 jar 名为 demo.jar）
java -jar demo.jar

# 验证健康接口（应在 Nginx 的 /backend-health 中被调用）
curl http://192.168.56.20:8081/actuator/health
# 返回示例：
# {"status":"UP","components":{"diskSpace":{"status":"UP","details":{"total":53660876800,"free":42847272960,"threshold":10485760}},"ping":{"status":"UP"}}}

# 在 Node A（Master）上执行
sudo pkill -f "nginx: master"

# 观察日志
sudo tail -f /var/log/keepalived-state.log
# 应看到：... entered FAULT state! ... 然后 ... is now BACKUP.

# 同时在 Node B（Backup）日志中应看到：
# ... is now MASTER. Enabling VIP...

# 验证 VIP 是否已漂移到 Node B
ip addr show eth0 | grep 192.168.56.100
# Node B 应输出：inet 192.168.56.100/24 scope global secondary eth0:1

# 从客户端（如另一台机器）curl VIP
curl http://192.168.56.100
# 应正常返回 Java 应用首页，毫秒级无感切换！ ✅

# 在 Java 服务节点（192.168.56.20）上停止应用
pkill -f "demo.jar"

# 等待 4 秒（2次失败 × 2秒间隔）
# 此时 Keepalived 会因 /backend-health 失败而降权

# 查看 Node A 的优先级变化
sudo ipvsadm -Ln | grep "192.168.56.100"
# 或查看 keepalived 进程日志：
sudo journalctl -u keepalived -n 20 --no-pager

# 预期：Node A 优先级降至 98（100-2），Node B 仍为 90，但若 Node A 降为 98 < Node B 90？不会切换！
# 因此，我们需让 Node B 优先级更高（如设为 95），或增加 weight（如设为 5）

# ✅ 正确做法：在 chk_nginx.sh 中，失败时 exit 1，Keepalived 会立即触发 vrrp_script 失败逻辑，
# 并结合 priority + weight 实现平滑降级。

# /etc/rsyslog.d/keepalived.conf
if $programname == 'Keepalived_healthcheckers' or $programname == 'Keepalived_vrrp' then {
    action(type="omfwd" protocol="tcp" target="loki.example.com" port="3100" template="RSYSLOG_SyslogProtocol23Format")
    stop
}

server {
    listen 443 ssl http2;
    server_name example.com;
    ssl_certificate /etc/ssl/certs/fullchain.pem;
    ssl_certificate_key /etc/ssl/private/privkey.pem;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256;
    location / {
        proxy_pass http://192.168.56.20:8080;
        # ... 其他 proxy_* 设置
    }
}

# 使用 nginx-upstream-check-module（需重新编译）
upstream backend_java {
    server 192.168.56.20:8080 max_fails=3 fail_timeout=30s;
    server 192.168.56.21:8080 max_fails=3 fail_timeout=30s;
    check interval=3 rise=2 fall=5 timeout=10 type=http;
    check_http_send "HEAD /actuator/health HTTP/1.0\r\n\r\n";
    check_http_expect_alive http_2xx http_3xx;
}

# 实时跟踪 Keepalived 状态
sudo journalctl -u keepalived -f

# 查看 VRRP 详细状态
sudo ipvsadm -Ln

# 抓包分析 VRRP 心跳（在 eth1 上）
sudo tcpdump -i eth1 -n vrrp -c 10