Nginx生产环境无缝升级与回滚方案

# 创建版本化安装目录
sudo mkdir -p /usr/local/nginx-v1.26.0/{sbin,conf,logs,html}

# 下载源码（官方 HTTPS）
wget https://nginx.org/download/nginx-1.26.0.tar.gz
tar -xzf nginx-1.26.0.tar.gz
cd nginx-1.26.0

# 关键：指定 --prefix 为版本化路径，--sbin-path 显式指向 sbin 子目录
./configure \
  --prefix=/usr/local/nginx-v1.26.0 \
  --sbin-path=/usr/local/nginx-v1.26.0/sbin/nginx \
  --conf-path=/usr/local/nginx-v1.26.0/conf/nginx.conf \
  --error-log-path=/usr/local/nginx-v1.26.0/logs/error.log \
  --http-log-path=/usr/local/nginx-v1.26.0/logs/access.log \
  --pid-path=/usr/local/nginx-v1.26.0/logs/nginx.pid \
  --lock-path=/usr/local/nginx-v1.26.0/logs/nginx.lock \
  --with-http_ssl_module \
  --with-http_v2_module \
  --with-http_realip_module \
  --with-http_stub_status_module \
  --with-pcre \
  --with-zlib

make -j$(nproc)
sudo make install

# 检查新 binary 是否可执行且版本正确
/usr/local/nginx-v1.26.0/sbin/nginx -v  # 输出: nginx version: nginx/1.26.0

# 检查是否能加载当前配置（dry-run）
sudo /usr/local/nginx-v1.26.0/sbin/nginx -c /etc/nginx/nginx.conf -t
# 必须输出: "syntax is ok" and "test is successful"

# 创建版本化配置目录
sudo mkdir -p /etc/nginx/conf.d/v1.26.0/

# 复制主配置（仅修改 include 路径）
sudo cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.v1.26.0
sudo sed -i 's|/etc/nginx/conf.d/|/etc/nginx/conf.d/v1.26.0/|g' /etc/nginx/nginx.conf.v1.26.0

# 复制所有站点配置到新目录（保持文件名一致，便于 diff）
sudo cp /etc/nginx/conf.d/*.conf /etc/nginx/conf.d/v1.26.0/

# 【关键】为新配置添加健康检查端点（供 Java 应用调用）
echo "
# Health check endpoint for Java service discovery
server {
    listen 8081;
    server_name _;
    location /healthz {
        return 200 '{
            \"status\": \"UP\",
            \"nginx_version\": \"1.26.0\",
            \"build_time\": \"$(date -Iseconds)\",
            \"config_hash\": \"$(sha256sum /etc/nginx/conf.d/v1.26.0/*.conf | sha256sum | cut -d' ' -f1)\"
        }';
        add_header Content-Type application/json;
    }
}" | sudo tee /etc/nginx/conf.d/v1.26.0/health.conf > /dev/null

#!/bin/bash
# save as: /usr/local/bin/nginx-upgrade-v1.26.0.sh
set -e

OLD_PID="/var/run/nginx.pid"
NEW_PID="/var/run/nginx-v1.26.0.pid"
NEW_BINARY="/usr/local/nginx-v1.26.0/sbin/nginx"
NEW_CONF="/etc/nginx/nginx.conf.v1.26.0"

echo "[INFO] Starting Nginx v1.26.0 upgrade..."

# Step 1: Verify new binary & config
if ! $NEW_BINARY -c "$NEW_CONF" -t; then
  echo "[ERROR] New Nginx config test failed!" >&2
  exit 1
fi

# Step 2: Send USR2 to OLD master → starts NEW master
echo "[INFO] Sending USR2 to old master (PID: $(cat $OLD_PID))"
sudo kill -USR2 $(cat $OLD_PID)

# Wait for new master to start (max 10s)
for i in {1..10}; do
  if [ -f "$NEW_PID" ] && ps -p $(cat $NEW_PID) > /dev/null 2>&1; then
    echo "[INFO] New master started successfully (PID: $(cat $NEW_PID))"
    break
  fi
  sleep 1
done
if [ ! -f "$NEW_PID" ]; then
  echo "[ERROR] New master failed to start!" >&2
  exit 1
fi

# Step 3: Send WINCH to OLD master → gracefully shutdown old workers
echo "[INFO] Sending WINCH to old master to shutdown old workers"
sudo kill -WINCH $(cat $OLD_PID)

# Wait for old workers to exit (max 30s, or until no nginx worker under old master)
OLD_MASTER_PID=$(cat $OLD_PID)
for i in {1..30}; do
  if ! pgrep -P "$OLD_MASTER_PID" nginx > /dev/null; then
    echo "[INFO] All old workers gracefully exited"
    break
  fi
  sleep 1
done

# Step 4: Send QUIT to OLD master → exit old master process
echo "[INFO] Sending QUIT to old master to terminate"
sudo kill -QUIT "$OLD_MASTER_PID"

# Step 5: Update symlink to new binary (for future systemctl usage)
sudo ln -sf /usr/local/nginx-v1.26.0/sbin/nginx /usr/sbin/nginx

echo "[SUCCESS] Nginx upgraded to v1.26.0 with zero downtime!"

sudo chmod +x /usr/local/bin/nginx-upgrade-v1.26.0.sh
sudo /usr/local/bin/nginx-upgrade-v1.26.0.sh

# 应只看到新 master 及其 worker，无旧 master 进程
ps aux | grep nginx | grep -v grep

# 检查监听端口归属（新 master 应持有 :80/:443）
sudo ss -tlnp | grep ':80\|:443'

# 检查新 PID 文件内容
cat /var/run/nginx-v1.26.0.pid  # 应为新 master PID

// NginxHealthChecker.java
@Component
@Slf4j
public class NginxHealthChecker {

    private final RestTemplate restTemplate;
    private final String nginxHealthUrl = "http://localhost:8081/healthz";

    public NginxHealthChecker(RestTemplateBuilder builder) {
        this.restTemplate = builder
                .setConnectTimeout(Duration.ofSeconds(2))
                .setReadTimeout(Duration.ofSeconds(2))
                .build();
    }

    /**
     * 检查 Nginx 实例健康状态，返回版本信息
     */
    public Optional<NginxVersionInfo> checkNginxHealth() {
        try {
            ResponseEntity<String> response = restTemplate.getForEntity(nginxHealthUrl, String.class);
            if (response.getStatusCode().is2xxSuccessful()) {
                // 解析 JSON 获取版本
                JsonNode root = new ObjectMapper().readTree(response.getBody());
                String version = root.path("nginx_version").asText();
                String status = root.path("status").asText();
                if ("UP".equals(status)) {
                    log.info("✅ Nginx health check passed. Version: {}", version);
                    return Optional.of(new NginxVersionInfo(version, root.path("build_time").asText()));
                }
            }
        } catch (Exception e) {
            log.warn("⚠️  Nginx health check failed: {}", e.getMessage());
        }
        return Optional.empty();
    }

    @Scheduled(fixedDelay = 30_000) // 每30秒检查一次
    public void scheduledHealthCheck() {
        checkNginxHealth()
                .ifPresentOrElse(
                        info -> {
                            if (!"1.26.0".equals(info.getVersion())) {
                                log.error("❌ Nginx version mismatch! Expected 1.26.0, got {}", info.getVersion());
                                triggerRollbackAlert(); // 发送告警
                            }
                        },
                        () -> {
                            log.error("❌ Nginx health endpoint unreachable. Triggering fallback.");
                            activateFallbackMode(); // 激活降级逻辑
                        }
                );
    }

    private void triggerRollbackAlert() {
        // 集成企业微信/钉钉机器人发送告警
        // 示例：发送到运维群
        String alertMsg = String.format(
                "🚨 Nginx Version Alert\nExpected: 1.26.0\nActual: %s\nTime: %s",
                getCurrentNginxVersion(), Instant.now()
        );
        // sendToDingTalk(alertMsg);
    }

    private void activateFallbackMode() {
        // 1. 切换至备用 Nginx 配置（如降级到静态页）
        // 2. 触发服务熔断（Hystrix / Resilience4j）
        // 3. 记录指标（Micrometer）
        Counter.builder("nginx.health.check.failures")
                .description("Count of failed Nginx health checks")
                .register(Metrics.globalRegistry)
                .increment();
    }

    private String getCurrentNginxVersion() {
        try {
            Process process = Runtime.getRuntime().exec("nginx -v 2>&1");
            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
            String line = reader.readLine();
            if (line != null && line.contains("nginx version: nginx/")) {
                return line.split("nginx/")[1].trim();
            }
        } catch (Exception ignored) {}
        return "UNKNOWN";
    }
}

// NginxVersionInfo.java
@Data
@AllArgsConstructor
public class NginxVersionInfo {
    private String version;
    private String buildTime;
}

management:
  endpoint:
    nginx-health:
      show-details: always
  endpoints:
    web:
      exposure:
        include: health,nginx-health,metrics,prometheus

@Component
public class NginxHealthIndicator implements HealthIndicator {
    private final NginxHealthChecker checker;
    public NginxHealthIndicator(NginxHealthChecker checker) {
        this.checker = checker;
    }
    @Override
    public Health health() {
        return checker.checkNginxHealth()
                .map(info -> Health.up()
                        .withDetail("version", info.getVersion())
                        .withDetail("buildTime", info.getBuildTime())
                        .build())
                .orElseGet(() -> Health.down()
                        .withDetail("reason", "Nginx health endpoint unreachable")
                        .build());
    }
}

{
  "status": "UP",
  "components": {
    "nginx-health": {
      "status": "UP",
      "details": {
        "version": "1.26.0",
        "buildTime": "2024-05-15T10:30:45Z"
      }
    }
  }
}

#!/bin/bash
set -e

BACKUP_PID="/var/run/nginx.pid.backup.v1.24.0"
OLD_BINARY="/usr/sbin/nginx.backup.v1.24.0"
OLD_CONF="/etc/nginx.backup.v1.24.0/nginx.conf"

echo "[ROLLBACK] Initiating rollback to Nginx v1.24.0..."

# Step 1: Stop current (v1.26.0) master if running
if [ -f "/var/run/nginx-v1.26.0.pid" ]; then
  echo "[INFO] Stopping current v1.26.0 master"
  sudo kill -QUIT $(cat /var/run/nginx-v1.26.0.pid) 2>/dev/null || true
  sleep 3
fi

# Step 2: Restore old config
echo "[INFO] Restoring old configuration from backup"
sudo rm -rf /etc/nginx/
sudo cp -r /etc/nginx.backup.v1.24.0/ /etc/nginx/

# Step 3: Start old master with old binary
echo "[INFO] Starting old Nginx v1.24.0 master"
sudo $OLD_BINARY -c "$OLD_CONF" -t
sudo $OLD_BINARY -c "$OLD_CONF"

# Step 4: Verify old master is running and listening
NEW_PID=$(sudo cat /var/run/nginx.pid)
if [ -z "$NEW_PID" ] || ! sudo kill -0 "$NEW_PID" 2>/dev/null; then
  echo "[ERROR] Old Nginx failed to start!" >&2
  exit 1
fi

# Step 5: Update symlink back to old binary
sudo ln -sf "$OLD_BINARY" /usr/sbin/nginx

echo "[SUCCESS] Rollback to Nginx v1.24.0 completed!"

sudo /usr/local/bin/nginx-rollback-to-v1.24.0.sh

// 在 scheduledHealthCheck() 方法中追加
if (checkNginxHealth().isEmpty()) {
    log.warn("Nginx health check failed 3 times consecutively. Preparing auto-rollback...");
    if (shouldTriggerAutoRollback()) {
        executeRollbackScript();
        notifyRollbackSuccess();
    }
}
private boolean shouldTriggerAutoRollback() {
    // 使用 Redis 计数器，防止抖动
    String key = "nginx:health:failures";
    Long count = redisTemplate.opsForValue().increment(key, 1);
    redisTemplate.expire(key, Duration.ofMinutes(5));
    return count >= 3;
}
private void executeRollbackScript() {
    try {
        Process proc = Runtime.getRuntime().exec("sudo /usr/local/bin/nginx-rollback-to-v1.24.0.sh");
        int exitCode = proc.waitFor();
        if (exitCode == 0) {
            log.info("✅ Auto-rollback executed successfully.");
        } else {
            log.error("❌ Auto-rollback script failed with exit code: {}", exitCode);
        }
    } catch (Exception e) {
        log.error("💥 Failed to execute rollback script", e);
    }
}

# 查看当前系统 OpenSSL
openssl version -a

# 查看新 binary 依赖的 OpenSSL
ldd /usr/local/nginx-v1.26.0/sbin/nginx | grep ssl

# 若不匹配，需在 configure 时指定 --with-openssl=/path/to/source

# 检查模块是否加载成功
/usr/local/nginx-v1.26.0/sbin/nginx -V 2>&1 | grep -i "modules"

# 在 http {} 块中添加（避免 400 错误）
underscores_in_headers on; # 或 off，根据业务需要

ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;

# 对旧版本压测
wrk -t4 -c100 -d30s http://localhost/

# 对新版本压测（启动临时实例）
/usr/local/nginx-v1.26.0/sbin/nginx -c /etc/nginx/nginx.conf.v1.26.0 -p /tmp/nginx-test
wrk -t4 -c100 -d30s http://localhost:8080/

@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
class NginxUpgradeSmokeTest {
    @Autowired
    private TestRestTemplate restTemplate;
    @Test
    void shouldServeStaticAssetsViaNginx() {
        ResponseEntity<String> response = restTemplate.getForEntity("/static/logo.png", String.class);
        assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
        assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.IMAGE_PNG);
    }
    @Test
    void shouldProxyToJavaBackend() {
        ResponseEntity<String> response = restTemplate.getForEntity("/api/users/me", String.class);
        assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
        assertThat(response.getBody()).contains("\"username\"");
    }
    @Test
    void shouldHandleHttpsRedirectCorrectly() {
        // 测试 HTTP → HTTPS 重定向
        HttpHeaders headers = new HttpHeaders();
        headers.set("X-Forwarded-Proto", "http");
        HttpEntity<Void> entity = new HttpEntity<>(headers);
        ResponseEntity<Void> redirect = restTemplate.exchange(
                "/secure", HttpMethod.GET, entity, Void.class);
        assertThat(redirect.getStatusCode()).isEqualTo(HttpStatus.MOVED_PERMANENTLY);
        assertThat(redirect.getHeaders().getLocation()).hasToString("https://localhost/secure");
    }
}

location /nginx_status {
    stub_status;
    allow 127.0.0.1;
    deny all;
}

@Service
public class NginxMetricsCollector {
    private final RestTemplate restTemplate;
    private final MeterRegistry meterRegistry;
    public NginxMetricsCollector(RestTemplateBuilder builder, MeterRegistry registry) {
        this.restTemplate = builder.build();
        this.meterRegistry = registry;
    }
    @Scheduled(fixedRate = 15_000)
    public void collectNginxMetrics() {
        try {
            String status = restTemplate.getForObject("http://localhost/nginx_status", String.class);
            parseAndRecordMetrics(status);
        } catch (Exception e) {
            log.warn("Failed to collect nginx metrics", e);
        }
    }
    private void parseAndRecordMetrics(String status) {
        // Active connections: 3
        // server accepts handled requests
        // 12345 12345 67890
        // Reading: 0 Writing: 1 Waiting: 2
        Pattern p = Pattern.compile("Active connections:\\s+(\\d+).*?" +
                "server accepts handled requests\\s+(\\d+)\\s+(\\d+)\\s+(\\d+).*?" +
                "Reading:\\s+(\\d+)\\s+Writing:\\s+(\\d+)\\s+Waiting:\\s+(\\d+)", Pattern.DOTALL);
        Matcher m = p.matcher(status);
        if (m.find()) {
            Gauge.builder("nginx.connections.active", () -> Double.parseDouble(m.group(1)))
                    .register(meterRegistry);
            Gauge.builder("nginx.requests.total", () -> Double.parseDouble(m.group(4)))
                    .register(meterRegistry);
            Gauge.builder("nginx.connections.reading", () -> Double.parseDouble(m.group(5)))
                    .register(meterRegistry);
        }
    }
}

@Bean
public Timer nginxHealthCheckTimer(MeterRegistry registry) {
    return Timer.builder("nginx.health.check.latency")
            .description("Latency of Nginx health endpoint calls")
            .register(registry);
}
// 在 checkNginxHealth() 中
long start = System.nanoTime();
Optional<NginxVersionInfo> result = ...;
nginxHealthCheckTimer.record(System.nanoTime() - start, TimeUnit.NANOSECONDS);

@Bean
public GlobalFilter nginxHopFilter() {
    return (exchange, chain) -> {
        ServerHttpRequest request = exchange.getRequest();
        // 从 X-Real-IP 或 X-Forwarded-For 解析 Nginx 跳数
        String hops = request.getHeaders().getFirst("X-Nginx-Hops");
        if (hops != null) {
            Span.current().setAttribute("nginx.hops", Integer.parseInt(hops));
        }
        return chain.filter(exchange);
    };
}