Nginx中生产环境常见错误502、503、504的排查与解决

http {
    # 1. 提升错误日志级别（生产谨慎用 info，可临时切 warn）
    error_log /var/log/nginx/error.log warn;

    # 2. 扩展 access log 格式，包含上游关键指标
    log_format upstream '$remote_addr - $remote_user [$time_local] '
                         '"$request" $status $body_bytes_sent '
                         '"$http_referer" "$http_user_agent" '
                         'rt=$request_time uct="$upstream_connect_time" '
                         'uht="$upstream_header_time" urt="$upstream_response_time" '
                         'ups="$upstream_addr"';

    access_log /var/log/nginx/access.log upstream;

    # 3. 启用 upstream 日志（记录每个 upstream server 的状态变化）
    upstream_conf;
}

@RestController
public class UnsafeController {
    @GetMapping("/unsafe-json")
    public void unsafeJsonResponse(HttpServletResponse response) throws IOException {
        // ⚠️ 严重错误：未设置状态码、Content-Type、未使用 ResponseEntity
        response.getOutputStream().write("{\"code\":0,\"msg\":\"success\"}".getBytes(StandardCharsets.UTF_8));
        // 缺少 response.getOutputStream().flush()，且未关闭流！
    }
}

@RestController
@RequestMapping("/api")
public class SafeController {
    // ✅ 正确：使用 ResponseEntity，自动设置状态码、Content-Type、Content-Length
    @GetMapping("/users")
    public ResponseEntity<List<User>> getUsers() {
        List<User> users = userService.findAll();
        return ResponseEntity.ok()
                .header("X-Processed-By", "Java-SpringBoot") // 自定义 Header
                .body(users);
    }
    // ✅ 正确：对流式响应，使用 StreamingResponseBody 并显式设置 Header
    @GetMapping(value = "/large-report", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
    public ResponseEntity<StreamingResponseBody> downloadReport() {
        return ResponseEntity.ok()
                .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=report.csv")
                .header(HttpHeaders.CONTENT_TYPE, "text/csv;charset=UTF-8")
                .body(outputStream -> {
                    try (PrintWriter writer = new PrintWriter(outputStream, true, StandardCharsets.UTF_8)) {
                        writer.println("id,name,email");
                        userService.exportAllUsers(writer); // 流式写入
                    }
                });
    }
}

upstream java_backend {
    server 10.0.1.5:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.6:8080 max_fails=3 fail_timeout=30s;

    # ✅ 关键：允许上游返回非标准但可解析的状态码（如 Spring Boot Actuator 的 200/503）
    #      但禁止 502（避免循环）
    proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
    proxy_next_upstream_tries 3;
    proxy_next_upstream_timeout 10s;

    # ✅ 关键：强制 Nginx 验证上游响应头完整性
    proxy_buffering on;
    proxy_buffer_size 128k;
    proxy_buffers 4 256k;
    proxy_busy_buffers_size 256k;
    proxy_max_temp_file_size 0; # 禁用临时文件，避免磁盘 IO 影响

    # ✅ 关键：设置合理的读取超时，避免因上游慢导致 502（实为 504）
    proxy_read_timeout 60s;
    proxy_send_timeout 60s;
}

# 模拟一个“非法”响应（无状态行）—— 触发 502
printf "Content-Type: application/json\r\n\r\n{\"error\":\"bad\"}" | nc 10.0.1.5 8080

# 使用 curl 查看 Nginx 是否透传上游 Header（调试 502 时关键！）
curl -I -v https://api.example.com/api/users
# 观察响应头中是否有 X-Upstream-Addr, X-Upstream-Response-Time 等 Nginx 注入头

@Component
public class DatabaseHealthIndicator implements ReactiveHealthIndicator {
    private final DataSource dataSource;
    public DatabaseHealthIndicator(DataSource dataSource) {
        this.dataSource = dataSource;
    }
    @Override
    public Mono<Health> health() {
        return Mono.fromCallable(() -> {
            try (Connection conn = dataSource.getConnection()) {
                // ✅ 关键：只执行轻量级检查（如 SELECT 1），避免长事务
                try (Statement stmt = conn.createStatement()) {
                    stmt.execute("SELECT 1");
                    return Health.up()
                            .withDetail("database", "PostgreSQL")
                            .withDetail("version", getDbVersion(conn))
                            .build();
                }
            } catch (Exception ex) {
                // ⚠️ 重要：捕获所有异常，避免 HealthCheck 抛出 RuntimeException 导致整个 endpoint 失效
                return Health.down()
                        .withDetail("error", ex.getMessage())
                        .withDetail("timestamp", Instant.now().toString())
                        .build();
            }
        });
    }
    private String getDbVersion(Connection conn) throws SQLException {
        return conn.getMetaData().getDatabaseProductVersion();
    }
}

upstream java_backend {
    # ✅ 开启主动健康检查（需 nginx-plus 或开源版编译时加入 http_upstream_check_module）
    check interval=3 rise=2 fall=5 timeout=1 type=http;
    check_http_send "HEAD /actuator/health HTTP/1.1\r\nHost: localhost\r\n\r\n";
    check_http_expect_alive http_2xx http_3xx;

    server 10.0.1.5:8080;
    server 10.0.1.6:8080;
}

# 暴露健康检查状态页（供运维监控）
server {
    listen 8081;
    location /upstream_status {
        check_status;
        access_log off;
        allow 127.0.0.1;
        deny all;
    }
}

// 使用 Resilience4j 实现优雅降级
@Value("${resilience4j.circuitbreaker.instances.payment.timeout-duration:3s}")
private Duration timeoutDuration;
@Bean
public CircuitBreaker circuitBreaker() {
    CircuitBreakerConfig config = CircuitBreakerConfig.custom()
            .failureRateThreshold(50) // 错误率 >50% 触发熔断
            .waitDurationInOpenState(Duration.ofSeconds(60)) // 保持 OPEN 60 秒
            .ringBufferSizeInHalfOpenState(10) // HALF_OPEN 状态下最多试 10 次
            .recordExceptions(TimeoutException.class, SQLException.class)
            .ignoreExceptions(BusinessException.class) // 业务异常不计入失败
            .build();
    return CircuitBreakerRegistry.of(config).circuitBreaker("paymentService");
}
// Controller 中使用
@GetMapping("/order/{id}")
public ResponseEntity<Order> getOrder(@PathVariable Long id) {
    Supplier<Order> orderSupplier = () -> paymentService.getOrder(id);
    try {
        Order order = circuitBreaker.executeSupplier(orderSupplier);
        return ResponseEntity.ok(order);
    } catch (CallNotPermittedException e) {
        // ✅ 熔断开启时，主动返回 503 + 友好提示
        return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE)
                .header("X-RateLimit-Reset", String.valueOf(System.currentTimeMillis() + 60_000))
                .body(Order.builder()
                        .status("SERVICE_UNAVAILABLE")
                        .message("Payment service is temporarily unavailable. Please try again later.")
                        .build());
    } catch (Exception e) {
        throw new RuntimeException("Failed to fetch order", e);
    }
}

# 1. 下载 Arthas（Alibaba 开源 Java 诊断神器）
curl -O https://arthas.aliyun.com/arthas-boot.jar

# 2. Attach 到 Java 进程（PID 从 jps 获取）
java -jar arthas-boot.jar 12345

# 3. 监控慢方法（如 findAllUsers 耗时 > 5s）
[arthas@12345]$ trace com.example.service.UserService findAllUsers --condition 'duration>5000'

# 4. 查看线程堆栈（定位阻塞点）
[arthas@12345]$ thread -n 3  # 显示 CPU 最高 3 个线程
[arthas@12345]$ thread 25   # 查看线程 ID 25 的完整堆栈

# application.yml
spring:
  datasource:
    hikari:
      # ✅ 关键：连接池大小必须匹配数据库最大连接数
      maximum-pool-size: 20
      minimum-idle: 5
      # ✅ 关键：连接超时必须 < Nginx proxy_read_timeout
      connection-timeout: 30000  # 30s
      validation-timeout: 3000
      idle-timeout: 600000
      max-lifetime: 1800000
  # ✅ 关键：WebMvc 线程池（Tomcat/Jetty）
  web:
    server:
      tomcat:
        threads:
          max: 200          # Tomcat 最大工作线程
          min-spare: 10     # 最小空闲线程
  # ✅ 关键：异步任务线程池（@Async）
  task:
    execution:
      pool:
        core-size: 10
        max-size: 50
        queue-capacity: 100

upstream java_backend {
    server 10.0.1.5:8080;
    server 10.0.1.6:8080;

    # ✅ 黄金公式：proxy_read_timeout ≥ (DB连接超时 + 业务逻辑P99 + GC停顿P99)
    #    假设 DB 超时 30s，业务 P99 15s，GC P99 2s → 至少 47s，建议设为 60s
    proxy_read_timeout 60s;
    proxy_send_timeout 60s;
    proxy_connect_timeout 5s; # 网络层握手，通常 1-3s 足够

    # ✅ 关键：启用 keepalive，复用连接，避免反复握手耗时
    keepalive 32;
}

server {
    location /api/ {
        proxy_pass http://java_backend;
        proxy_http_version 1.1;
        proxy_set_header Connection '';
        # ✅ 关键：透传真实客户端 IP，用于后端日志追踪
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

@Service
public class ReportService {
    private final RestTemplate restTemplate;
    public ReportService(RestTemplateBuilder builder) {
        // ✅ 为 RestTemplate 设置独立超时，避免拖垮整个应用
        this.restTemplate = builder
                .setConnectTimeout(Duration.ofSeconds(5))   // 连接超时
                .setReadTimeout(Duration.ofSeconds(30))     // 读取超时（必须 < Nginx proxy_read_timeout）
                .build();
    }
    public Report generateReport(ReportRequest request) {
        try {
            // ✅ 使用 CompletableFuture 实现异步调用 + 超时控制
            return CompletableFuture.supplyAsync(() -> {
                try {
                    // 调用外部报表服务
                    return restTemplate.postForObject(
                            "https://report-service/v1/generate",
                            request,
                            Report.class);
                } catch (ResourceAccessException ex) {
                    throw new ReportGenerationException("External report service unavailable", ex);
                }
            }, Executors.newFixedThreadPool(5))
            .orTimeout(45, TimeUnit.SECONDS) // ✅ 整体超时 45s < Nginx 60s
            .exceptionally(throwable -> {
                if (throwable instanceof TimeoutException) {
                    throw new ReportGenerationException("Report generation timeout after 45s");
                }
                throw new RuntimeException(throwable);
            })
            .join();
        } catch (CompletionException e) {
            throw new ReportGenerationException("Failed to generate report", e.getCause());
        }
    }
}

# Dockerfile
FROM openjdk:17-jdk-slim

# ✅ 关键：使用 JVM 容器感知参数（JDK 10+ 原生支持）
#      -XX:+UseContainerSupport 自动识别容器内存限制
#      -XX:MaxRAMPercentage=75.0 将最大堆设为容器内存的 75%
ENV JAVA_OPTS="-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+UseG1GC"

COPY app.jar /app.jar
ENTRYPOINT ["sh", "-c", "java $JAVA_OPTS -jar /app.jar"]

# Kubernetes deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: java-app
spec:
  template:
    spec:
      containers:
      - name: app
        image: my-java-app:1.0
        resources:
          requests:
            memory: "1536Mi"   # ✅ 必须 ≥ JVM MaxRAMPercentage 计算出的堆上限
            cpu: "500m"
          limits:
            memory: "2Gi"       # ✅ 容器内存上限，JVM 以此为基准
            cpu: "1000m"
        env:
        - name: JAVA_OPTS
          value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+UseG1GC"

@Component
public class GcWatcher {
    private static final Logger log = LoggerFactory.getLogger(GcWatcher.class);
    @EventListener
    public void handleGcEvent(GarbageCollectionEvent event) {
        if ("G1 Old Generation".equals(event.getGarbageCollectorName()) &&
                event.getDuration() > 2000 && // GC 耗时 >2s
                System.currentTimeMillis() - event.getStartTime() < 60_000) { // 近 1 分钟内
            long gcCount = ManagementFactory.getGarbageCollectorMXBean(
                    "G1 Old Generation").getCollectionCount();
            // ✅ 连续 3 次长时间 GC，触发自愈
            if (gcCount >= 3) {
                log.error("Critical: 3+ long GC detected in 60s. Triggering graceful shutdown.");
                Runtime.getRuntime().addShutdownHook(new Thread(() -> {
                    try {
                        // 等待正在处理的请求完成（Spring Boot Actuator 提供）
                        Thread.sleep(30_000);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                    }
                }));
                System.exit(143); // SIGTERM
            }
        }
    }
}

# 在 Nginx 配置中注入指标
log_format metrics '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent '
                    '$request_time $upstream_response_time '
                    '$upstream_addr $upstream_status';

# 通过 Logstash 或 Filebeat 将 metrics 日志发送至 Elasticsearch
# Kibana 中创建告警：当 5xx 率 >5% 且持续 2 分钟 → 触发 K8s API 扩容