Nginx与后端服务的连接复用问题的解决方法

1000 conn/s × 60 s = 60,000 > 32,768 → 必然失败

connect() failed (99: Cannot assign requested address) while connecting to upstream

http {
    # 全局客户端连接保活基础设置
    keepalive_timeout  65s;          # 连接空闲 65 秒后关闭（建议 60–75s）
    keepalive_requests 100;          # 单连接最多处理 100 个请求（防内存泄漏）
    reset_timedout_connection on;    # 对超时连接立即发送 RST，快速回收资源
    client_header_timeout 10s;       # 客户端发请求头超时
    client_body_timeout 10s;         # 客户端发请求体超时
    send_timeout 10s;                # Nginx 发送响应超时
    # 强制启用 HTTP/1.1 Keep-Alive（即使客户端未声明）
    add_header Connection keep-alive;
}

server {
    listen 443 ssl http2;  # 关键：添加 http2
    ssl_certificate /path/to/cert.pem;
    ssl_certificate_key /path/to/key.pem;
    # ... 其他 SSL 配置
}

upstream java_backend {
    server 10.0.1.10:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.11:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.12:8080 max_fails=3 fail_timeout=30s;
    # ✅ 核心：启用上游连接复用，最大空闲连接数为 32
    # 注意：这不是“总连接数”，而是“每个 worker 进程维护的空闲连接池大小”
    keepalive 32;
    # 可选：指定健康检查间隔（配合 keepalive 更有效）
    # health_check interval=5 fails=3 passes=2;
}
server {
    location /api/ {
        proxy_pass http://java_backend;
        # ✅ 强制向上游发起 Keep-Alive 请求（HTTP/1.1）
        proxy_http_version 1.1;
        proxy_set_header Connection '';
        # ✅ 传递原始 Host，便于后端日志与路由
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        # ✅ 超时设置（必须大于后端处理时间，但不宜过大）
        proxy_connect_timeout 5s;
        proxy_send_timeout 30s;
        proxy_read_timeout 30s;
        # ✅ 缓冲区调优（减少小包发送，提升吞吐）
        proxy_buffering on;
        proxy_buffer_size 128k;
        proxy_buffers 4 256k;
        proxy_busy_buffers_size 256k;
    }
}

N ≈ (峰值 QPS × 平均请求处理时间) ÷ (1 - 复用率目标)

server:
  port: 8080
  tomcat:
    # ✅ 启用 HTTP/1.1 Keep-Alive（默认已开，显式强调）
    connection-timeout: 30000   # 30s，需 ≥ Nginx proxy_read_timeout
    # ✅ 最大连接数（应对突发流量）
    max-connections: 10000
    # ✅ accept 队列长度（防连接拒绝）
    accept-count: 200
    # ✅ 禁用压缩（由 Nginx 统一处理，避免 CPU 浪费）
    compression:
      enabled: false
# ✅ 关键：配置内嵌 Tomcat 的连接器（Connector）属性
spring:
  web:
    resources:
      cache:
        cachecontrol:
          max-age: 3600
# ✅ 如果使用 WebClient（推荐），务必配置连接池
spring:
  webflux:
    client:
      max-in-memory-size: 10MB
  # ⚠️ 注意：WebClient 默认使用 Reactor Netty，其连接池需单独配置

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import reactor.netty.resources.ConnectionProvider;
@Configuration
public class WebClientConfig {
    @Bean
    public WebClient webClient() {
        // ✅ 创建带连接池的 HttpClient
        ConnectionProvider provider = ConnectionProvider.builder("backend-pool")
                .maxConnections(500)           // 每个地址最大连接数
                .pendingAcquireMaxCount(-1)   // 无限制等待（生产慎用，建议设为 1000）
                .pendingAcquireTimeout(Duration.ofSeconds(45))
                .maxIdleTime(Duration.ofSeconds(60))     // ✅ 空闲连接最长存活 60s
                .maxLifeTime(Duration.ofMinutes(30))     // 连接最大生命周期 30min
                .evictInBackground(Duration.ofSeconds(30)) // 后台清理任务间隔
                .build();
        HttpClient httpClient = HttpClient.create(provider)
                .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 5000)
                .responseTimeout(Duration.ofSeconds(30))
                // ✅ 关键：启用 Keep-Alive（Reactor Netty 默认开启，此处显式确认）
                .keepAlive(true)
                // ✅ 关键：禁用自动重试（由业务层控制）
                .followRedirect(false);
        return WebClient.builder()
                .clientConnector(new ReactorClientHttpConnector(httpClient))
                .build();
    }
}

@GetMapping("/users/{id}")
public ResponseEntity<String> getUser(@PathVariable Long id) throws IOException {
    URL url = new URL("http://backend:8080/api/users/" + id);
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    conn.setRequestMethod("GET");
    conn.setConnectTimeout(5000);
    conn.setReadTimeout(10000);
    int status = conn.getResponseCode();
    String body = IOUtils.toString(conn.getInputStream(), StandardCharsets.UTF_8);
    conn.disconnect(); // ❌ 错误！强制关闭底层 Socket，破坏复用
    return ResponseEntity.ok(body);
}

@RestController
@RequestMapping("/api")
public class UserController {
    private final WebClient webClient;
    public UserController(WebClient webClient) {
        this.webClient = webClient;
    }
    @GetMapping("/users/{id}")
    public Mono<ResponseEntity<String>> getUser(@PathVariable Long id) {
        return webClient.get()
                .uri("http://backend:8080/api/users/{id}", id)
                .retrieve()
                .onStatus(HttpStatus::isError, response -> 
                    Mono.error(new RuntimeException("Backend error: " + response.statusCode())))
                .bodyToMono(String.class)
                .map(body -> ResponseEntity.ok().body(body))
                .onErrorResume(e -> Mono.just(ResponseEntity.status(502).body("Backend unavailable")));
    }
}

# 抓取到上游 10.0.1.10:8080 的流量
sudo tcpdump -i any -nn host 10.0.1.10 and port 8080 -w nginx_upstream.pcap

# 查看所有 ESTABLISHED 连接（重点关注 :8080）
ss -tn state established '( dport = :8080 )' | wc -l

# 查看 TIME_WAIT 连接（应大幅减少）
ss -tn state time-wait '( dport = :8080 )' | wc -l

# 查看每个上游 IP 的连接分布
ss -tn state established '( dport = :8080 )' | awk '{print $5}' | sort | uniq -c | sort -nr

log_format upstream_log '[$time_local] $remote_addr - $remote_user '
                         '"$request" $status $body_bytes_sent '
                         '"$http_referer" "$http_user_agent" '
                         'rt=$request_time uct="$upstream_connect_time" '
                         'uht="$upstream_header_time" urt="$upstream_response_time" '
                         'upstream_addr="$upstream_addr"';
access_log /var/log/nginx/upstream.log upstream_log;

upstream grpc_backend {
    server 10.0.1.20:9090;
    # ✅ HTTP/2 upstream 必须启用 keepalive，且协议指定为 h2
    keepalive 100;
}
server {
    location /grpc/ {
        # ✅ 强制使用 HTTP/2 协议转发
        proxy_pass https://grpc_backend;
        proxy_http_version 2;
        proxy_set_header Host $host;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
        # ✅ gRPC 特定头
        proxy_set_header X-Forwarded-For $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

location /api.grpc. {
    # gRPC-Web 需要 upgrade 头
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection "upgrade";
    # 后端是 gRPC 服务（非 gRPC-Web）
    proxy_pass http://grpc_backend;
    proxy_cache off;
    proxy_buffering off;
}

const server = http.createServer(app);
// ✅ 关键：设置 keepAlive 选项
server.keepAliveTimeout = 65 * 1000;     // 匹配 Nginx keepalive_timeout
server.maxHeadersCount = 1000;
server.timeout = 30000; // 30s，匹配 proxy_read_timeout
// 启动
server.listen(8080);

srv := &http.Server{
    Addr:         ":8080",
    Handler:      router,
    ReadTimeout:  30 * time.Second,   // 匹配 proxy_read_timeout
    WriteTimeout: 30 * time.Second,
    IdleTimeout:  65 * time.Second,   // ✅ 关键：空闲超时，匹配 keepalive_timeout
}
log.Fatal(srv.ListenAndServe())

$ wrk -t4 -c100 -d30s http://nginx/api/users/1
Running 30s test @ http://nginx/api/users/1
  4 threads and 100 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    42.60ms   62.12ms 524.84ms   85.23%
    Req/Sec   532.11    129.77     1.02k    72.50%
  63744 requests in 30.01s, 11.22MB read
  Socket errors: connect 0, read 0, write 0, timeout 128
Requests/sec:   2124.31
Transfer/sec:    382.24KB

$ wrk -t4 -c100 -d30s http://nginx/api/users/1
Running 30s test @ http://nginx/api/users/1
  4 threads and 100 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    18.12ms   12.45ms 128.73ms   82.17%
    Req/Sec  1472.21    218.65     2.15k    68.33%
  176528 requests in 30.00s, 31.16MB read
  Socket errors: connect 0, read 0, write 0, timeout 0
Requests/sec:   5884.27
Transfer/sec:      1.04MB

# 使用 lua-resty-upstream-healthcheck 模块
upstream dynamic_backend {
    server 0.0.0.0:1; # placeholder
    keepalive 32;
}
location /api/ {
    content_by_lua_block {
        local balancer = require "ngx.balancer"
        local healthcheck = require "resty.upstream.healthcheck"
        -- 动态解析服务发现地址
        local srvs = resolve_service("java-backend.default.svc.cluster.local")
        for _, srv in ipairs(srvs) do
            balancer.set_current_peer(srv.host, srv.port)
        end
    }
}