Nginx中生产环境常见错误502、503、504的排查与解决

http {
    # 1. 提升错误日志级别（生产谨慎用 info，可临时切 warn）
    error_log /var/log/nginx/error.log warn;

    # 2. 扩展 access log 格式，包含上游关键指标
    log_format upstream '$remote_addr - $remote_user [$time_local] '
                         '"$request" $status $body_bytes_sent '
                         '"$http_referer" "$http_user_agent" '
                         'rt=$request_time uct="$upstream_connect_time" '
                         'uht="$upstream_header_time" urt="$upstream_response_time" '
                         'ups="$upstream_addr"';

    access_log /var/log/nginx/access.log upstream;

    # 3. 启用 upstream 日志（记录每个 upstream server 的状态变化）
    upstream_conf;
}

@restcontroller
public class unsafecontroller {
    @getmapping("/unsafe-json")
    public void unsafejsonresponse(httpservletresponse response) throws ioexception {
        // ⚠️ 严重错误：未设置状态码、content-type、未使用 responseentity
        response.getoutputstream().write("{\"code\":0,\"msg\":\"success\"}".getbytes(standardcharsets.utf_8));
        // 缺少 response.getoutputstream().flush()，且未关闭流！
    }
}

@restcontroller
@requestmapping("/api")
public class safecontroller {
    // ✅ 正确：使用 responseentity，自动设置状态码、content-type、content-length
    @getmapping("/users")
    public responseentity<list<user>> getusers() {
        list<user> users = userservice.findall();
        return responseentity.ok()
                .header("x-processed-by", "java-springboot") // 自定义 header
                .body(users);
    }
    // ✅ 正确：对流式响应，使用 streamingresponsebody 并显式设置 header
    @getmapping(value = "/large-report", produces = mediatype.application_octet_stream_value)
    public responseentity<streamingresponsebody> downloadreport() {
        return responseentity.ok()
                .header(httpheaders.content_disposition, "attachment; filename=report.csv")
                .header(httpheaders.content_type, "text/csv;charset=utf-8")
                .body(outputstream -> {
                    try (printwriter writer = new printwriter(outputstream, true, standardcharsets.utf_8)) {
                        writer.println("id,name,email");
                        userservice.exportallusers(writer); // 流式写入
                    }
                });
    }
}

upstream java_backend {
    server 10.0.1.5:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.6:8080 max_fails=3 fail_timeout=30s;

    # ✅ 关键：允许上游返回非标准但可解析的状态码（如 spring boot actuator 的 200/503）
    #      但禁止 502（避免循环）
    proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
    proxy_next_upstream_tries 3;
    proxy_next_upstream_timeout 10s;

    # ✅ 关键：强制 nginx 验证上游响应头完整性
    proxy_buffering on;
    proxy_buffer_size 128k;
    proxy_buffers 4 256k;
    proxy_busy_buffers_size 256k;
    proxy_max_temp_file_size 0; # 禁用临时文件，避免磁盘 io 影响

    # ✅ 关键：设置合理的读取超时，避免因上游慢导致 502（实为 504）
    proxy_read_timeout 60s;
    proxy_send_timeout 60s;
}

# 模拟一个“非法”响应（无状态行）—— 触发 502
printf "content-type: application/json\r\n\r\n{\"error\":\"bad\"}" | nc 10.0.1.5 8080

# 使用 curl 查看 nginx 是否透传上游 header（调试 502 时关键！）
curl -i -v https://api.example.com/api/users
# 观察响应头中是否有 x-upstream-addr, x-upstream-response-time 等 nginx 注入头

@component
public class databasehealthindicator implements reactivehealthindicator {
    private final datasource datasource;
    public databasehealthindicator(datasource datasource) {
        this.datasource = datasource;
    }
    @override
    public mono<health> health() {
        return mono.fromcallable(() -> {
            try (connection conn = datasource.getconnection()) {
                // ✅ 关键：只执行轻量级检查（如 select 1），避免长事务
                try (statement stmt = conn.createstatement()) {
                    stmt.execute("select 1");
                    return health.up()
                            .withdetail("database", "postgresql")
                            .withdetail("version", getdbversion(conn))
                            .build();
                }
            } catch (exception ex) {
                // ⚠️ 重要：捕获所有异常，避免 healthcheck 抛出 runtimeexception 导致整个 endpoint 失效
                return health.down()
                        .withdetail("error", ex.getmessage())
                        .withdetail("timestamp", instant.now().tostring())
                        .build();
            }
        });
    }
    private string getdbversion(connection conn) throws sqlexception {
        return conn.getmetadata().getdatabaseproductversion();
    }
}

upstream java_backend {
    # ✅ 开启主动健康检查（需 nginx-plus 或开源版编译时加入 http_upstream_check_module）
    check interval=3 rise=2 fall=5 timeout=1 type=http;
    check_http_send "head /actuator/health http/1.1\r\nhost: localhost\r\n\r\n";
    check_http_expect_alive http_2xx http_3xx;

    server 10.0.1.5:8080;
    server 10.0.1.6:8080;
}

# 暴露健康检查状态页（供运维监控）
server {
    listen 8081;
    location /upstream_status {
        check_status;
        access_log off;
        allow 127.0.0.1;
        deny all;
    }
}

// 使用 resilience4j 实现优雅降级
@value("${resilience4j.circuitbreaker.instances.payment.timeout-duration:3s}")
private duration timeoutduration;
@bean
public circuitbreaker circuitbreaker() {
    circuitbreakerconfig config = circuitbreakerconfig.custom()
            .failureratethreshold(50) // 错误率 >50% 触发熔断
            .waitdurationinopenstate(duration.ofseconds(60)) // 保持 open 60 秒
            .ringbuffersizeinhalfopenstate(10) // half_open 状态下最多试 10 次
            .recordexceptions(timeoutexception.class, sqlexception.class)
            .ignoreexceptions(businessexception.class) // 业务异常不计入失败
            .build();
    return circuitbreakerregistry.of(config).circuitbreaker("paymentservice");
}
// controller 中使用
@getmapping("/order/{id}")
public responseentity<order> getorder(@pathvariable long id) {
    supplier<order> ordersupplier = () -> paymentservice.getorder(id);
    try {
        order order = circuitbreaker.executesupplier(ordersupplier);
        return responseentity.ok(order);
    } catch (callnotpermittedexception e) {
        // ✅ 熔断开启时，主动返回 503 + 友好提示
        return responseentity.status(httpstatus.service_unavailable)
                .header("x-ratelimit-reset", string.valueof(system.currenttimemillis() + 60_000))
                .body(order.builder()
                        .status("service_unavailable")
                        .message("payment service is temporarily unavailable. please try again later.")
                        .build());
    } catch (exception e) {
        throw new runtimeexception("failed to fetch order", e);
    }
}

# 1. 下载 arthas（alibaba 开源 java 诊断神器）
curl -o https://arthas.aliyun.com/arthas-boot.jar

# 2. attach 到 java 进程（pid 从 jps 获取）
java -jar arthas-boot.jar 12345

# 3. 监控慢方法（如 findallusers 耗时 > 5s）
[arthas@12345]$ trace com.example.service.userservice findallusers --condition 'duration>5000'

# 4. 查看线程堆栈（定位阻塞点）
[arthas@12345]$ thread -n 3  # 显示 cpu 最高 3 个线程
[arthas@12345]$ thread 25   # 查看线程 id 25 的完整堆栈

# application.yml
spring:
  datasource:
    hikari:
      # ✅ 关键：连接池大小必须匹配数据库最大连接数
      maximum-pool-size: 20
      minimum-idle: 5
      # ✅ 关键：连接超时必须 < nginx proxy_read_timeout
      connection-timeout: 30000  # 30s
      validation-timeout: 3000
      idle-timeout: 600000
      max-lifetime: 1800000
  # ✅ 关键：webmvc 线程池（tomcat/jetty）
  web:
    server:
      tomcat:
        threads:
          max: 200          # tomcat 最大工作线程
          min-spare: 10     # 最小空闲线程
  # ✅ 关键：异步任务线程池（@async）
  task:
    execution:
      pool:
        core-size: 10
        max-size: 50
        queue-capacity: 100

upstream java_backend {
    server 10.0.1.5:8080;
    server 10.0.1.6:8080;

    # ✅ 黄金公式：proxy_read_timeout ≥ (db连接超时 + 业务逻辑p99 + gc停顿p99)
    #    假设 db 超时 30s，业务 p99 15s，gc p99 2s → 至少 47s，建议设为 60s
    proxy_read_timeout 60s;
    proxy_send_timeout 60s;
    proxy_connect_timeout 5s; # 网络层握手，通常 1-3s 足够

    # ✅ 关键：启用 keepalive，复用连接，避免反复握手耗时
    keepalive 32;
}

server {
    location /api/ {
        proxy_pass http://java_backend;
        proxy_http_version 1.1;
        proxy_set_header connection '';
        # ✅ 关键：透传真实客户端 ip，用于后端日志追踪
        proxy_set_header x-real-ip $remote_addr;
        proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
        proxy_set_header x-forwarded-proto $scheme;
    }
}

@service
public class reportservice {
    private final resttemplate resttemplate;
    public reportservice(resttemplatebuilder builder) {
        // ✅ 为 resttemplate 设置独立超时，避免拖垮整个应用
        this.resttemplate = builder
                .setconnecttimeout(duration.ofseconds(5))   // 连接超时
                .setreadtimeout(duration.ofseconds(30))     // 读取超时（必须 < nginx proxy_read_timeout）
                .build();
    }
    public report generatereport(reportrequest request) {
        try {
            // ✅ 使用 completablefuture 实现异步调用 + 超时控制
            return completablefuture.supplyasync(() -> {
                try {
                    // 调用外部报表服务
                    return resttemplate.postforobject(
                            "https://report-service/v1/generate",
                            request,
                            report.class);
                } catch (resourceaccessexception ex) {
                    throw new reportgenerationexception("external report service unavailable", ex);
                }
            }, executors.newfixedthreadpool(5))
            .ortimeout(45, timeunit.seconds) // ✅ 整体超时 45s < nginx 60s
            .exceptionally(throwable -> {
                if (throwable instanceof timeoutexception) {
                    throw new reportgenerationexception("report generation timeout after 45s");
                }
                throw new runtimeexception(throwable);
            })
            .join();
        } catch (completionexception e) {
            throw new reportgenerationexception("failed to generate report", e.getcause());
        }
    }
}

# dockerfile
from openjdk:17-jdk-slim

# ✅ 关键：使用 jvm 容器感知参数（jdk 10+ 原生支持）
#      -xx:+usecontainersupport 自动识别容器内存限制
#      -xx:maxrampercentage=75.0 将最大堆设为容器内存的 75%
env java_opts="-xx:+usecontainersupport -xx:maxrampercentage=75.0 -xx:+useg1gc"

copy app.jar /app.jar
entrypoint ["sh", "-c", "java $java_opts -jar /app.jar"]

# kubernetes deployment.yaml
apiversion: apps/v1
kind: deployment
metadata:
  name: java-app
spec:
  template:
    spec:
      containers:
      - name: app
        image: my-java-app:1.0
        resources:
          requests:
            memory: "1536mi"   # ✅ 必须 ≥ jvm maxrampercentage 计算出的堆上限
            cpu: "500m"
          limits:
            memory: "2gi"       # ✅ 容器内存上限，jvm 以此为基准
            cpu: "1000m"
        env:
        - name: java_opts
          value: "-xx:+usecontainersupport -xx:maxrampercentage=75.0 -xx:+useg1gc"

@component
public class gcwatcher {
    private static final logger log = loggerfactory.getlogger(gcwatcher.class);
    @eventlistener
    public void handlegcevent(garbagecollectionevent event) {
        if ("g1 old generation".equals(event.getgarbagecollectorname()) &&
                event.getduration() > 2000 && // gc 耗时 >2s
                system.currenttimemillis() - event.getstarttime() < 60_000) { // 近 1 分钟内
            long gccount = managementfactory.getgarbagecollectormxbean(
                    "g1 old generation").getcollectioncount();
            // ✅ 连续 3 次长时间 gc，触发自愈
            if (gccount >= 3) {
                log.error("critical: 3+ long gc detected in 60s. triggering graceful shutdown.");
                runtime.getruntime().addshutdownhook(new thread(() -> {
                    try {
                        // 等待正在处理的请求完成（spring boot actuator 提供）
                        thread.sleep(30_000);
                    } catch (interruptedexception e) {
                        thread.currentthread().interrupt();
                    }
                }));
                system.exit(143); // sigterm
            }
        }
    }
}

# 在 nginx 配置中注入指标
log_format metrics '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent '
                    '$request_time $upstream_response_time '
                    '$upstream_addr $upstream_status';

# 通过 logstash 或 filebeat 将 metrics 日志发送至 elasticsearch
# kibana 中创建告警：当 5xx 率 >5% 且持续 2 分钟 → 触发 k8s api 扩容