Nginx结合Prometheus+Grafana实现监控告警的实战指南

# nginx.conf
server {
    listen 127.0.0.1:8080;
    server_name localhost;

    location /nginx_status {
        stub_status on;
        allow 127.0.0.1;   # 仅允许本机访问
        deny all;
    }
}

active connections: 3 
server accepts handled requests
 12345 12345 67890 
reading: 0 writing: 2 waiting: 1 

# 1. 安装依赖
sudo apt update && sudo apt install -y build-essential libpcre3-dev libssl-dev zlib1g-dev

# 2. 下载 nginx 源码（与当前版本严格一致！）
wget https://nginx.org/download/nginx-1.24.0.tar.gz
tar -zxvf nginx-1.24.0.tar.gz

# 3. 下载 vts 模块
git clone https://github.com/vozlt/nginx-module-vts.git

# 4. 重新编译 nginx（保留原有配置参数，追加 --add-module）
cd nginx-1.24.0
./configure \
  --prefix=/etc/nginx \
  --sbin-path=/usr/sbin/nginx \
  --modules-path=/usr/lib/nginx/modules \
  --conf-path=/etc/nginx/nginx.conf \
  --error-log-path=/var/log/nginx/error.log \
  --http-log-path=/var/log/nginx/access.log \
  --pid-path=/var/run/nginx.pid \
  --with-http_ssl_module \
  --with-http_v2_module \
  --with-http_realip_module \
  --add-module=../nginx-module-vts  # 👈 关键：添加 vts 模块

make && sudo make install

# nginx.conf 全局块
vhost_traffic_status_zone;  # 必须声明，定义共享内存区

http {
    # ... 其他 http 配置 ...

    # 新增 vts 状态页（prometheus 将从此抓取）
    server {
        listen 127.0.0.1:8081;
        server_name _;

        location /status/format/json {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format json;  # 返回 json
        }

        # 可选：提供 html 可视化界面（调试用）
        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }
}

{
  "nginx_version": "1.24.0",
  "load_timestamp": 1715823456,
  "serverzones": {
    "example.com": {
      "requestcounter": 12345,
      "inbytes": 87654321,
      "outbytes": 234567890,
      "responses": {
        "1xx": 0,
        "2xx": 11234,
        "3xx": 567,
        "4xx": 123,
        "5xx": 42
      },
      "cached": 0,
      "uncached": 12345,
      "processing": 3,
      "connections": {
        "active": 3,
        "reading": 0,
        "writing": 2,
        "waiting": 1
      }
    }
  },
  "upstreamzones": {
    "backend-api": {
      "peers": [
        {
          "server": "10.0.1.10:8080",
          "requestcounter": 6789,
          "inbytes": 45678901,
          "outbytes": 123456789,
          "responses": {"1xx":0,"2xx":6234,"3xx":123,"4xx":45,"5xx":12},
          "sent": 123456789,
          "received": 45678901,
          "fails": 0,
          "unavail": 0,
          "healthchecks": {"checks":123,"fails":0,"unhealthy":0,"lastpassed":1715823455}
        }
      ]
    }
  }
}

global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
  # 1. 抓取本机 nginx vts 指标（prometheus 格式）
  - job_name: 'nginx-vts'
    static_configs:
      - targets: ['127.0.0.1:8081']  # vts metrics 端口
    metrics_path: /status/format/prometheus  # 👈 关键：使用 prometheus 格式
    relabel_configs:
      # 为所有指标添加固定标签，便于多实例区分
      - source_labels: [__address__]
        target_label: instance
        replacement: nginx-prod-main
      - target_label: job
        replacement: nginx-vts
      # 重写指标名前缀（可选，避免与其他 exporter 冲突）
      - action: labelmap
        regex: __meta_vts_(.+)
  # 2. 抓取 prometheus 自身指标（用于监控 prometheus 健康）
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  # 3. 抓取 node exporter（服务器基础指标，与 nginx 关联分析）
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']

# 下载 prometheus（略去下载步骤，假设已解压到 /opt/prometheus）
cd /opt/prometheus
./prometheus --config.file=prometheus.yml --web.listen-address="0.0.0.0:9090"

# 查看 nginx 总请求数（按 server zone）
sum by (server) (nginx_vts_server_request_counter_total)

# 查看 5xx 错误率（过去5分钟）
sum(rate(nginx_vts_server_responses_total{code="5xx"}[5m])) 
/ 
sum(rate(nginx_vts_server_request_counter_total[5m]))

# 查看 upstream peer 的失败次数
nginx_vts_upstream_peer_fails_total{upstream="backend-api"}

// pom.xml 关键依赖
<dependencies>
    <dependency>
        <groupid>org.springframework.boot</groupid>
        <artifactid>spring-boot-starter-web</artifactid>
    </dependency>
    <dependency>
        <groupid>io.micrometer</groupid>
        <artifactid>micrometer-registry-prometheus</artifactid>
    </dependency>
    <!-- 添加 actuator，暴露 /actuator/prometheus -->
    <dependency>
        <groupid>org.springframework.boot</groupid>
        <artifactid>spring-boot-starter-actuator</artifactid>
    </dependency>
</dependencies>

// src/main/java/com/example/usercontroller.java
@restcontroller
@requestmapping("/api/v1/users")
public class usercontroller {
    private final meterregistry meterregistry;
    private final counter errorcounter;
    public usercontroller(meterregistry meterregistry) {
        this.meterregistry = meterregistry;
        // 自定义错误计数器，带业务维度
        this.errorcounter = counter.builder("user.service.errors")
                .description("count of business errors in user service")
                .tag("type", "database_timeout") // 可扩展为 network, cache, auth...
                .register(meterregistry);
    }
    @getmapping("/{id}")
    public responseentity<user> getuser(@pathvariable long id) {
        try {
            // 模拟数据库查询（可能超时）
            user user = fetchuserfromdb(id);
            return responseentity.ok(user);
        } catch (databasetimeoutexception e) {
            // 记录业务错误，同时触发 nginx 502（若上游超时）
            errorcounter.increment();
            // 记录到 micrometer，会被 prometheus 抓取
            meterregistry.counter("user.db.timeout.total", "service", "user-service").increment();
            return responseentity.status(500).build();
        }
    }
    private user fetchuserfromdb(long id) throws databasetimeoutexception {
        // 模拟随机超时（用于测试告警）
        if (math.random() > 0.95) {
            throw new databasetimeoutexception("simulated db timeout");
        }
        return new user(id, "john doe", "john@example.com");
    }
}
// 自定义异常
class databasetimeoutexception extends runtimeexception {
    public databasetimeoutexception(string msg) {
        super(msg);
    }
}

# application.yml
management:
  endpoints:
    web:
      exposure:
        include: health,info,prometheus  # 👈 关键：暴露 prometheus 端点
  endpoint:
    prometheus:
      scrape-interval: 15s
# 暴露 actuator 端点在 /actuator
server:
  port: 8080

# help user_service_errors count of business errors in user service
# type user_service_errors counter
user_service_errors{type="database_timeout",} 3.0
# help user_db_timeout_total  
# type user_db_timeout_total counter
user_db_timeout_total{service="user-service",} 3.0

# nginx.conf
upstream backend-api {
    server 127.0.0.1:8080 max_fails=3 fail_timeout=30s;
    # 可添加更多实例实现负载均衡
    # server 10.0.1.11:8080;
}

server {
    listen 80;
    server_name example.com;

    location /api/ {
        proxy_pass http://backend-api;
        proxy_set_header host $host;
        proxy_set_header x-real-ip $remote_addr;
        # 关键：传递原始请求路径，vts 会自动记录
        proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;

        # 设置超时，避免长连接拖垮 nginx
        proxy_connect_timeout 5s;
        proxy_send_timeout 30s;
        proxy_read_timeout 30s;

        # 启用健康检查（vts 会自动采集）
        health_check interval=5 rise=2 fall=3;
    }
}

// nginx 层 5xx 错误率（过去5分钟）
sum(rate(nginx_vts_server_responses_total{code="5xx", server="example.com"}[5m])) 
/ 
sum(rate(nginx_vts_server_request_counter_total{server="example.com"}[5m]))

// java 应用层 db 超时错误率（过去5分钟）
sum(rate(user_db_timeout_total{service="user-service"}[5m]))
/
sum(rate(http_server_requests_seconds_count{application="user-service"}[5m]))

groups:
- name: nginx-alerts
  rules:
  # 规则1：nginx 连接耗尽风险（紧急！）
  - alert: nginxconnectionexhaustionwarning
    expr: |
      (nginx_vts_server_connections_active{server="example.com"} 
       / nginx_vts_server_worker_connections{server="example.com"}) * 100 > 80
    for: 2m
    labels:
      severity: warning
      team: infra
    annotations:
      summary: "nginx connection usage > 80% on {{ $labels.instance }}"
      description: "active connections: {{ $value }}%. check upstream latency or increase worker_connections."
  # 规则2：上游服务全宕机（严重！）
  - alert: upstreamalldown
    expr: |
      count by (upstream) (
        nginx_vts_upstream_peer_state{upstream="backend-api", state="down"}
      ) == count by (upstream) (
        nginx_vts_upstream_peer_state{upstream="backend-api"}
      )
    for: 1m
    labels:
      severity: critical
      team: backend
    annotations:
      summary: "all peers down in upstream {{ $labels.upstream }}"
      description: "no healthy backend instances available. immediate investigation required!"
  # 规则3：5xx 错误率突增（注意！）
  - alert: highnginx5xxrate
    expr: |
      sum(rate(nginx_vts_server_responses_total{code="5xx", server="example.com"}[5m])) 
      / 
      sum(rate(nginx_vts_server_request_counter_total{server="example.com"}[5m])) > 0.05
    for: 3m
    labels:
      severity: warning
      team: sre
    annotations:
      summary: "5xx error rate > 5% for 3 minutes on {{ $labels.instance }}"
      description: "current rate: {{ $value | humanize }}%. correlate with upstream and application metrics."

global:
  resolve_timeout: 5m
route:
  group_by: ['alertname', 'team']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 1h
  receiver: 'wechat-notifications'
receivers:
- name: 'wechat-notifications'
  wechat_configs:
  - send_resolved: true
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=your_webhook_key'  # 替换为你的企业微信机器人 key
    message: |
      {{ define "__text_alert_list" }}{{ range .alerts }}
      [{{ .labels.severity }}] {{ .labels.alertname }}
      {{ .annotations.summary }}
      {{ .annotations.description }}
      details: {{ .generatorurl }}
      {{ end }}{{ end }}
      {{ if gt (len .alerts.firing) 0 -}}
      🔥 firing:
      {{ template "__text_alert_list" .alerts.firing }}
      {{ end }}
      {{ if gt (len .alerts.resolved) 0 -}}
      ✅ resolved:
      {{ template "__text_alert_list" .alerts.resolved }}
      {{ end }}

./alertmanager --config.file=alertmanager.yml --web.external-url=http://localhost:9093

alerting:
  alertmanagers:
  - static_configs:
    - targets: ['localhost:9093']

# helm install ingress-nginx ingress-nginx/ingress-nginx \
#   --set controller.metrics.enabled=true \
#   --set controller.metrics.servicemonitor.enabled=true \
#   --set controller.stats.enabled=true

# 查看所有 ingress 的 5xx 错误率（按 namespace 分组）
sum by (namespace, ingress) (
  rate(nginx_ingress_controller_requests_total{status=~"5.."}[5m])
)
/
sum by (namespace, ingress) (
  rate(nginx_ingress_controller_requests_total[5m])
)

# 检测 ssl 证书即将过期（< 7 天）
min by (host) (
  nginx_ingress_controller_ssl_expire_time_seconds - time()
) < 604800

- action: labeldrop
  regex: "(exported_instance|job)"