基于Nginx+Keepalived实现高可用(主从切换)配置指南

sestatus | grep "current mode"
# 输出应为：current mode: permissive

# 安装依赖
sudo dnf groupinstall "development tools" -y
sudo dnf install pcre-devel zlib-devel openssl-devel wget tar gcc make -y

# 下载并编译 nginx（以 1.24.0 为例）
cd /tmp
wget https://nginx.org/download/nginx-1.24.0.tar.gz
tar -zxvf nginx-1.24.0.tar.gz
cd nginx-1.24.0

./configure \
  --prefix=/usr/local/nginx \
  --sbin-path=/usr/local/nginx/sbin/nginx \
  --conf-path=/usr/local/nginx/conf/nginx.conf \
  --pid-path=/usr/local/nginx/logs/nginx.pid \
  --with-http_ssl_module \
  --with-http_v2_module \
  --with-http_realip_module \
  --with-http_stub_status_module \
  --with-stream \
  --with-stream_ssl_module \
  --with-compat

make && sudo make install

/usr/local/nginx/sbin/nginx -v 2>&1 | grep -e "(ssl|http_v2|stream)"
# 应输出包含 ssl, http_v2, stream 字样

sudo dnf install keepalived -y
# 启用开机自启（但先不启动）
sudo systemctl enable keepalived

# 全局块
user  nginx;
worker_processes  auto;
worker_rlimit_nofile 65535;
events {
    use epoll;
    worker_connections  65535;
    multi_accept on;
}
# http 块：处理 web 流量
http {
    include       mime.types;
    default_type  application/octet-stream;
    # 日志格式增强：记录 upstream 响应时间、状态、地址
    log_format main '$remote_addr - $remote_user [$time_local] '
                      '"$request" $status $body_bytes_sent '
                      '"$http_referer" "$http_user_agent" '
                      'rt=$request_time uct="$upstream_connect_time" '
                      'uht="$upstream_header_time" urt="$upstream_response_time" '
                      'upstream_addr=$upstream_addr';
    access_log  /usr/local/nginx/logs/access.log  main;
    error_log   /usr/local/nginx/logs/error.log  warn;
    sendfile        on;
    tcp_nopush      on;
    tcp_nodelay     on;
    keepalive_timeout  65;
    types_hash_max_size 2048;
    # 启用健康检查状态页（供 keepalived 脚本调用）
    server {
        listen 127.0.0.1:8081;
        location /healthz {
            return 200 "ok\n";
            add_header content-type text/plain;
        }
        # 检查 upstream 是否可达（curl 后端 java 服务）
        location /backend-health {
            proxy_pass http://192.168.56.20:8080/actuator/health;
            proxy_set_header host $host;
            proxy_set_header x-real-ip $remote_addr;
            proxy_read_timeout 3;
            proxy_connect_timeout 3;
        }
    }
    # 主业务 server 块
    server {
        listen       80;
        server_name  _;
        # 静态资源缓存
        location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg)$ {
            expires 1y;
            add_header cache-control "public, immutable";
        }
        # 反向代理到 java 后端
        location / {
            proxy_pass http://192.168.56.20:8080;
            proxy_set_header host $host;
            proxy_set_header x-real-ip $remote_addr;
            proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
            proxy_set_header x-forwarded-proto $scheme;
            # 超时设置（防止长连接阻塞）
            proxy_connect_timeout 5;
            proxy_send_timeout 30;
            proxy_read_timeout 30;
            # 传递真实客户端 ip（配合 real_ip 模块）
            set_real_ip_from 192.168.56.0/24;
            real_ip_header x-forwarded-for;
        }
    }
}

# 创建 nginx 用户（若不存在）
sudo useradd -r -s /sbin/nologin nginx

# 创建日志目录
sudo mkdir -p /usr/local/nginx/logs

# 启动 nginx
sudo /usr/local/nginx/sbin/nginx

# 验证配置语法 & 端口监听
sudo /usr/local/nginx/sbin/nginx -t
sudo ss -tlnp | grep ':80\|:8081'

# 测试健康接口
curl http://127.0.0.1:8081/healthz    # 应返回 ok
curl http://127.0.0.1:8081/backend-health  # 应返回 {"status":"up"}

! configuration file for keepalived
global_defs {
    # 路由器标识（每台机器唯一，用于日志区分）
    router_id nginx_master_01
    # 通知邮箱（可选，生产环境建议对接企业微信/钉钉）
    # notification_email {
    #    admin@example.com
    # }
    # smtp_server 127.0.0.1
    # smtp_connect_timeout 30
}
# 自定义健康检查脚本（关键！）
vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2                    # 每2秒执行一次
    weight 2                      # 成功时权重+2，失败时-2
    fall 2                        # 连续2次失败才判定为 down
    rise 2                        # 连续2次成功才判定为 up
}
# vrrp 实例定义
vrrp_instance vi_1 {
    state master                    # 当前节点角色：master
    interface eth0                  # vip 绑定的业务网卡（非心跳网卡！）
    virtual_router_id 51            # vrid，主备必须一致（1-255）
    priority 100                    # 优先级，越高越可能成为 master（backup 设为 90）
    advert_int 1                    # vrrp 通告间隔（秒）
    authentication {
        auth_type pass
        auth_pass 1111              # 认证密码（主备必须一致）
    }
    # 虚拟 ip（vip）配置
    virtual_ipaddress {
        192.168.56.100/24 dev eth0 label eth0:1  # vip + 子网掩码 + 绑定网卡别名
    }
    # 跟踪脚本：当 chk_nginx 失败时，降低本节点优先级，触发切换
    track_script {
        chk_nginx
    }
    # 防脑裂：当检测到与 backup 心跳中断时，执行强制降级
    # （需配合独立心跳网卡 eth1）
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

! configuration file for keepalived
global_defs {
    router_id nginx_backup_01
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight 2
    fall 2
    rise 2
}
vrrp_instance vi_1 {
    state backup                    # 角色为 backup
    interface eth0
    virtual_router_id 51            # 必须与 master 一致！
    priority 90                       # 低于 master，确保初始为 backup
    advert_int 1
    authentication {
        auth_type pass
        auth_pass 1111              # 密码必须一致！
    }
    virtual_ipaddress {
        192.168.56.100/24 dev eth0 label eth0:1
    }
    track_script {
        chk_nginx
    }
    # 关键：启用非抢占模式！
    # 一旦 backup 升为 master，即使原 master 恢复也不抢回 vip
    nopreempt
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

#!/bin/bash
# /etc/keepalived/check_nginx.sh
# 功能：双重健康检查
# 1. 检查 nginx master 进程是否存在
# 2. 检查 nginx 能否通过 /backend-health 接口连通 java 后端

nginx_pid_file="/usr/local/nginx/logs/nginx.pid"
nginx_health_url="http://127.0.0.1:8081/backend-health"
timeout=3

# 检查 nginx 进程
if ! kill -0 $(cat $nginx_pid_file 2>/dev/null) 2>/dev/null; then
    echo "[error] nginx process not running"
    exit 1
fi

# 检查 nginx 健康接口（端到端）
if ! curl -s --max-time $timeout -f $nginx_health_url >/dev/null 2>&1; then
    echo "[error] nginx /backend-health check failed"
    exit 1
fi

# 检查 nginx 自身健康（快速兜底）
if ! curl -s --max-time $timeout -f http://127.0.0.1:8081/healthz >/dev/null 2>&1; then
    echo "[error] nginx /healthz check failed"
    exit 1
fi

echo "[ok] all health checks passed"
exit 0

sudo chmod +x /etc/keepalived/check_nginx.sh
sudo chown root:root /etc/keepalived/check_nginx.sh

#!/bin/bash
# /etc/keepalived/notify.sh
# 参数：$1 = master|backup|fault

role=$1
timestamp=$(date '+%y-%m-%d %h:%m:%s')
vip="192.168.56.100"
node_ip=$(hostname -i | awk '{print $1}')

log_file="/var/log/keepalived-state.log"

echo "[$timestamp] keepalived $role on $(hostname) ($node_ip). vip: $vip" >> $log_file

case "$role" in
    "master")
        echo "[$timestamp] $(hostname) is now master. enabling vip..." >> $log_file
        # 可在此处添加：启动特定服务、更新 dns、发送企业微信消息等
        ;;
    "backup")
        echo "[$timestamp] $(hostname) is now backup. vip released." >> $log_file
        ;;
    "fault")
        echo "[$timestamp] $(hostname) entered fault state! check network & nginx." >> $log_file
        ;;
esac

sudo tail -f /var/log/keepalived-state.log

# 编辑 /etc/sysctl.conf
echo "
# keepalived vip arp 优化
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.eth0.arp_ignore = 1
net.ipv4.conf.eth0.arp_announce = 2
" | sudo tee -a /etc/sysctl.conf

# 生效配置
sudo sysctl -p

<dependencies>
    <!-- web -->
    <dependency>
        <groupid>org.springframework.boot</groupid>
        <artifactid>spring-boot-starter-web</artifactid>
    </dependency>
    <!-- actuator（健康检查核心） -->
    <dependency>
        <groupid>org.springframework.boot</groupid>
        <artifactid>spring-boot-starter-actuator</artifactid>
    </dependency>
    <!-- 可选：暴露所有端点（生产环境请按需开放） -->
    <dependency>
        <groupid>org.springframework.boot</groupid>
        <artifactid>spring-boot-devtools</artifactid>
        <scope>runtime</scope>
        <optional>true</optional>
    </dependency>
</dependencies>

management:
  endpoints:
    web:
      exposure:
        include: health,info,metrics,prometheus  # 至少暴露 health
  endpoint:
    health:
      show-details: when_authorized  # 生产环境建议设为 never 或 when_authorized
      probes:
        enabled: true
  server:
    port: 8081  # actuator 独立端口（与业务端口 8080 分离）
# 业务服务器配置
server:
  port: 8080
  address: 0.0.0.0
spring:
  application:
    name: demo-java-service

package com.example.demo.health;

import org.springframework.boot.actuate.health.health;
import org.springframework.boot.actuate.health.healthindicator;
import org.springframework.stereotype.component;

import java.util.concurrent.atomic.atomicboolean;

@component
public class databasehealthindicator implements healthindicator {

    private final atomicboolean dbavailable = new atomicboolean(true);

    @override
    public health health() {
        if (dbavailable.get()) {
            return health.up()
                    .withdetail("database", "connected")
                    .withdetail("version", "postgresql 14.5")
                    .build();
        } else {
            return health.down()
                    .withdetail("error", "database connection failed")
                    .build();
        }
    }
}

# 打包并运行（假设 jar 名为 demo.jar）
java -jar demo.jar

# 验证健康接口（应在 nginx 的 /backend-health 中被调用）
curl http://192.168.56.20:8081/actuator/health
# 返回示例：
# {"status":"up","components":{"diskspace":{"status":"up","details":{"total":53660876800,"free":42847272960,"threshold":10485760}},"ping":{"status":"up"}}}

# 在 node a（master）上执行
sudo pkill -f "nginx: master"

# 观察日志
sudo tail -f /var/log/keepalived-state.log
# 应看到：... entered fault state! ... 然后 ... is now backup.

# 同时在 node b（backup）日志中应看到：
# ... is now master. enabling vip...

# 验证 vip 是否已漂移到 node b
ip addr show eth0 | grep 192.168.56.100
# node b 应输出：inet 192.168.56.100/24 scope global secondary eth0:1

# 从客户端（如另一台机器）curl vip
curl http://192.168.56.100
# 应正常返回 java 应用首页，毫秒级无感切换！ ✅

# 在 java 服务节点（192.168.56.20）上停止应用
pkill -f "demo.jar"

# 等待 4 秒（2次失败 × 2秒间隔）
# 此时 keepalived 会因 /backend-health 失败而降权

# 查看 node a 的优先级变化
sudo ipvsadm -ln | grep "192.168.56.100"
# 或查看 keepalived 进程日志：
sudo journalctl -u keepalived -n 20 --no-pager

# 预期：node a 优先级降至 98（100-2），node b 仍为 90，但若 node a 降为 98 < node b 90？不会切换！
# 因此，我们需让 node b 优先级更高（如设为 95），或增加 weight（如设为 5）

# ✅ 正确做法：在 chk_nginx.sh 中，失败时 exit 1，keepalived 会立即触发 vrrp_script 失败逻辑，
# 并结合 priority + weight 实现平滑降级。

# /etc/rsyslog.d/keepalived.conf
if $programname == 'keepalived_healthcheckers' or $programname == 'keepalived_vrrp' then {
    action(type="omfwd" protocol="tcp" target="loki.example.com" port="3100" template="rsyslog_syslogprotocol23format")
    stop
}

server {
    listen 443 ssl http2;
    server_name example.com;
    ssl_certificate /etc/ssl/certs/fullchain.pem;
    ssl_certificate_key /etc/ssl/private/privkey.pem;
    ssl_protocols tlsv1.2 tlsv1.3;
    ssl_ciphers ecdhe-ecdsa-aes128-gcm-sha256:ecdhe-rsa-aes128-gcm-sha256;
    location / {
        proxy_pass http://192.168.56.20:8080;
        # ... 其他 proxy_* 设置
    }
}

# 使用 nginx-upstream-check-module（需重新编译）
upstream backend_java {
    server 192.168.56.20:8080 max_fails=3 fail_timeout=30s;
    server 192.168.56.21:8080 max_fails=3 fail_timeout=30s;
    check interval=3 rise=2 fall=5 timeout=10 type=http;
    check_http_send "head /actuator/health http/1.0\r\n\r\n";
    check_http_expect_alive http_2xx http_3xx;
}

# 实时跟踪 keepalived 状态
sudo journalctl -u keepalived -f

# 查看 vrrp 详细状态
sudo ipvsadm -ln

# 抓包分析 vrrp 心跳（在 eth1 上）
sudo tcpdump -i eth1 -n vrrp -c 10