Nginx本地缓存瓶颈怎么破？外置缓存方案详解

客户端 → nginx(openresty) → [redis cluster] → 源站
              ↓                    ↑
         lua缓存逻辑          分布式存储
         (get/set/stale)      (一致性哈希)

# 安装openresty（内置luajit + ngx_lua模块）
wget https://openresty.org/package/openresty-1.27.1.tar.gz
tar xzf openresty-1.27.1.tar.gz && cd openresty-1.27.1
./configure --with-luajit --with-http_redis2_module --with-http_lua_module
make && make install

# 安装lua-resty-redis库（若未内置）
luarocks install lua-resty-redis

local redis = require "resty.redis"
local cjson = require "cjson.safe"

local _m = {}

-- redis连接池配置
local redis_conf = {
    host = "redis-cluster.internal",
    port = 6379,
    pool_size = 100,        -- 每worker连接池大小
    backlog = 200,          -- 等待队列
    connect_timeout = 100,  -- ms
    read_timeout = 200,     -- ms
}

-- 获取redis连接（带连接池）
local function get_redis()
    local red = redis:new()
    red:set_timeouts(redis_conf.connect_timeout, 
                     redis_conf.read_timeout, 
                     redis_conf.read_timeout)
    
    local ok, err = red:connect(redis_conf.host, redis_conf.port)
    if not ok then
        ngx.log(ngx.err, "redis connect failed: ", err)
        return nil, err
    end
    
    return red, nil
end

-- 释放连接到池中
local function release_redis(red)
    local ok, err = red:set_keepalive(10000, redis_conf.pool_size)
    if not ok then
        ngx.log(ngx.err, "redis keepalive failed: ", err)
    end
end

-- 读取缓存
function _m.get(key)
    local red, err = get_redis()
    if not red then return nil, err end
    
    local res, err = red:get(key)
    release_redis(red)
    
    if not res or res == ngx.null then
        return nil, "miss"
    end
    
    return cjson.decode(res), nil
end

-- 写入缓存（带ttl）
function _m.set(key, value, ttl)
    local red, err = get_redis()
    if not red then return false, err end
    
    local encoded = cjson.encode(value)
    local ok, err = red:setex(key, ttl, encoded)
    release_redis(red)
    
    return ok ~= nil, err
end

-- 删除缓存
function _m.delete(key)
    local red, err = get_redis()
    if not red then return false, err end
    
    local ok, err = red:del(key)
    release_redis(red)
    
    return ok ~= nil, err
end

return _m

http {
    # lua共享字典：用于本地二级缓存和锁
    lua_shared_dict local_cache 100m;
    lua_shared_dict cache_locks 10m;
    init_by_lua_block {
        cache_handler = require "cache_handler"
    }
    server {
        listen 80;
        location /api/ {
            content_by_lua_block {
                local key = ngx.var.scheme .. ":" .. ngx.var.host .. ngx.var.uri
                -- 第1层：本地共享字典缓存（微秒级）
                local local_cache = ngx.shared.local_cache
                local val = local_cache:get(key)
                if val then
                    ngx.header["x-cache"] = "local-hit"
                    ngx.say(val)
                    return
                end
                -- 第2层：redis外置缓存
                local data, err = cache_handler.get(key)
                if data then
                    -- 回填本地缓存（短ttl，防热点穿透）
                    local_cache:set(key, cjson.encode(data), 5)
                    ngx.header["x-cache"] = "redis-hit"
                    ngx.say(cjson.encode(data))
                    return
                end
                -- 第3层：缓存击穿防护（分布式锁）
                local locks = ngx.shared.cache_locks
                local lock_key = "lock:" .. key
                local elapsed, err = locks:add(lock_key, true, 3)
                if elapsed then
                    -- 获得锁，回源
                    local res = ngx.location.capture("/internal/backend")
                    if res.status == 200 then
                        local body = res.body
                        -- 写入redis（长ttl）
                        cache_handler.set(key, cjson.decode(body), 300)
                        -- 写入本地缓存
                        local_cache:set(key, body, 5)
                        ngx.header["x-cache"] = "miss"
                        ngx.say(body)
                    else
                        ngx.status = res.status
                        ngx.say(res.body)
                    end
                    locks:delete(lock_key)
                else
                    -- 未获得锁，短暂等待后重试或直接回源
                    ngx.sleep(0.1)
                    local retry_data = cache_handler.get(key)
                    if retry_data then
                        ngx.header["x-cache"] = "lock-wait-hit"
                        ngx.say(cjson.encode(retry_data))
                    else
                        -- 降级：直接回源（不缓存）
                        local res = ngx.location.capture("/internal/backend")
                        ngx.header["x-cache"] = "bypass"
                        ngx.status = res.status
                        ngx.say(res.body)
                    end
                end
            }
        }
        # 内部回源location
        location /internal/backend {
            internal;
            proxy_pass http://backend;
            proxy_set_header host $host;
        }
    }
}

-- ❌ 错误：每次请求新建连接
local red = redis:new()
red:connect(...)
-- 请求结束连接销毁
-- ✅ 正确：使用set_keepalive复用连接
red:set_keepalive(10000, 100)  -- 空闲10s，池大小100

-- ✅ 推荐：命名空间 + 版本 + 业务标识
local key = "api:v2:user:profile:" .. user_id

-- ❌ 避免：直接使用uri
local key = ngx.var.uri  -- 参数顺序变化导致重复缓存

-- redis不可用时，降级到本地缓存或直接回源
local data, err = cache_handler.get(key)
if err and (err == "timeout" or err == "connection refused") then
    ngx.log(ngx.warn, "redis degraded, fallback to backend")
    -- 跳过缓存，直接回源
    -- 或尝试读取过期的本地缓存作为兜底
end

-- 按前缀批量删除（redis scan + del，非keys）
function _m.delete_pattern(pattern)
    local red, err = get_redis()
    if not red then return false, err end
    local cursor = "0"
    repeat
        local res, err = red:scan(cursor, "match", pattern, "count", 100)
        if not res then break end
        cursor = res[1]
        local keys = res[2]
        if #keys > 0 then
            red:del(unpack(keys))
        end
    until cursor == "0"
    release_redis(red)
    return true, nil
end