Nginx外置缓存之nginx + memcached步骤详解

你的缓存需求？
├─ 需要复杂数据结构（list/hash/set） → redis
├─ 需要持久化/主从复制 → redis
├─ value > 1mb → redis
└─ 纯kv + 临时数据 + 极致低延迟 → memcached ✅
你的qps和延迟要求？
├─ qps < 5万 & p99 < 1ms → redis足够
├─ qps > 10万 & p99 < 0.3ms → memcached ✅
└─ cpu密集型序列化/反序列化 → memcached（多线程优势）

# 安装openresty（内置lua-resty-memcached）
wget https://openresty.org/package/openresty-1.27.1.tar.gz
tar xzf openresty-1.27.1.tar.gz && cd openresty-1.27.1
./configure --with-luajit --with-http_lua_module
make && make install
# 验证库可用
/usr/local/openresty/bin/resty -e 'print(require "resty.memcached")'

local memcached = require "resty.memcached"
local cjson = require "cjson.safe"
local _m = {}
-- memcached连接池配置
local mc_conf = {
    host = "memcached.internal",
    port = 11211,
    pool_size = 200,        -- 每worker连接池大小
    backlog = 500,          -- 等待队列
    connect_timeout = 100,  -- ms
    read_timeout = 200,     -- ms
}
-- 获取连接（带连接池）
local function get_mc()
    local mc, err = memcached:new()
    if not mc then
        ngx.log(ngx.err, "memcached new failed: ", err)
        return nil, err
    end
    mc:set_timeouts(mc_conf.connect_timeout, 
                    mc_conf.read_timeout, 
                    mc_conf.read_timeout)
    local ok, err = mc:connect(mc_conf.host, mc_conf.port)
    if not ok then
        ngx.log(ngx.err, "memcached connect failed: ", err)
        return nil, err
    end
    return mc, nil
end
-- 释放连接到池中
local function release_mc(mc)
    local ok, err = mc:set_keepalive(10000, mc_conf.pool_size)
    if not ok then
        ngx.log(ngx.err, "memcached keepalive failed: ", err)
    end
end
-- 读取缓存
function _m.get(key)
    local mc, err = get_mc()
    if not mc then return nil, err end
    local res, flags, err = mc:get(key)
    release_mc(mc)
    if not res then
        if err == "not found" then
            return nil, "miss"
        end
        return nil, err
    end
    return res, nil  -- memcached返回原始字符串，自行解码
end
-- 写入缓存（带ttl）
function _m.set(key, value, ttl)
    local mc, err = get_mc()
    if not mc then return false, err end
    local ok, err = mc:set(key, value, ttl or 300)
    release_mc(mc)
    return ok, err
end
-- 删除缓存
function _m.delete(key)
    local mc, err = get_mc()
    if not mc then return false, err end
    local ok, err = mc:delete(key)
    release_mc(mc)
    return ok, err
end
-- 原子递增（计数器场景）
function _m.incr(key, delta, init_ttl)
    local mc, err = get_mc()
    if not mc then return nil, err end
    -- 先尝试incr，若key不存在则set初始值
    local val, err = mc:incr(key, delta)
    if not val and err == "not found" then
        local ok, serr = mc:set(key, tostring(delta), init_ttl or 60)
        if ok then
            release_mc(mc)
            return delta, nil
        end
        release_mc(mc)
        return nil, serr
    end
    release_mc(mc)
    return val, err
end
return _m

http {
    lua_shared_dict mc_local_cache 50m;  -- l1本地缓存
    init_by_lua_block {
        mcache = require "mcache"
    }
    server {
        listen 80;
        location /api/session/ {
            content_by_lua_block {
                local session_id = ngx.var.cookie_session_id
                if not session_id then
                    ngx.status = 401
                    return ngx.say('{"error":"unauthorized"}')
                end
                local key = "sess:" .. session_id
                -- l1: 本地共享字典（微秒级）
                local local_cache = ngx.shared.mc_local_cache
                local val = local_cache:get(key)
                if val then
                    ngx.header["x-cache"] = "l1-hit"
                    ngx.say(val)
                    return
                end
                -- l2: memcached
                local data, err = mcache.get(key)
                if data then
                    local_cache:set(key, data, 3)  -- l1 ttl极短
                    ngx.header["x-cache"] = "mc-hit"
                    ngx.say(data)
                    return
                end
                -- miss: 回源
                local res = ngx.location.capture("/internal/session_backend")
                if res.status == 200 then
                    mcache.set(key, res.body, 1800)  -- 30分钟
                    local_cache:set(key, res.body, 3)
                    ngx.header["x-cache"] = "miss"
                    ngx.say(res.body)
                else
                    ngx.status = res.status
                    ngx.say(res.body)
                end
            }
        }
        # 计数器接口（利用memcached原子incr）
        location /api/counter/ {
            content_by_lua_block {
                local key = "cnt:" .. ngx.var.uri
                local val, err = mcache.incr(key, 1, 3600)
                if val then
                    ngx.say(tostring(val))
                else
                    ngx.log(ngx.err, "counter incr failed: ", err)
                    ngx.status = 500
                    ngx.say('{"error":"counter unavailable"}')
                end
            }
        }
        location /internal/session_backend {
            internal;
            proxy_pass http://session_service;
        }
    }
}

# /etc/sysconfig/memcached 或启动参数
options="-m 8192 -c 4096 -t 16 -i 2m -r 100 -o modern"

总并发连接 = nginx worker数 × pool_size
推荐值：worker=8, pool_size=200 → 总连接=1600
memcached -c 应 ≥ 1600 × 1.5 = 2400

-- ✅ 推荐：命名空间 + 业务标识 + 版本
local key = "sess:v2:" .. session_id
local key = "cnt:api:/users:list"
-- ❌ 避免：过长key（memcached限制250字节）
local key = "session:user:profile:data:" .. long_uuid  -- 可能超限
-- ❌ 避免：特殊字符（空格、换行、控制符）
local key = "key with space"  -- 协议解析错误

-- ✅ 直接存储二进制数据（如图片缩略图、protobuf）
mcache.set("thumb:" .. id, binary_data, 3600)
-- ✅ json文本也可直接存储
mcache.set("api:" .. key, cjson.encode(data), 300)
-- ⚠️ 注意：get返回的是原始字符串，需自行判断是否解码
local raw = mcache.get(key)
local data = cjson.decode(raw)  -- 若确定是json才解码

local data, err = mcache.get(key)
if err and (err == "timeout" or err == "connection refused") then
    ngx.log(ngx.warn, "memcached degraded: ", err)
    -- 降级：跳过缓存，直接回源
    -- 或返回预设默认值
    res = ngx.location.capture("/internal/backend")
    ngx.header["x-cache-degraded"] = "mc-unavailable"
    ngx.say(res.body)
    return
end

-- ✅ 批量读取
local keys = {"k1", "k2", "k3"}
local results, err = mc:get(keys)  -- 返回table
-- ❌ 无批量写入
-- 只能循环set，或使用pipeline（lua-resty-memcached不原生支持）