Redis中redis-shake实现数据迁移同步

# ===================== 内核参数 ============================
连接数
cat >> /etc/security/limits.conf << eof
* soft nofile 65535
* hard nofile 65535
* soft nproc 65535
* hard nproc 65535 
eof

cat >>  /etc/sysctl.conf  << eof
vm.overcommit_memory = 1
net.ipv4.tcp_max_tw_buckets = 150000
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 9000 65500
eof

# ====================== 基础配置 ===================================
setenforce 0 && sed -i 's/^selinux=enforcing/selinux=disabled/g' /etc/selinux/config && getenforce

systemctl disable firewalld && systemctl stop firewalld

useradd -s /sbin/nologin redis

# 下载 redis-shake
wget https://github.com/tair-opensource/redisshake/releases/download/v4.2.0/redis-shake-linux-amd64.tar.gz

# 解压 redis-shake
tar xf redis-shake-linux-amd64.tar.gz

# 创建目录
mkdir -pv /usr/local/redis-shake/config

vim /usr/local/redis-shake/config/shake.toml

[sync_reader]
cluster = true            # set to true if source is a redis cluster
address = 192.168.1.1:6379" # when cluster is true, set address to one of the cluster node
username = ""              # keep empty if not using acl
password = "xxxxx"              # keep empty if no authentication is required
tls = false                #
sync_rdb = true            # set to false if you don't want to sync rdb
sync_aof = true          # set to false if you don't want to sync aof
prefer_replica = false     # set to true if you want to sync from replica node
try_diskless = false       # set to true if you want to sync by socket and source repl-diskless-sync=yes

#[scan_reader]
#cluster = false            # set to true if source is a redis cluster
#address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node
#username = ""              # keep empty if not using acl
#password = ""              # keep empty if no authentication is required
#tls = false
#dbs = []                   # set you want to scan dbs such as [1,5,7], if you don't want to scan all
#scan = true                # set to false if you don't want to scan keys
#ksn = false                # set to true to enabled redis keyspace notifications (ksn) subscription
#count = 1                  # number of keys to scan per iteration

# [rdb_reader]
# filepath = "/tmp/dump.rdb"

# [aof_reader]
# filepath = "/tmp/.aof"
# timestamp = 0            # subsecond

[redis_writer]
cluster = true            # set to true if target is a redis cluster
sentinel = false           # set to true if target is a redis sentinel
master = ""                # set to master name if target is a redis sentinel
address = "192.168.1.2:6379" # when cluster is true, set address to one of the cluster node
username = ""              # keep empty if not using acl
password = "xxxxx"              # keep empty if no authentication is required
tls = false
off_reply = false          # turn off the server reply

[filter]
# allow keys with specific prefixes or suffixes
# examples:
#   allow_key_prefix = ["user:", "product:"]
#   allow_key_suffix = [":active", ":valid"]
# leave empty to allow all keys
allow_key_prefix = []
allow_key_suffix = []

# block keys with specific prefixes or suffixes
# examples:
#   block_key_prefix = ["temp:", "cache:"]
#   block_key_suffix = [":tmp", ":old"]
# leave empty to block nothing
block_key_prefix = []
block_key_suffix = []

# specify allowed and blocked database numbers (e.g., allow_db = [0, 1, 2], block_db = [3, 4, 5])
# leave empty to allow all databases
allow_db = []
block_command = []

# allow or block specific command groups
# available groups:
#   server, string, cluster, connection, bitmap, list, sorted_set,
#   generic, transactions, scripting, tairhash, tairstring, tairzset,
#   geo, hash, hyperloglog, pubsub, set, sentinel, stream
# examples:
#   allow_command_group = ["string", "hash"]  # only allow string and hash commands
#   block_command_group = ["scripting", "pubsub"]  # block scripting and pubsub commands
# leave empty to allow all command groups
allow_command_group = []
block_command_group = []

# function for custom data processing
# for best practices and examples, visit:
# https://tair-opensource.github.io/redisshake/zh/function/best_practices.html
function = ""

[advanced]
dir = "/usr/local/redis-shake/data"
ncpu = 0        # runtime.gomaxprocs, 0 means use runtime.numcpu() cpu cores
pprof_port = 0  # pprof port, 0 means disable
status_port = 0 # status port, 0 means disable

# log
log_file = "shake.log"
log_level = "info"     # debug, info or warn
log_interval = 5       # in seconds

# redis-shake gets key and value from rdb file, and uses restore command to
# create the key in target redis. redis restore will return a "target key name
# is busy" error when key already exists. you can use this configuration item
# to change the default behavior of restore:
# panic:   redis-shake will stop when meet "target key name is busy" error.
# rewrite: redis-shake will replace the key with new value.
# skip:  redis-shake will skip restore the key when meet "target key name is busy" error.
rdb_restore_command_behavior = "panic" # panic, rewrite or skip

# redis-shake uses pipeline to improve sending performance.
# adjust this value based on the destination redis performance:
# - higher values may improve performance for capable destinations.
# - lower values are recommended for destinations with poor performance.
# 1024 is a good default value for most cases.
pipeline_count_limit = 1024

# this setting corresponds to the 'client-query-buffer-limit' in redis configuration.
# the default value is typically 1gb.
# it's recommended not to modify this value unless absolutely necessary.
target_redis_client_max_querybuf_len = 1073741824  # 1gb in bytes

# this setting corresponds to the 'proto-max-bulk-len' in redis configuration.
# it defines the maximum size of a single string element in the redis protocol.
# the value must be 1mb or greater. default is 512mb.
# it's recommended not to modify this value unless absolutely necessary.
target_redis_proto_max_bulk_len = 512_000_000

# if the source is elasticache, you can set this item. aws elasticache has custom
# psync command, which can be obtained through a ticket.
aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync"

# destination will delete itself entire database before fetching files
# from source during full synchronization.
# this option is similar redis replicas rdb diskless load option:
#   repl-diskless-load on-empty-db
empty_db_before_sync = true

[module]
# the data format for bf.loadchunk is not compatible in different versions. v2.6.3 <=> 20603
target_mbbloom_version = 20603

vim /usr/lib/systemd/system/redis-shake.service

[unit]
description=redis-shake
after=data.mount

[service]
type=simple
execstart=/usr/local/redis-shake/redis-shake  /usr/local/redis-shake/config/shake.toml
execstop=/bin/kill -sigterm $mainpid
privatetmp=true
user=redis
group=redis

[install]
wantedby=multi-user.target

systemctl start redis-shake
systemctl enable redis-shake