迁移单点redis数据到RedisCluster

#!/usr/bin/env python3.7.4
# -*- coding:utf-8 -*-
# --------------------
# File: rds2rdsCluster.py
# Project: 将单点redis的数据迁移到cluster
# Author: Aven
# Last Modified: 2020-07-09 14:20:29
# 安装依赖命令如下:
# pip3 install redis-py-cluster
# pip3 install redis
# --------------------

import sys
import redis
from rediscluster import RedisCluster
import logging
from datetime import datetime as dt
from pathlib import Path

# init log
logger = logging.getLogger()
formatter = logging.Formatter(fmt="%(asctime)s %(levelname)s - %(message)s", datefmt="[%Y/%m/%d %H:%M:%S]")
logger.setLevel(logging.INFO)
# file log
base_path = Path(__file__).parent
log_dir = base_path / ‘logs‘
log_dir.mkdir(parents=True, exist_ok=True)
filename = dt.strftime(dt.now(), ‘%Y-%m-%d.log‘)
handler = logging.FileHandler(log_dir/filename, ‘a‘, ‘utf-8‘)
handler.setFormatter(formatter)
logger.addHandler(handler)
# console log
console = logging.StreamHandler()
console.setLevel(logging.INFO)
console.setFormatter(formatter)
logger.addHandler(console)

sourceReidsHost=‘192.168.1.16‘
sourceReidsPort=6379
sourceReidsDatabase=0
sourceReidsPassword=None
sourceRedis = redis.StrictRedis(host=sourceReidsHost, port=sourceReidsPort, db=sourceReidsDatabase,password=sourceReidsPassword)

targetRedisClusterNodes = [
   {‘host‘: ‘192.168.1.23‘, ‘port‘: 7001},
   {‘host‘: ‘192.168.1.23‘, ‘port‘: 7002},
   {‘host‘: ‘192.168.1.23‘, ‘port‘: 7003}
]
targetRedisClusterPassword = ‘123456‘

targetRedisCluster = RedisCluster(startup_nodes=targetRedisClusterNodes, decode_responses=True, password=targetRedisClusterPassword)
count = 0

patterns = [    
    (r‘hkey1:*‘,‘hash‘),    
    (r‘hkey2:*‘,‘hash‘),
    (r‘skey3:*‘,‘string‘)
]
#默认过期时间1个月
defaultRedisTimeoutSeconds=30*24*60*60
def copyData(kind,keys):
    if "hash"==kind:
        return copyHashData(keys)
    elif "string"==kind:
        return copyStringData(keys)

def copyStringData(keys):
    for key in keys:
        data=sourceRedis.get(key)
        targetRedisCluster.setex(key,defaultRedisTimeoutSeconds,data)
        logging.info(‘copy the key :{0},type:string‘.format(key))

def copyHashData(keys):
    for key in keys:
        data=sourceRedis.hgetall(key)
        targetRedisCluster.hmset(key,data)
        targetRedisCluster.expire(key,defaultRedisTimeoutSeconds)
        logging.info(‘copy the key :{0},type:hash‘.format(key))

logging.info ("Begin Scan From Redis({})..".format(sourceReidsHost))
cur=0
keys=[]
for pattern in patterns:
    while (1==1):
        count=count+1
        a, b = sourceRedis.scan(cursor=cur, match=pattern[0], count=1000)
        cur=a
        if(cur==0):
            #logging.info(‘the key pattern:{} has None Data ‘.format(pattern))
            break
        if(len(b)>0):
            #keys.append(b)
            keys=keys+b
            # copy the data to redisCluster
            copyData(pattern[1],b)
    
logging.info (‘Scaned Over! copied {0} keys‘.format(len(keys)))