Python实现MySQL备（tuo）份（ku）脚本

  1 #!/usr/bin/python2.7
  2 # -*- coding: utf-8 -*-
  3 import MySQLdb
  4 import time
  5 import threadpool
  6 import sys
  7 import copy
  8 import threading
  9 import os
 10 import datetime
 11 
 12 reload(sys)
 13 sys.setdefaultencoding("utf8")
 14 mutex = threading.Lock()
 15 
 16 #-----------------------参数列表--------------------------------
 17 # 数据库连接参数
 18 host= "127.0.0.1"
 19 port = 3306
 20 user= "root"
 21 passwd= ""
 22 database = ‘performance_schema‘
 23 charset= ‘utf8‘
 24 
 25 # 数据库文件存储位置
 26 file_path = ‘C:/Users/lulu/Desktop/performance_schema/‘
 27 # 日志文件存储位置
 28 log_path = file_path + ‘log/‘
 29 # 如果需要单拖某些表，可写在此列表中，如果为空，表示拖所有的表（weilai的表示例为 u‘sp_bank‘）
 30 dumptablelist = ()
 31 # 一次拖的行数
 32 once_rows = 100
 33 # 开始爬取的索引
 34 start_lable = 0
 35 # 线程数
 36 thread_num = 1
 37 # 分隔符
 38 divide_char = ‘,‘
 39 #================================================================
 40 
 41 def conndb():
 42     try:
 43         db = MySQLdb.connect(
 44             host= host,
 45             port = port,
 46             user= user,
 47             passwd= passwd,
 48             db = database,
 49             charset= charset
 50             )
 51     except MySQLdb.Error:
 52         errorsleep(‘Connect Error..reconnect...‘, 3)
 53         return conndb()
 54     return db
 55 
 56 def errorsleep(errorstr, delay = 0):
 57     print errorstr
 58     time.sleep(delay)
 59 
 60 def get_tabledata(mylist):
 61 
 62     table_name     = mylist[0]
 63     lable         = mylist[1]
 64     once_rows     = mylist[2]
 65 
 66     dberror = 0
 67     encodeerror = 0
 68     db = None
 69     cur = None
 70     file_name = file_path + table_name + ‘.txt‘
 71     file_fp = open(file_name, "a")
 72     log_name = log_path + table_name + ‘.log‘
 73     log_fp = open(log_name, "a")
 74 
 75     try:
 76 
 77         db = conndb()
 78         cur = db.cursor()
 79 
 80         cur.execute("SELECT * FROM "+ table_name + " limit " + str(lable) + "," + str(once_rows))
 81         print "SELECT * FROM "+ table_name + " limit " + str(lable) + "," + str(once_rows)
 82 
 83         res = cur.fetchall()
 84         results_s = list(res)
 85 
 86         sum = ""
 87         for it in results_s:
 88             for i in it:
 89                 sum += "%s" % i
 90                 sum += divide_char
 91 
 92             mutex.acquire()
 93             file_fp.write(sum + ‘\n‘)
 94             file_fp.flush()
 95             mutex.release()
 96             sum = ""
 97 
 98         log_fp.write(str(datetime.datetime.now()) + ‘ -- Finished :  from ‘ + str(lable) + ‘ to ‘ + str(lable+once_rows) + ‘\n‘)
 99 
100     except MySQLdb.Error:
101         errorsleep(‘Download Error..reload...‘, 3)
102         dberror = 1
103     except UnicodeDecodeError:
104         print str(lable) + ‘ Error..retry...‘
105         encodeerror = 1
106 
107     if file_fp != None:
108         file_fp.close()
109     if log_fp != None:
110         log_fp.close()
111     if db != None:
112         db.close()
113     if cur != None:
114         cur.close()
115 
116     if dberror == 1:
117         return get_tabledata(mylist)
118     elif encodeerror == 1:
119         return get_tabledata(mylist)
120     else:
121         return
122 
123 def get_alltable(cur):
124     # 获取所有的表
125     cur.execute("SHOW TABLES")
126     return list(cur.fetchall())
127 
128 def get_column(cur,table):
129     # 获取table的字段
130     cur.execute("SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE table_name = ‘" + table + "‘ AND table_schema = ‘" + database + "‘")
131     results_col_name = list(cur.fetchall())
132     return results_col_name     # 改成list
133 
134 def get_tablerows(cur,table):
135     # 获取table的行数
136     cur.execute("SELECT COUNT(*) FROM " + table)
137     results_cows = list(cur.fetchall())
138     rows_num = (results_cows[0])[0]
139     return rows_num
140 
141 if __name__ == ‘__main__‘:
142 
143     db = conndb()
144     cur = db.cursor()
145 
146     tables = get_alltable(cur)
147     table_num = len(tables)
148 
149     for i in range(table_num):
150 
151         table_name = (tables[i])[0]
152 
153         if (len(dumptablelist) != 0) and (table_name not in dumptablelist):
154             continue
155 
156         # 打开数据库存储文件
157         if not os.path.exists(file_path):
158             os.makedirs(file_path)
159         file_fp = ‘‘
160         file_name = file_path + table_name + ‘.txt‘
161         file_fp = open(file_name, "a")
162 
163         # 打开对应日志文件
164         if not os.path.exists(log_path):
165             os.makedirs(log_path)
166         log_fp = ‘‘
167         log_name = log_path + table_name + ‘.log‘
168         log_fp = open(log_name, "a")
169 
170         results_col_name = get_column(cur,table_name)
171 
172         # 将表的字段写入文件开头
173         col_name = ‘‘
174         for num_col_name in range(len(results_col_name)):
175             col_name += (results_col_name[num_col_name])[0]
176             col_name += ‘,‘
177 
178         file_fp.write( ‘(‘ + col_name + ‘)\n‘)
179         file_fp.close()
180 
181         rows_num = get_tablerows(cur,table_name)
182 
183         log_fp.write(‘Total rows num : ‘ + str(rows_num) + ‘\n‘)
184         log_fp.close()
185 
186         if rows_num != 0:
187 
188             lable = start_lable   # 开始爬取的索引
189             times = (rows_num-lable)/once_rows + 1
190 
191             arglist = []
192             for a in range(times):
193                 arglist.append([table_name, lable, once_rows])
194                 lable += once_rows
195 
196             pool = threadpool.ThreadPool(thread_num)
197             requests = threadpool.makeRequests(get_tabledata,copy.copy(arglist))
198             [pool.putRequest(req) for req in requests]
199             pool.wait()
200 
201         print ‘$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$‘
202         print ‘------------- ‘ + table_name + ‘ is done!! ---------------‘
203         print ‘$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$‘