Linux内核补丁批量自动下载工具

#!/usr/bin/python
# -*- coding: utf-8 -*-
######################################################################
# Purpose:    auto download kernel module patch from offical web
# Useage:    ./patch_spider.py
#            example: patch_spider.py linux-3.10.y fs/ubifs 2013-08-15:2016-11-11
# Version:    Initial    Version    by Alex
######################################################################

import httplib, urllib, urllib2
import sys, os
import datetime
import re

DEBUG = 0
if DEBUG: from pprint import *

class PatchSpider(object):
    def __init__(self):
        pass

    def run(self):
        # main function
        print ‘get patches info...‘
        patch_ids = self.get_patch_id()
        print ‘saving patches into...‘
        ret = self.save_patch_file(patch_ids)    
        print ‘done‘
        
        return 0
        
    def get_patch_id(self):
        # get online data
        response_data = []
        urlstr = "http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/log/%s?h=%s&ofs=%d"
        dt_format = ‘%Y-%m-%d‘
        
        start, end = args[‘date-range‘].split(‘:‘)
        if not start.strip(): start = datetime.datetime.now().strftime(dt_format)
        if not end.strip(): end = datetime.datetime.now().strftime(dt_format)
        dt_start = datetime.datetime.strptime(start, dt_format)
        dt_end = datetime.datetime.strptime(end, dt_format)
        
        for ofs in range(0, 10000, 50):
            url = urlstr % (args[‘module‘], args[‘version‘], ofs)
            req = urllib2.Request(url)

            try: 
                response = urllib2.urlopen(req).read()

                for item in self.extract_from(response):
                    date, id, msg = item
                    dt_date = datetime.datetime.strptime(date, dt_format)
                    if dt_start <= dt_date <= dt_end:
                        print date, id, msg
                        response_data.append(item)
                    else:
                        return response_data
                        
            except urllib2.HTTPError, e:
                print(‘HTTPError = ‘ + str(e.code))
            except urllib2.URLError, e:
                print(‘URLError = ‘ + str(e.reason))
            except httplib.HTTPException, e:
                print(‘HTTPException‘)
            except Exception:
                raise

        return response_data

    def save_patch_file(self, l):
        format = ‘http://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/patch/%s?id=%s‘
        folder = self.assemble_path(args[‘module‘])
        
        if os.path.exists(folder):
            print ‘%s existed, please backup your data!‘ %folder
            return -1
        else:
            os.makedirs(folder)
        
        log = ["%s %s %s\n" %(i[0],i[1],i[2]) for i in l]
        open(os.path.join(folder, ‘patches.log‘), ‘w‘).writelines(log)
        
        total = len(l)
        for index, item in enumerate(l):
            
            _,id,msg = item
            patch = format % (args[‘module‘], id)
            file = os.path.join(folder, self.assemble_path(msg) + ‘.patch‘)
                
            urllib.urlretrieve(patch, file)
            print "[%d/%d] save %s" %(index+1, total, file)
    
    
    def extract_from(self, html):
        res = r"""<tr><td>.*?
        <span\ title.*?>(?P<date>\d+-\d+-\d+)</span>.*?
        <a\ href.*?id=(?P<id>\w+)‘>(?P<msg>.*?)</a>.*?
        </td></tr>"""
        rec = re.compile(res, re.DOTALL|re.VERBOSE)
        return re.findall(rec,html)

    def assemble_path(self, msg):
        path = ‘‘
        for c in msg:
            path += c if c.isalnum() else ‘_‘
    
        return path

        
# default args        
args = {
    "version": "linux-3.10.y",
    "module": "fs/ubifs",
    "date-range": "2013-08-15:",
    }
    
if len(sys.argv) > 1:
    args[‘version‘] = sys.argv[1]        
if len(sys.argv) > 2:
    args[‘module‘] = sys.argv[2]
if len(sys.argv) > 3:
    args[‘date-range‘] = sys.argv[3]
    
ret = PatchSpider().run()
sys.exit(ret)