爬取京东上商品的所有详细信息

  1 # coding=utf-8
  2 
  3 #导入模块
  4 import urllib2,re,urllib
  5 from bs4 import BeautifulSoup
  6 import json,time
  7 
  8 
  9 #定义抓取类
 10 class JD:
 11 
 12     #记录抓取产品个数
 13     prodNum = 1
 14     #初始化参数
 15     def __init__(self,baseurl,page):
 16         self.baseurl = baseurl
 17         self.page = page
 18         #拼装成url
 19         self.url = self.baseurl+‘&‘+‘page=‘+str(self.page)
 20     #获取html源代码
 21     def getHtml(self,url):
 22         #请求抓取对象
 23         request = urllib2.Request(url)
 24         #响应对象
 25         reponse = urllib2.urlopen(request)
 26         #读取源代码
 27         html = reponse.read()
 28         #返回源代码
 29         return html
 30     #获取总页数
 31     def getNum(self,html):
 32         #封装成BeautifulSoup对象
 33         soup = BeautifulSoup(html)
 34         #定位到总页数节点
 35         items = soup.find_all(‘span‘,class_=‘p-skip‘)
 36         #获取总页数
 37         for item in items:
 38             pagenum = item.find(‘em‘).find(‘b‘).string
 39         return pagenum
 40     #获取所有产品id列表
 41     def getIds(self,html):
 42         #生成匹配规则
 43         pattern =  re.compile(‘<a target="_blank" href="//item.jd.com/(.*?).html".*?>‘)
 44         #查询匹配对象
 45         items = re.findall(pattern,html)
 46         return items
 47     #根据产品id获取同款产品列表
 48     def getIdByItems(self,id):
 49         #拼装成url
 50         url = basePd+str(id)+‘.html‘
 51         #调用抓取函数返回源代码
 52         html = self.getHtml(url)
 53         # 封装成BeautifulSoup对象
 54         soup = BeautifulSoup(html)
 55         #查询匹配对象
 56         items = soup.find(‘div‘,class_=‘dd clearfix‘)
 57         l = []
 58         #生成列表
 59         for item in items:
 60             pattern = re.compile(‘href="//item.jd.com/(.*?).html".*?>‘)
 61             id = re.findall(pattern,str(item))
 62             if id:
 63                 l += id
 64         return l
 65 
 66     #获取产品价格
 67     def getPrice(self,id):
 68         url = ‘http://p.3.cn/prices/mgets?skuIds=J_‘+str(id)
 69         jsonString = self.getHtml(url)
 70         jsonObject = json.loads(jsonString.decode())
 71         price_jd = jsonObject[0][‘p‘]
 72         price_mk = jsonObject[0][‘m‘]
 73         print ‘京东价格：‘,str(price_jd)
 74         print ‘市场价格：‘,str(price_mk)
 75 
 76     #获取产品图片
 77     def getImg(self,html,subid):
 78         pattern = re.compile(r‘<img id=.*?data-origin="(.*?)" alt=.*?‘, re.S)
 79         items = re.findall(pattern, html)
 80         for item in items:
 81             imgurl = ‘http:%s‘ % (item)
 82             urllib.urlretrieve(imgurl, ‘d:/temp/jdimg/%s.jpg‘ % (str(subid)))
 83 
 84     #获取内容
 85     def getContent(self,html,subid):
 86         soup = BeautifulSoup(html)
 87         title = soup.find(‘div‘,class_=‘sku-name‘)
 88         print ‘\n-----------------第‘+ str(JD.prodNum) +‘个产品--------------------\n‘
 89         for t in title:
 90             print ‘名称: ‘,t.string
 91         time.sleep(1)
 92         #价格
 93         self.getPrice(subid)
 94         #编码
 95         print ‘产品编码：%s‘ % (str(subid))
 96         items1 = soup.find_all(‘ul‘,class_=‘parameter1 p-parameter-list‘)
 97         #商品基本信息
 98         for item in items1:
 99             p = item.findAll(‘p‘)
100             for i in p:
101                 print i.string
102         # 商品基本信息
103         items2 = soup.find_all(‘ul‘, class_=‘parameter2 p-parameter-list‘)
104         for item in items2:
105             p = item.findAll(‘li‘)
106             for i in p:
107                 print i.string
108         #规格与包装
109         items3 = soup.find_all(‘div‘,class_=‘Ptable-item‘)
110         for item in items3:
111             contents1 = item.findAll(‘dt‘)
112             contents2 = item.findAll(‘dd‘)
113             for i in range(len(contents1)):
114                 print contents1[i].string,contents2[i].string
115         JD.prodNum += 1
116     #启动抓取程序
117     def start(self):
118         html = spider.getHtml(self.url)
119         pageNum = self.getNum(html)
120         print ‘正在抓取网页请稍后............‘
121         time.sleep(3)
122         print ‘抓取完毕，本次共抓取‘,pageNum,‘页。‘
123         time.sleep(1)
124         print ‘正在解析内容.........‘
125         #循环1--页数
126         for page in range(1,int(pageNum)+1):
127             url = self.baseurl+‘&‘+‘page=‘+str(page)
128             html = self.getHtml(url)
129             ids = self.getIds(html)
130             #循环2--产品列表
131             for id in ids:
132                 urlprod = basePd+str(id)+‘.html‘
133                 htmlprod = self.getHtml(urlprod)
134                 subids = self.getIdByItems(id)
135                 #循环3--产品组列表
136                 for subid in subids:
137                     urlsubprod = basePd+str(subid)+‘.html‘
138                     subhtml = self.getHtml(urlsubprod)
139                     time.sleep(1)
140                     self.getContent(subhtml,subid)
141                     self.getImg(subhtml,subid)
142 
143 #产品列表base页
144 basePd  = ‘http://item.jd.com/‘
145 #抓取入口URL
146 baseURL = ‘http://list.jd.com/list.html?cat=9987,653,655‘
147 #生成爬虫抓取对象
148 spider = JD(baseURL,1)
149 
150 #开始抓取
151 spider.start()