urllib2的基本使用

  1 import urllib2
  2 
  3 # 向指定的url发送请求，并返回服务器响应的类文件对象
  4 response = urllib2.urlopen("http://www.baidu.com/")
  5 
  6 # 类文件对象支持文件的操作方法，如read()方法返回为字符串 
  7 html = response.read()
  8 
  9 print htm

  1 # -*- coding:utf-8 -*-
  2 import urllib2
  3 
  4 url = "http://www.baidu.com/"
  5 
  6 # url 作为Request()方法的一个参数，并构造一个Request对象
  7 request = urllib2.Request(url)
  8 
  9 # Request对象作为urlopen()方法的参数，发给服务器响应
 10 response = urllib2.urlopen(request)
 11 
 12 html = response.read()
 13 
 14 print html

  1 # -*- coding:utf-8 -*-
  2 import urllib2
  3 
  4 url = "http://www.baidu.com/"
  5 headers = {"User-Agent" : "User-Agent:Mozilla/5.0(compatible;MSIE9.0;Windows    NT6.1;Trident/5.0)"}
  6 
  7 
  8 # url和headers一起 作为Request()方法的参数，并构造一个Request对象
  9 request = urllib2.Request(url, headers=headers)
 10 
 11 # Request对象作为urlopen()方法的参数，发给服务器响应
 12 response = urllib2.urlopen(request)
 13 
 14 html = response.read()
 15 
 16 print html

  1 # -*- coding:utf-8 -*-
  2 import urllib2
  3 
  4 url = "http://www.baidu.com/"
  5 headers = {"User-Agent" : "User-Agent:Mozilla/5.0(compatible;MSIE9.0;Windows    NT6.1;Trident/5.0)"}
  6 
  7 
  8 # url和headers一起 作为Request()方法的参数，并构造一个Request对象
  9 request = urllib2.Request(url, headers=headers)
 10 # 调用Request.add_header(),添加或者修改特定的header
 11 request = add_header("Connectin", "keep-alive")
 12 
 13 # Request对象作为urlopen()方法的参数，发给服务器响应
 14 response = urllib2.urlopen(request)
 15 
 16 html = response.read()
 17 
 18 print html

 1 import urllib2
 2 import random
 3 
 4 url = "http://www.itcast.cn"
 5 
 6 ua_list = [
 7     "Mozilla/5.0 (Windows NT 6.1; ) Apple.... ",
 8     "Mozilla/5.0 (X11; CrOS i686 2268.111.0)... ",
 9     "Mozilla/5.0 (Macintosh; U; PPC Mac OS X.... ",
10     "Mozilla/5.0 (Macintosh; Intel Mac OS... "
11 ]
12 
13 user_agent = random.choice(ua_list)
14 
15 request = urllib2.Request(url)
16 
17 #也可以通过调用Request.add_header() 添加/修改一个特定的header
18 request.add_header("User-Agent", user_agent)
19 
20 # 第一个字母大写，后面的全部小写
21 request.get_header("User-agent")
22 
23 response = urllib2.urlopen(request)
24 
25 html = response.read()
26 print html