标签：images lint 页面 bsp base 避免 indicator ror 访问
爬遍整个网络
1 当我们访问整个网络的时候，我们不可避免的会访问不同的网站，但是不同的网站会有完全不同的结构和内容...
现在一步一步的构建访问整个网络的脚本
I 从一个网站开始，每一次都爬向不同的网站。如果在一个页面找不到指向其他网站的链接，获取本网站其他界面信息，直到找到其他网站的链接。
# -*- coding:utf-8 -*- from urllib.request import urlopen from urllib.error import HTTPError from bs4 import BeautifulSoup from random import choice import re basename = "http://en.wikipedia.org" visitedpages = set() def getInternalLinks(bsObj,includeUrl): return [eachlink.attrs[‘href‘] for eachlink in bsObj.find_all("a",href=re.compile("^(/|.*" + includeUrl + ")")) if ‘href‘ in eachlink.attrs] def getExternalLinks(bsObj,excludeUrl): return [eachlink.attrs[‘href‘] for eachlink in bsObj.find_all("a",href=re.compile("^(http|www)((?!" + excludeUrl + ").)*$")) if ‘href‘ in eachlink.attrs] def splitAddress(address): addressParts = address.replace("http://","").split("/") return addressParts def getRandomExternalLink(startingPage): html = urlopen(startingPage) with html: bsObj = BeautifulSoup(html,"html.parser") externalLinks = getExternalLinks(bsObj,splitAddress(startingPage)[0]) if len(externalLinks) == 0: internalLinks = getInternalLinks(bsObj, splitAddress(startingPage)[0]) return choice(internalLinks) else: return choice(externalLinks) def followExternalLink(startingPage): externalLink = getRandomExternalLink("http://www.oreilly.com/") if externalLink in visitedpages: print("visited") else: print("the random external link is " + externalLink) visitedpages.add(externalLink) followExternalLink(externalLink) if __name__ == "__main__": #print(splitAddress("http://www.oreilly.com/")[0]) #print(getRandomExternalLink("http://www.oreilly.com/")) followExternalLink("http://www.oreilly.com/")
II 从一个网站开始，查找这个网站所有界面信息，获取整个网站指向其他网站的链接
# -*- coding:utf-8 -*- from urllib.request import urlopen from urllib.error import HTTPError from bs4 import BeautifulSoup from random import choice import re def getInternalLinks(bsObj,includeUrl): return [eachlink.attrs[‘href‘] for eachlink in bsObj.find_all("a",href=re.compile("^(/|.*" + includeUrl + ")")) if ‘href‘ in eachlink.attrs] def getExternalLinks(bsObj,excludeUrl): return [eachlink.attrs[‘href‘] for eachlink in bsObj.find_all("a",href=re.compile("^(http|www)((?!" + excludeUrl + ").)*$")) if ‘href‘ in eachlink.attrs] def splitAddress(address): addressParts = address.replace("http://","").split("/") return addressParts allINlinks = set() allEXlinks = set() def getAllexternalLinks(startPage): try: with urlopen(startPage) as html: bsObj = BeautifulSoup(html,"html.parser") except HTTPError as e: print(e) else: allinternallinks = getInternalLinks(bsObj,splitAddress(startPage)[0]) allexternallinks = getExternalLinks(bsObj,splitAddress(startPage)[0]) print("************external*******************************") for eachexternallink in allexternallinks: if eachexternallink not in allEXlinks: allEXlinks.add(eachexternallink) print(eachexternallink) print("************internal*******************************") for eachinternallink in allinternallinks: if eachinternallink not in allINlinks: allINlinks.add(eachinternallink) print(eachinternallink) getAllexternalLinks(eachinternallink) if __name__ == "__main__": getAllexternalLinks("http://www.oreilly.com/")
***************还存在问题的代码***************************
标签：images lint 页面 bsp base 避免 indicator ror 访问
原文地址：http://www.cnblogs.com/someoneHan/p/6234508.html
