豆瓣python源码_python爬虫1——获取网站源代码(豆瓣图书top250信息)

#-*- coding: utf-8 -*-

importrequestsimportreimportsys

reload(sys)

sys.setdefaultencoding(&＃39;utf-8&＃39;)classSpider(object):def __init__(self):print(&＃39;开始爬取豆瓣图书top250的内容。。。。。。&＃39;)#传入url&＃xff0c;返回网页源代码

defgetSourceCode(self, url):

html&＃61;requests.get(url)returnhtml.text#从源代码中提取出我们需要的内容块&＃xff1a;{书名、作者出版社等、评分、评价人数}。

defgetEveryBookContent(self, sourceCode):

everyBookContent&＃61; re.findall(&＃39;

(.*?)

returneveryBookContent#从内容块中提取出数据

defgetBookInfo(self, eachBookContent):

bookInfo&＃61;{}#bookInfo[&＃39;title&＃39;] &＃61; re.subn(&＃39;( |\n|
|?span.*?>)&＃39;, "", re.search(&＃39;(.*?)&＃39;, eachBookContent, re.S).group(1))[0]

bookInfo[&＃39;title&＃39;] &＃61; re.sub(&＃39;( |\n|
|?span.*?>)&＃39;, "", re.search(&＃39;(.*?)&＃39;, eachBookContent, re.S).group(1))

bookInfo[&＃39;author&＃39;] &＃61; re.search(&＃39;

(.*?)

bookInfo[&＃39;discussNum&＃39;] &＃61; re.sub(&＃39;( |\n|
)&＃39;, "", re.search(&＃39;\((.*?)\)&＃39;, eachBookContent, re.S).group(1))

bookInfo[&＃39;score&＃39;] &＃61; re.search(&＃39;(.*?)&＃39;, eachBookContent, re.S).group(1)returnbookInfo#将结果保存到文件

defsaveBookInfo(self, bookList):

f&＃61; open("bookList.txt", "a")for each inbookList:

f.writelines(&＃39;书 名:\t {}\n&＃39;.format(each[&＃39;title&＃39;]))

f.writelines(&＃39;作 者:\t {}\n&＃39;.format(each[&＃39;author&＃39;]))

f.writelines(&＃39;评论数:\t {}\n&＃39;.format(each[&＃39;discussNum&＃39;]))

f.writelines(&＃39;评 分:\t {}\n\n&＃39;.format(each[&＃39;score&＃39;]))

f.close()defstart(self, url):

sourceCode&＃61;self.getSourceCode(url)

everyBookContent&＃61;self.getEveryBookContent(sourceCode)

bookList&＃61;[]for each ineveryBookContent:

bookList.append(self.getBookInfo(each))

self.saveBookInfo(bookList)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:

douban&＃61;Spider()

url&＃61; &＃39;http://book.douban.com/top250?start&＃61;0&＃39;i&＃61;0while i <&＃61; 225:

url&＃61; &＃39;http://book.douban.com/top250?start&＃61;{}&＃39;.format(i)

douban.start(url)

i&＃43;&＃61; 25