(3)分布式下的爬虫Scrapy应该如何做-递归爬取方式，数据输出方式以及数据库链接

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from cnblogs.items import CnblogsItem

class CnblogsSpider(BaseSpider):
    name = "cnblogs"
    allowed_domains = ["cnblogs.com"]
    start_urls = [
        'http://www.cnblogs.com/#p%s' % p for p in xrange(1, 11)
        ]

    def parse(self, response):
        self.log("Fetch douban homepage page: %s" % response.url)
        hxs = HtmlXPathSelector(response)

        # authors = hxs.select('//a[@class="titlelnk"]')

        items = hxs.select('//a[contains(@class, "titlelnk")]')

        listitems = []

        for author in items:
            # print author.select('text()').extract()
            item = CnblogsItem()
            # property
            item['Title'] = ''.join(author.select('text()').extract())
            item['TitleUrl'] = author.select('@href').extract()
            listitems.append(item)

        return listitems

# -*- coding: utf-8 -*-
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from cnblogs.items import CnblogsItem
from scrapy.http import Request
from scrapy import log
# please pay attention to the encoding of info,otherwise raise error of decode 
import sys
reload(sys)
sys.setdefaultencoding('utf8')

class BlogsSpider(BaseSpider):
    name = "cnblogs_blogs"
    allowed_domains = ["cnblogs.com"]
    start_urls = [
        'http://www.cnblogs.com/fnng/default.aspx?page=1'
        ]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        # authors = hxs.select('//a[@class="titlelnk"]')
        # sel.xpath('//a[@class="PostTitle"]').xpath('text()')
        items = hxs.select('//a[@class="PostTitle"]')
        a_page = hxs.select('//div[@id="pager"]/a')
        for a_item in items:
            item = CnblogsItem()
            # property
            item['Title'] = ''.join(a_item.xpath('text()').extract())
            item['TitleUrl'] = a_item.xpath('@href').extract()
            yield item

        # get the page index
        log.msg(len(a_page))
        if len(a_page) > 0:
            for a_item in a_page:
                page_text = ''.join(a_item.xpath('text()').extract())
                if page_text == '下一页'.encode('utf-8') or 'Next' in page_text:
                    next_url = ''.join(a_item.xpath('@href').extract())
                    log.msg(next_url)
                    yield Request(next_url, callback=self.parse)
                    break