用Pymongo保存数据

爬取豆瓣电影top250movie.douban.com/top250的电影数据&＃xff0c;并保存在MongoDB中。

items.py

class DoubanspiderItem(scrapy.Item):# 电影标题title &＃61; scrapy.Field()# 电影评分score &＃61; scrapy.Field()# 电影信息content &＃61; scrapy.Field()# 简介info &＃61; scrapy.Field()


spiders/douban.py

import scrapy
from doubanSpider.items import DoubanspiderItemclass DoubanSpider(scrapy.Spider):name &＃61; "douban"allowed_domains &＃61; ["movie.douban.com"]start &＃61; 0url &＃61; &＃39;https://movie.douban.com/top250?start&＃61;&＃39;end &＃61; &＃39;&filter&＃61;&＃39;start_urls &＃61; [url &＃43; str(start) &＃43; end]def parse(self, response):item &＃61; DoubanspiderItem()movies &＃61; response.xpath("//div[&＃64;class&＃61;\&＃39;info\&＃39;]")for each in movies:title &＃61; each.xpath(&＃39;div[&＃64;class&＃61;"hd"]/a/span[&＃64;class&＃61;"title"]/text()&＃39;).extract()content &＃61; each.xpath(&＃39;div[&＃64;class&＃61;"bd"]/p/text()&＃39;).extract()score &＃61; each.xpath(&＃39;div[&＃64;class&＃61;"bd"]/div[&＃64;class&＃61;"star"]/span[&＃64;class&＃61;"rating_num"]/text()&＃39;).extract()info &＃61; each.xpath(&＃39;div[&＃64;class&＃61;"bd"]/p[&＃64;class&＃61;"quote"]/span/text()&＃39;).extract()item[&＃39;title&＃39;] &＃61; title[0]# 以;作为分隔&＃xff0c;将content列表里所有元素合并成一个新的字符串item[&＃39;content&＃39;] &＃61; &＃39;;&＃39;.join(content)item[&＃39;score&＃39;] &＃61; score[0]item[&＃39;info&＃39;] &＃61; info[0]# 提交itemyield itemif self.start <&＃61; 225:self.start &＃43;&＃61; 25yield scrapy.Request(self.url &＃43; str(self.start) &＃43; self.end, callback&＃61;self.parse)


pipelines.py

from scrapy.conf import settings
import pymongoclass DoubanspiderPipeline(object):def __init__(self):# 获取setting主机名、端口号和数据库名host &＃61; settings[&＃39;MONGODB_HOST&＃39;]port &＃61; settings[&＃39;MONGODB_PORT&＃39;]dbname &＃61; settings[&＃39;MONGODB_DBNAME&＃39;]# pymongo.MongoClient(host, port) 创建MongoDB链接client &＃61; pymongo.MongoClient(host&＃61;host,port&＃61;port)# 指向指定的数据库mdb &＃61; client[dbname]# 获取数据库里存放数据的表名self.post &＃61; mdb[settings[&＃39;MONGODB_DOCNAME&＃39;]]def process_item(self, item, spider):data &＃61; dict(item)# 向指定的表里添加数据self.post.insert(data)return item


settings.py

BOT_NAME &＃61; &＃39;doubanSpider&＃39;SPIDER_MODULES &＃61; [&＃39;doubanSpider.spiders&＃39;]
NEWSPIDER_MODULE &＃61; &＃39;doubanSpider.spiders&＃39;ITEM_PIPELINES &＃61; {&＃39;doubanSpider.pipelines.DoubanspiderPipeline&＃39; : 300}# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT &＃61; &＃39;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36&＃39;# MONGODB 主机环回地址127.0.0.1
MONGODB_HOST &＃61; &＃39;127.0.0.1&＃39;
# 端口号&＃xff0c;默认是27017
MONGODB_PORT &＃61; 27017
# 设置数据库名称
MONGODB_DBNAME &＃61; &＃39;DouBan&＃39;
# 存放本次数据的表名称
MONGODB_DOCNAME &＃61; &＃39;DouBanMovies&＃39;

运行启动MongoDB数据库需要两个命令&＃xff1a;mongod&＃xff1a;是mongoDB数据库进程本身
mongo&＃xff1a;是命令行shell客户端sudo mongod # 首先启动数据库服务&＃xff0c;再执行Scrapy
sudo mongo # 启动数据库shell在mongo shell下使用命令:# 查看当前数据库
> db# 列出所有的数据库
> show dbs# 连接DouBan数据库
> use DouBan# 列出所有表
> show collections# 查看表里的数据
> db.DouBanMoives.find()