使用itempipeline处理保存数据

class MysqlPipeline(object):
    def __init__(self):
        self.conn = MySQLdb.conn(host="127.0.0.1",user="root",password="root",
                                                        
                                    db="article_spider",charset="utf8",use_unicode=True)
        self.cursor = self.conn.cursor()
        
    def process_item(self,item,spider）:
            insert_sql = """  insert into article(title,create_date,url,url_object_id,     
                              comment_nums, fav_nums, praise_nums, tags, content) 
                              value(%s,%s,%s,%s,%s,%s,%s,%s,%s); """
            self.cursor.execute(insert_sql,(item[""],...))
            self.conn.commit()

 1 class MysqlTwistedPipeline(object):
 2     #采用异步的机制写入mysql
 3     def __init__(self,dbpool):
 4         self.dbpool =dbpool
 5 
 6     @classmethod
 7     def from_settings(cls,settings):
 8         dbparms = dict(
 9             host = "127.0.0.1",#settings["MYSQL_HOST"],
10             db = "bitcoin",#settings["MYSQL_DBNAME"],
11             user ="root",#settings["MYSQL_USER"]
12             password ="root",#settings["MYSQL_PASSWORD"],
13             charset="utf8",
14             use_unicode =True,#不然没办法保存中文
15             cursorclass = MySQLdb.cursors.DictCursor
16         )
17         dbpool = adbapi.ConnectionPool("MySQLdb",**dbparms)
18         return cls(dbpool)
19 
20     def process_item(self,item,spider):
21         ##使用twisted将mysql插入变成异步执行
22         query = self.dbpool.runInteraction(self.do_insert,item)
23         query.addErrback(self.handle_error,item,spider)#处理异常
24 
25 
26     def handle_error(self,failure,item,spider):
27         #处理异步插入的异常
28         print(failure)
29 
30     def do_insert(self,cursor,item):
31         #执行具体的插入
32         #根据不同的的item构建不同的sql语句插入到mysql中
33         insert_sql,params = item.get_insert_sql()
34 
35         cursor.execute(insert_sql,params)
36             ###自动commit