python - 现在用scrapy爬一个网站始终遇到521错误，是怎么回事呢？

朋友我最近也在爬这个网站，给你点意见你可以看看吧。愿意交流下的话加下我的好友吧。
cnvd正常的情况都可以爬的到。

www.cnvd.org.cn这个东西就比较恶心了，很多的头都反521。

# -*- coding:utf-8 -*-
#coding = utf-8
import urllib
import urllib2
import re
import random
import socket
import MySQLdb as mdb
import cookielib

pagenumber=0
url = 'http://ics.cnvd.org.cn/?max=100&offset='+str(pagenumber)  
# url='http://www.cnvd.org.cn/flaw/show/CNVD-2016-05694'  
  
cookie_support= urllib2.HTTPCookieProcessor(cookielib.CookieJar())
opener = urllib2.build_opener(cookie_support,urllib2.HTTPHandler)
urllib2.install_opener(opener)
opener2= urllib2.build_opener(cookie_support,urllib2.HTTPHandler)
urllib2.install_opener(opener2)

user_agents = [
             'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
             'Opera/9.25 (Windows NT 5.1; U; en)',
             'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
             'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
             'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
             'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
             "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
             "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 ",
             "Mozilla-Firefox-Spider(Wenanry)"
             ] 

agent = random.choice(user_agents)

UserAgent = "Mozilla-Firefox-Spider(Wenanry)"

opener.addheaders = [("User-agent",UserAgent),("Accept","*/*"),('Referer','http://www.miit.gov.cn/')]

con = mdb.connect('127.0.0.1', 'root', 'root', 'test', port=3307,charset="utf8");
with con:
    cur=con.cursor()
    cur.execute("CREATE TABLE IF NOT EXISTS \
        BUGnews(Id INT PRIMARY KEY AUTO_INCREMENT, WebUrl VARCHAR(50))")
    try:
        res = opener.open(url)
        content= res.read()        
#         request = urllib2.Request(url,headers = header1)
#         response = urllib2.urlopen(request)
#         content= response.read().decode('utf-8')

        print content
        pattern= re.compile('<a.*?href="(.*?)" title=',re.S)
        items= re.findall(pattern,content)
        for item in items:
            url2=item
#             opener2.addheaders = [("User-agent","Mozilla-Firefox-Spider(Wenanry)"),("Accept","*/*"),('Referer','http://www.google.com')]
#             res2=opener2.open(url2)
#             contentnews= res2.read()
#             print contentnews
            sql="insert into BUGnews(WebUrl) VALUES (%s)"
            params=item.encode('utf-8')
            cur.execute(sql,params)
    #         res2 = opener2.open(url2)
    #         content2= res2.read()
    #         print content2
    
    except urllib2.URLError, e:
            if hasattr(e,"code"):
                print e.code
            if hasattr(e,"reason"):
                print e.reason
                
                
            

你的爬虫肯定被该网站检测到了，headers用了都不行那么只有看它是限制的你的ip还是账号，如果不用登录那么你先换个ip跑跑试试，看是否正常，或者直接在爬虫服务器上手动访问看能否成功

------更新-------

应题主要求，我把测试代码贴出来，这里的cookie是我刚才通过手动访问拿到的，带入它访问就没问题，至于cookie的过期时间，我没有仔细看，如果题主不懂我再好好看吧。

import requests

url = 'http://www.cnvd.org.cn'
headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate, sdch',
            'Accept-Language':'zh-CN,zh;q=0.8',
            'AlexaToolbar-ALX_NS_PH': 'AlexaToolbar/alx-4.0',
            'Cache-Control': 'max-age=0',
            'Connection':'keep-alive',
            'Host':'www.cnvd.org.cn',
            'Referer':'http://www.cnvd.org.cn/',
            'Upgrade-Insecure-Requests':'1',
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
            'Cookie':'__jsluid=224f8fc054404821896d6b6bd2415533; __jsl_clearance=1472619039.65|0|4opPNLlmaO6pFXwTMO%2BQ5UAhfEA%3D; JSESSIONID=AE8735BE6328B81C7CD4352B75F25316; bdshare_firstime=1472619047205'

        }
cont = requests.get(url, headers=headers).text
print(cont)

希望对你有帮助