python - scrapy爬虫递归调用失效

 小水沉伦 发布于 2022-11-02 11:12
# -*- coding: UTF-8 -*-

from scrapy.spiders import Spider
from mycrawler.items import BasicItem
import scrapy

header = {
    'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0"}


class DlinkSpider(Spider):
    name = "hikvision1"
    start_urls = [
        # "http://www.hikvisioneurope.com/portal/index.php?dir=Product%20Firmware/"
        "http://www.hikvisioneurope.com/portal/index.php?dir=Product%20Firmware/Cameras/DS-2CD2X22FWD%2C2X42FWD%2C2X52F/"
    ]

    # must be lower character
    suffix = ["zip", "obj", "exe", "drv", "com", "lan",
              "dlf", "tar", "tgz", "gz", "iso", "img", "dmg", "bin"]

    def parse(self, response):

        t = response.headers.get(
            'Content-Type').split(r'/')[-1].split(r';')[0]
        print response.headers, "content-type\n\n"
        if t == 'html':
            for a in response.css('table table a')[1:]:
                url = response.urljoin(a.xpath('@href').extract()[0])
                yield scrapy.Request(url,
                                     callback=self.parse,
                                     headers=header,
                                     meta={'filename': a.css('a::text').extract()})

        else:
            if t in self.suffix:
                item = BasicItem()
                item["Firm"] = "Hikvision"
                item["Link"] = response.url
                item["Rawlink"] = response.url
                item["Filename"] = response.meta.get('filename')
                item["Title"] = item["Filename"]
                item["Info"] = {}
                print item
运行不完整,就是callback=self.parse执行不了
1 个回答
  • 我一直明白了,其实不是没有执行,而是scrapy框架惹的祸,因为它每次都会把内容下载下来再作判断,所以就造成了读取内容超时,失败。

    2022-11-04 06:12 回答
撰写答案
今天,你开发时遇到什么问题呢?
立即提问
热门标签
PHP1.CN | 中国最专业的PHP中文社区 | PNG素材下载 | DevBox开发工具箱 | json解析格式化 |PHP资讯 | PHP教程 | 数据库技术 | 服务器技术 | 前端开发技术 | PHP框架 | 开发工具 | 在线工具
Copyright © 1998 - 2020 PHP1.CN. All Rights Reserved 京公网安备 11010802041100号 | 京ICP备19059560号-4 | PHP1.CN 第一PHP社区 版权所有