import urllib.request import urllib.parse import urllib.error import re import random def open_url(url): req_header = {'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0', 'Accept':'*/*', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'} req = urllib.request.Request(url, None, req_header) with urllib.request.urlopen(req, None, timeout=5) as respond: html = respond.read() return html def get_proxy_urls(): proxy_list = [] for i in range(1,11): str1 = r'http://www.kuaidaili.com/proxylist/%d/' % i proxy_list.append(str1) return proxy_list def get_ip(url): comp = '''(.+?) (.+?) (.+?) (.+?) (.+?) (.+?) (.+?) (.+?) ''' text_html = open_url(url).decond('utf_8') ip_list = re.findall(comp, text_html) return ip_list def test(): url = random.choice(get_proxy_urls()) print(url) print(open_url(url)) print(get_ip(url)) if __name__ == '__main__': test()
在urllib.request.urlopen()处出现异常
urllib.error.HTTPError: HTTP Error 521:
是加入的cookie有问题,改过之后好了,谢谢
应该是网站的反爬机制,可以看看这篇scrapy遇到了521
具体情况还是得看具体目标的反爬策略,实在不行就用他们的代理爬他们的代理吧2333333333
参考博文https://my.oschina.net/jhao10...
原博主用的是PyV8执行JS代码,我换了PyExecJS
import execjs import re import requests url = "http://www.kuaidaili.com/proxylist/1/" HERDERS = { "Host": "www.kuaidaili.com", 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36', } def executejs(html): # 提取其中的JS加密函数 js_string = ''.join(re.findall(r'(function .*?)</script>',html)) # 提取其中执行JS函数的参数 js_func_arg = re.findall(r'setTimeout\(\"\D+\((\d+)\)\"', html)[0] js_func_name = re.findall(r'function (\w+)',js_string)[0] # 修改JS函数,使其返回Cookie内容 js_string = js_string.replace('eval("qo=eval;qo(po);")', 'return po') func = execjs.compile(js_string) return func.call(js_func_name,js_func_arg) def parse_cookie(string): string = string.replace("document.cookie='", "") clearance = string.split(';')[0] return {clearance.split('=')[0]: clearance.split('=')[1]} # 第一次访问获取动态加密的JS first_html = requests.get(url=url,headers=HERDERS).content.decode('utf-8') # 执行JS获取Cookie cookie_str = executejs(first_html) # 将Cookie转换为字典格式 cookie = parse_cookie(cookie_str) print('cookies = ',cookie) # 带上cookies参数,再次请求 response = requests.get(url=url,headers=HERDERS,cookies=cookie) print(response.status_code)