可扩展的Cookies池,用无头浏览器登录并生成cookie供给爬虫使用
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

47 lines
1.6 KiB

import json
import requests
from requests.exceptions import ConnectionError
from cookiespool.db import *
class ValidTester(object):
def __init__(self, name='default'):
self.name = name
self.cookies_db = RedisClient('cookies', self.name)
self.account_db = RedisClient('accounts', self.name)
def test(self, username, cookies):
raise NotImplementedError
def run(self):
cookies_groups = self.cookies_db.all()
for username, cookies in cookies_groups.items():
self.test(username, cookies)
class WeiboValidTester(ValidTester):
def __init__(self, name='weibo'):
ValidTester.__init__(self, name)
def test(self, username, cookies):
print('正在测试Cookies', '用户名', username)
try:
cookies = json.loads(cookies)
except TypeError:
print('Cookies不合法', username)
self.cookies_db.delete(username)
print('删除Cookies', username)
return
try:
test_url = TEST_URL_MAP[self.name]
response = requests.get(test_url, cookies=cookies, timeout=5, allow_redirects=False)
if response.status_code == 200:
print('Cookies有效', username)
print('部分测试结果', response.text[0:50])
else:
print(response.status_code, response.headers)
print('Cookies失效', username)
self.cookies_db.delete(username)
print('删除Cookies', username)
except ConnectionError as e:
print('发生异常', e.args)