可扩展的Cookies池,用无头浏览器登录并生成cookie供给爬虫使用
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

52 lines
1.8 KiB

8 years ago
import json
import requests
from requests.exceptions import ConnectionError
from cookiespool.db import *
class ValidTester(object):
def __init__(self, name='default'):
self.name = name
7 years ago
self.cookies_db = RedisClient('cookies', self.name)
self.account_db = RedisClient('accounts', self.name)
8 years ago
def test(self, account, cookies):
raise NotImplementedError
7 years ago
8 years ago
def run(self):
accounts = self.cookies_db.all()
7 years ago
print(accounts)
8 years ago
for account in accounts:
username = account.get('username')
cookies = self.cookies_db.get(username)
self.test(account, cookies)
class WeiboValidTester(ValidTester):
def __init__(self, name='weibo'):
ValidTester.__init__(self, name)
7 years ago
8 years ago
def test(self, account, cookies):
print('Testing Account', account.get('username'))
try:
cookies = json.loads(cookies)
except TypeError:
print('Invalid Cookies Value', account.get('username'))
self.cookies_db.delete(account.get('username'))
print('Deleted User', account.get('username'))
return None
try:
7 years ago
test_url = TEST_URL_MAP[self.name]
8 years ago
response = requests.get(test_url, cookies=cookies, timeout=5, allow_redirects=False)
if response.status_code == 200:
print('Valid Cookies', account.get('username'))
else:
print(response.status_code, response.headers)
print('Invalid Cookies', account.get('username'))
self.cookies_db.delete(account.get('username'))
print('Deleted User', account.get('username'))
except ConnectionError as e:
print('Error', e.args)
print('Invalid Cookies', account.get('username'))