Browse Source

version 1.0

pull/4/merge
Germey 8 years ago
commit
6396b1f8d0
  1. 3
      .gitignore
  2. 3
      README.md
  3. 0
      cookiespool/__init__.py
  4. 59
      cookiespool/api.py
  5. 52
      cookiespool/config.py
  6. 210
      cookiespool/db.py
  7. 48
      cookiespool/error.py
  8. 169
      cookiespool/generator.py
  9. 24
      cookiespool/importer.py
  10. 56
      cookiespool/scheduler.py
  11. 61
      cookiespool/tester.py
  12. 137
      cookiespool/verify.py
  13. 8
      run.py
  14. 17
      tests/login.py

3
.gitignore vendored

@ -0,0 +1,3 @@
/.idea
*.pyc
ghostdriver.log

3
README.md

@ -0,0 +1,3 @@
# CookiesPool
CookiesPool Based on Redis and Flask

0
cookiespool/__init__.py

59
cookiespool/api.py

@ -0,0 +1,59 @@
from flask import Flask, g
from cookiespool.config import *
from cookiespool.db import CookiesRedisClient, AccountRedisClient
__all__ = ['app']
app = Flask(__name__)
@app.route('/')
def index():
return '<h2>Welcome to Cookie Pool System</h2>'
def get_conn():
"""
获取
:return:
"""
for name in GENERATOR_MAP:
print(name)
if not hasattr(g, name):
setattr(g, name + '_cookies', eval('CookiesRedisClient' + '(name="' + name + '")'))
setattr(g, name + '_account', eval('AccountRedisClient' + '(name="' + name + '")'))
return g
@app.route('/<name>/random')
def random(name):
"""
获取随机的Cookie, 访问地址如 /weibo/random
:return: 随机Cookie
"""
g = get_conn()
cookies = getattr(g, name + '_cookies').random()
return cookies
@app.route('/<name>/add/<username>/<password>')
def add(name, username, password):
"""
添加用户, 访问地址如 /weibo/add/user/password
"""
g = get_conn()
result = getattr(g, name + '_account').set(username, password)
return result
@app.route('/<name>/count')
def count(name):
"""
获取Cookies总数
"""
g = get_conn()
count = getattr(g, name + '_cookies').count()
return str(int) if isinstance(count, int) else count
if __name__ == '__main__':
app.run(host='0.0.0.0')

52
cookiespool/config.py

@ -0,0 +1,52 @@
# Redis数据库地址
REDIS_HOST = 'localhost'
REDIS_PORT = 6379
REDIS_PASSWORD = 'foobared'
REDIS_DOMAIN = '*'
REDIS_NAME = '*'
# 云打码相关配置到yundama.com申请注册
YUNDAMA_USERNAME = 'Germey'
YUNDAMA_PASSWORD = '940629cqc'
YUNDAMA_APP_ID = '3372'
YUNDAMA_APP_KEY = '1b586a30bfda5c7fa71c881075ba49d0'
YUNDAMA_API_URL = 'http://api.yundama.com/api.php'
YUNDAMA_MAX_RETRY = 20
# 产生器默认使用的浏览器
DEFAULT_BROWSER = 'PhantomJS'
# 产生器类
GENERATOR_MAP = {
'weibo': 'WeiboCookiesGenerator'
}
# 测试类
TESTER_MAP = {
'weibo': 'WeiboValidTester'
}
# 产生器和验证器循环周期
CYCLE = 120
# API地址和端口
API_HOST = '127.0.0.1'
API_PORT = 5000
# 进程开关
# 产生器
GENERATOR_PROCESS = True
# 验证器
VALID_PROCESS = False
# API
API_PROCESS = True

210
cookiespool/db.py

@ -0,0 +1,210 @@
import random
import redis
from cookiespool.config import *
from cookiespool.error import *
class RedisClient(object):
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD):
"""
初始化Redis连接
:param host: 地址
:param port: 端口
:param password: 密码
"""
if password:
self._db = redis.Redis(host=host, port=port, password=password)
else:
self._db = redis.Redis(host=host, port=port)
self.domain = REDIS_DOMAIN
self.name = REDIS_NAME
def _key(self, key):
"""
得到格式化的key
:param key: 最后一个参数key
:return:
"""
return "{domain}:{name}:{key}".format(domain=self.domain, name=self.name, key=key)
def set(self, key, value):
"""
设置键值对
:param key:
:param value:
:return:
"""
raise NotImplementedError
def get(self, key):
"""
根据键名获取键值
:param key:
:return:
"""
raise NotImplementedError
def delete(self, key):
"""
根据键名删除键值对
:param key:
:return:
"""
raise NotImplementedError
def keys(self):
"""
得到所有的键名
:return:
"""
return self._db.keys('{domain}:{name}:*'.format(domain=self.domain, name=self.name))
def flush(self):
"""
清空数据库, 慎用
:return:
"""
self._db.flushall()
class CookiesRedisClient(RedisClient):
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, domain='cookies', name='default'):
"""
管理Cookies的对象
:param host: 地址
:param port: 端口
:param password: 密码
:param domain: , 如cookies, account等
:param name: 名称, 一般为站点名, weibo, 默认 default
"""
RedisClient.__init__(self, host, port, password)
self.domain = domain
self.name = name
def set(self, key, value):
try:
self._db.set(self._key(key), value)
except:
raise SetCookieError
def get(self, key):
try:
return self._db.get(self._key(key)).decode('utf-8')
except:
return None
def delete(self, key):
try:
print('Delete', key)
return self._db.delete(self._key(key))
except:
raise DeleteCookieError
def random(self):
"""
随机得到一Cookies
:return:
"""
try:
keys = self.keys()
return self._db.get(random.choice(keys))
except:
raise GetRandomCookieError
def all(self):
"""
获取所有账户, 以字典形式返回
:return:
"""
try:
for key in self._db.keys('{domain}:{name}:*'.format(domain=self.domain, name=self.name)):
group = key.decode('utf-8').split(':')
if len(group) == 3:
username = group[2]
yield {
'username': username,
'cookies': self.get(username)
}
except Exception as e:
print(e.args)
raise GetAllCookieError
def count(self):
"""
获取当前Cookies数目
:return: 数目
"""
return len(self.keys())
class AccountRedisClient(RedisClient):
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, domain='account', name='default'):
RedisClient.__init__(self, host, port, password)
self.domain = domain
self.name = name
def set(self, key, value):
try:
return self._db.set(self._key(key), value)
except:
raise SetAccountError
def get(self, key):
try:
return self._db.get(self._key(key)).decode('utf-8')
except:
raise GetAccountError
def all(self):
"""
获取所有账户, 以字典形式返回
:return:
"""
try:
for key in self._db.keys('{domain}:{name}:*'.format(domain=self.domain, name=self.name)):
group = key.decode('utf-8').split(':')
if len(group) == 3:
username = group[2]
yield {
'username': username,
'password': self.get(username)
}
except Exception as e:
print(e.args)
raise GetAllAccountError
def delete(self, key):
"""
通过用户名删除用户
:param key:
:return:
"""
try:
return self._db.delete(self._key(key))
except:
raise DeleteAccountError
if __name__ == '__main__':
"""
conn = CookiesRedisClient()
conn.set('name', 'Mike')
conn.set('name2', 'Bob')
conn.set('name3', 'Amy')
print(conn.get('name'))
conn.delete('name')
print(conn.keys())
print(conn.random())
"""
# 测试
conn = AccountRedisClient(name='weibo')
conn.set('14760253606', 'gmidy8470')
conn.set('14760253607', 'uoyuic8427')
conn.set('18459749258', 'rktfye8937')
conn.set('18459748505', 'astvar3647')
print(conn.get('18459748505'))
print(conn.keys())
accounts = conn.all()
for account in accounts:
print(account)

48
cookiespool/error.py

@ -0,0 +1,48 @@
class CookiePoolError(Exception):
def __str__(self):
return repr('Cookie Pool Error')
class SetCookieError(CookiePoolError):
def __str__(self):
return repr('Set Cookie Error')
class GetCookieError(CookiePoolError):
def __str__(self):
return repr('Get Cookie Error')
class DeleteCookieError(CookiePoolError):
def __str__(self):
return repr('Delete Cookie Error')
class GetRandomCookieError(CookiePoolError):
def __str__(self):
return repr('Get Random Cookie Error')
class GetAllCookieError(CookiePoolError):
def __str__(self):
return repr('Get All Cookie Error')
class SetAccountError(CookiePoolError):
def __str__(self):
return repr('Set Account Error')
class DeleteAccountError(CookiePoolError):
def __str__(self):
return repr('Delete Account Error')
class GetAccountError(CookiePoolError):
def __str__(self):
return repr('Get Account Error')
class GetAllAccountError(CookiePoolError):
def __str__(self):
return repr('Get All Account Error')

169
cookiespool/generator.py

@ -0,0 +1,169 @@
import json
import requests
import time
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities
from selenium.common.exceptions import WebDriverException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from cookiespool.db import CookiesRedisClient, AccountRedisClient
from cookiespool.verify import Yundama
from cookiespool.config import *
from requests.exceptions import ConnectionError
class CookiesGenerator(object):
def __init__(self, name='default', browser_type=DEFAULT_BROWSER):
"""
父类, 初始化一些对象
:param name: 名称
:param browser: 浏览器, 若不使用浏览器则可设置为 None
"""
self.name = name
self.cookies_db = CookiesRedisClient(name=self.name)
self.account_db = AccountRedisClient(name=self.name)
self.browser_type = browser_type
def _init_browser(self, browser_type):
"""
通过browser参数初始化全局浏览器供模拟登录使用
:param browser: 浏览器 PhantomJS/ Chrome
:return:
"""
if browser_type == 'PhantomJS':
caps = DesiredCapabilities.PHANTOMJS
caps[
"phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Linux; U; Android 2.3.6; en-us; Nexus S Build/GRK39F) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"
self.browser = webdriver.PhantomJS(desired_capabilities=caps)
elif browser_type == 'Chrome':
self.browser = webdriver.Chrome()
def new_cookies(self, username, password):
raise NotImplementedError
def set_cookies(self, account):
"""
根据账户设置新的Cookies
:param account:
:return:
"""
results = self.new_cookies(account.get('username'), account.get('password'))
if results:
username, cookies = results
print('Saving Cookies to Redis', username, cookies)
self.cookies_db.set(username, cookies)
def run(self):
"""
运行, 得到所有账户, 然后顺次模拟登录
:return:
"""
accounts = self.account_db.all()
cookies = self.cookies_db.all()
# Account 中对应的用户
accounts = list(accounts)
# Cookies中对应的用户
valid_users = [cookie.get('username') for cookie in cookies]
print('Getting', len(accounts), 'accounts from Redis')
if len(accounts):
self._init_browser(browser_type=self.browser_type)
for account in accounts:
if not account.get('username') in valid_users:
print('Getting Cookies of ', self.name, account.get('username'), account.get('password'))
self.set_cookies(account)
print('Generator Run Finished')
def close(self):
try:
print('Closing Browser')
self.browser.close()
del self.browser
except TypeError:
print('Browser not opened')
class WeiboCookiesGenerator(CookiesGenerator):
def __init__(self, name='weibo', browser_type=DEFAULT_BROWSER):
"""
初始化操作, 微博需要声明一个云打码引用
:param name: 名称微博
:param browser: 使用的浏览器
"""
CookiesGenerator.__init__(self, name, browser_type)
self.name = name
self.ydm = Yundama(YUNDAMA_USERNAME, YUNDAMA_PASSWORD, YUNDAMA_APP_ID, YUNDAMA_APP_KEY)
def _success(self, username):
wait = WebDriverWait(self.browser, 5)
success = wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'me_portrait_w')))
if success:
print('登录成功')
self.browser.get('http://weibo.cn/')
if "我的首页" in self.browser.title:
print(self.browser.get_cookies())
cookies = {}
for cookie in self.browser.get_cookies():
cookies[cookie["name"]] = cookie["value"]
print(cookies)
print('成功获取到Cookies')
return (username, json.dumps(cookies))
def new_cookies(self, username, password):
"""
生成Cookies
:param username: 用户名
:param password: 密码
:return: 用户名和Cookies
"""
print('Generating Cookies of', username)
self.browser.delete_all_cookies()
self.browser.get('http://my.sina.com.cn/profile/unlogin')
wait = WebDriverWait(self.browser, 20)
try:
login = wait.until(EC.visibility_of_element_located((By.ID, 'hd_login')))
login.click()
user = wait.until(
EC.visibility_of_element_located((By.CSS_SELECTOR, '.loginformlist input[name="loginname"]')))
user.send_keys(username)
psd = wait.until(
EC.visibility_of_element_located((By.CSS_SELECTOR, '.loginformlist input[name="password"]')))
psd.send_keys(password)
submit = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, '.login_btn')))
submit.click()
try:
result = self._success(username)
if result:
return result
except TimeoutException:
print('出现验证码,开始识别验证码')
yzm = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, '.loginform_yzm .yzm')))
url = yzm.get_attribute('src')
cookies = self.browser.get_cookies()
cookies_dict = {}
for cookie in cookies:
cookies_dict[cookie.get('name')] = cookie.get('value')
response = requests.get(url, cookies=cookies_dict)
result = self.ydm.identify(stream=response.content)
if not result:
print('验证码识别失败, 跳过识别')
return
door = wait.until(
EC.visibility_of_element_located((By.CSS_SELECTOR, '.loginform_yzm input[name="door"]')))
door.send_keys(result)
submit.click()
result = self._success(username)
if result:
return result
except WebDriverException as e:
print(e.args)
if __name__ == '__main__':
generator = WeiboCookiesGenerator()
generator._init_browser('Chrome')
generator.new_cookies('15197170054', 'gmwkms222')

24
cookiespool/importer.py

@ -0,0 +1,24 @@
import requests
from cookiespool.db import AccountRedisClient
conn = AccountRedisClient(name='weibo')
def set(account, sep='----'):
username, password = account.split(sep)
result = conn.set(username, password)
print('账号', username, '密码', password)
print('录入成功' if result else '录入失败')
def scan():
print('请输入账号密码组, 输入exit退出读入')
while True:
account = input()
if account == 'exit':
break
set(account)
if __name__ == '__main__':
scan()

56
cookiespool/scheduler.py

@ -0,0 +1,56 @@
import time
from multiprocessing import Process
from cookiespool.tester import *
from cookiespool.config import *
from cookiespool.api import app
import os
import signal
class Scheduler(object):
@staticmethod
def valid_cookie(cycle=CYCLE):
while True:
print('Checking Cookies')
try:
for name, cls in TESTER_MAP.items():
tester = eval(cls + '()')
tester.run()
print('Tester Finished')
del tester
time.sleep(cycle)
except Exception as e:
print(e.args)
@staticmethod
def generate_cookie(cycle=CYCLE):
while True:
print('Generating Cookies')
try:
for name, cls in GENERATOR_MAP.items():
generator = eval(cls + '()')
generator.run()
print('Generator Finished')
generator.close()
print('Deleted Generator')
time.sleep(cycle)
except Exception as e:
print(e.args)
@staticmethod
def api():
app.run(host=API_HOST, port=API_PORT)
def run(self):
if GENERATOR_PROCESS:
generate_process = Process(target=Scheduler.generate_cookie)
generate_process.start()
if VALID_PROCESS:
valid_process = Process(target=Scheduler.valid_cookie)
valid_process.start()
if API_PROCESS:
api_process = Process(target=Scheduler.api)
api_process.start()

61
cookiespool/tester.py

@ -0,0 +1,61 @@
import json
from bs4 import BeautifulSoup
import requests
from requests.exceptions import ConnectionError
from cookiespool.db import *
from cookiespool.generator import WeiboCookiesGenerator
class ValidTester(object):
def __init__(self, name='default'):
self.name = name
self.cookies_db = CookiesRedisClient(name=self.name)
self.account_db = AccountRedisClient(name=self.name)
def test(self, account, cookies):
raise NotImplementedError
def run(self):
accounts = self.cookies_db.all()
for account in accounts:
username = account.get('username')
cookies = self.cookies_db.get(username)
self.test(account, cookies)
class WeiboValidTester(ValidTester):
def __init__(self, name='weibo'):
ValidTester.__init__(self, name)
def test(self, account, cookies):
print('Testing Account', account.get('username'))
try:
cookies = json.loads(cookies)
except TypeError:
# Cookie 格式不正确
print('Invalid Cookies Value', account.get('username'))
self.cookies_db.delete(account.get('username'))
print('Deleted User', account.get('username'))
return None
try:
response = requests.get('http://weibo.cn', cookies=cookies)
if response.status_code == 200:
html = response.text
soup = BeautifulSoup(html, 'lxml')
title = soup.title.string
if title == '我的首页':
print('Valid Cookies', account.get('username'))
else:
print('Title is', title)
# Cookie已失效
print('Invalid Cookies', account.get('username'))
self.cookies_db.delete(account.get('username'))
print('Deleted User', account.get('username'))
except ConnectionError as e:
print('Error', e.args)
print('Invalid Cookies', account.get('username'))
if __name__ == '__main__':
tester = WeiboValidTester()
tester.run()

137
cookiespool/verify.py

@ -0,0 +1,137 @@
import requests
import time
from requests.exceptions import ConnectionError
from cookiespool.config import *
class Yundama():
def __init__(self, username, password, app_id, app_key, api_url=YUNDAMA_API_URL):
self.username = username
self.password = password
self.app_id = str(app_id) if not isinstance(app_id, str) else app_id
self.app_key = app_key
self.api_url = api_url
def login(self):
"""
登录云打码账户
:return:
"""
try:
data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.app_id,
'appkey': self.app_key}
response = requests.post(self.api_url, data=data)
if response.status_code == 200:
result = response.json()
print(result)
if 'ret' in result.keys() and result.get('ret') < 0:
return self.error(result.get('ret'))
else:
return result
return None
except ConnectionError:
return None
def upload(self, files, timeout, code_type):
"""
上传验证码得到识别结果
:param files:
:param timeout:
:param code_type:
:return:
"""
try:
data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.app_id,
'appkey': self.app_key, 'codetype': str(code_type), 'timeout': str(timeout)}
response = requests.post(self.api_url, data=data, files=files)
if response.status_code == 200:
return response.json()
return None
except ConnectionError:
return None
def retry(self, cid, try_count=1):
"""
临时识别不出, 传入cid重试
:param cid: 验证码ID
:param try_count: 重试次数
:return: 验证码结果
"""
if try_count >= YUNDAMA_MAX_RETRY:
return None
print('Retrying: ', cid, 'Count: ', try_count)
time.sleep(2)
try:
data = {'method': 'result', 'cid': cid}
print(data)
response = requests.post(self.api_url, data=data)
if response.status_code == 200:
result = response.json()
print(result)
if 'ret' in result.keys() and result.get('ret') < 0:
print(self.error(result.get('ret')))
if result.get('ret') == 0 and 'text' in result.keys():
return result.get('text')
else:
return self.retry(cid, try_count + 1)
return None
except ConnectionError:
return None
def identify(self, file=None, stream=None, timeout=60, code_type=5000):
"""
主函数
:param file: 文件名
:param stream: 文件流, 优先于文件名
:param timeout: 超时时间
:param code_type: 验证码类型
:return: 识别结果
"""
if stream:
files = {'file': stream}
elif file:
files = {'file': open(file, 'rb')}
else:
return None
result = self.upload(files, timeout, code_type)
if 'ret' in result.keys() and result.get('ret') < 0:
print(self.error(result.get('ret')))
if result.get('text'):
print('验证码识别成功', result.get('text'))
return result.get('text')
else:
return self.retry(result.get('cid'))
def error(self, code):
"""
报错原因
:param code: 错误码
:return: 错误原因
"""
map = {
-1001: '密码错误',
-1002: '软件ID/密钥有误',
-1003: '用户被封',
-1004: 'IP被封',
-1005: '软件被封',
-1006: '登录IP与绑定的区域不匹配',
-1007: '账号余额为零',
-2001: '验证码类型有误',
-2002: '验证码图片太大',
-2003: '验证码图片损坏',
-2004: '上传验证码图片失败',
-3001: '验证码ID不存在 ',
-3002: '验证码还在识别',
-3003: '验证码识别超时',
-3004: '验证码看不清',
-3005: '验证码报错失败',
-4001: '充值卡号不正确或已使用',
-5001: '注册用户失败'
}
return '云打码' + map.get(code)
if __name__ == '__main__':
ydm = Yundama(YUNDAMA_USERNAME, YUNDAMA_PASSWORD, YUNDAMA_APP_ID, YUNDAMA_APP_KEY)
result = ydm.identify(file='getimage.jpg')
print(result)

8
run.py

@ -0,0 +1,8 @@
from cookiespool.scheduler import Scheduler
def main():
s = Scheduler()
s.run()
if __name__ == '__main__':
main()

17
tests/login.py

@ -0,0 +1,17 @@
import requests
from bs4 import BeautifulSoup
cookies = {
'SSOLoginState': '1493555070',
'SCF': 'ArrMd41qtHmW87eTIsI-sT1IjDG8oncB9A0HbSmyDw1FwO5sbI_j6_ZellQQ07ZjTXTIBrM3Y_tpKym39f1tYWs.',
'SUB': '_2A250AacuDeRhGeRG6FIX9ybIzDiIHXVXDclmrDV6PUJbktBeLVfCkW18FWjD8r3ddYXy2abmqSauclujaw..',
'_T_WM': '5e9c698a350ddeba1c5d77e1958af21b', 'ALF': '1496147067', 'SUHB': '0G0vWR88D2VokZ',
'SUBP': '0033WrSXqPxfM725Ws9jqgMF55529P9D9W5APD86CuQDllBusA-6OZaq5JpX5o2p5NHD95QE1he7SoMRShMXWs4Dqcjci--fi-zXiK.Xi--fi-iWiKnci--ciKn4iKy2i--Xi-zRi-2Ri--4iKL2iK.4i--Ri-2NiKnf'
}
response = requests.get('http://weibo.cn', cookies=cookies)
if response.status_code == 200:
html = response.text
soup = BeautifulSoup(html, 'lxml')
title = soup.title.string
print(title)
Loading…
Cancel
Save