-
Notifications
You must be signed in to change notification settings - Fork 1
/
proxyhelper.py
61 lines (46 loc) · 1.62 KB
/
proxyhelper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
import db as db
import queue
global _proxies_queues
_proxies_queues = {}
def _fetch_proxies(protocol='http', token_level=0):
global _proxies_queues
protocol_key = protocol + '_' + str(token_level)
if protocol_key not in _proxies_queues:
_proxies_queues[protocol_key] = queue.Queue()
_queue = _proxies_queues[protocol_key]
if _queue.empty():
all_proxies = db.proxies.find({'protocol': protocol, 'succeed': {'$gt': 0}}).sort([('usability', -1)]).limit(
int(token_level) * 1000 + 100)
for proxy in all_proxies:
_queue.put(proxy)
return _queue
def next(protocol, token=None):
token_level = 0
if token:
token_in_db = db.tokens.find_one({'token': token, 'status': "active"})
if token_in_db:
token_level = token_in_db['level']
_proxies = _fetch_proxies(protocol, token_level)
proxy = _proxies.get_nowait()
_proxies.put(proxy)
return proxy
def get_all(protocol, token=None):
token_level = 0
if not token:
return []
token_in_db = db.tokens.find_one({'token': token, 'status': "active"})
if token_in_db:
token_level = token_in_db['token_level']
print('token_level:' + str(token_level))
return list(db.proxies.find({'protocol': protocol, 'succeed': {'$gt': 0}}).sort([('usability', -1)]).limit(
int(token_level) * 1000 + 100))
global _proxy_count
_proxy_count = 0
def get_proxy_count():
global _proxy_count
return _proxy_count
def update_proxy():
global _proxies_queues
global _proxy_count
_proxy_count = db.proxies.count_documents({})
_proxies_queues = {}