feat: 添加 fetch_proxies4 方法以从新 API 获取代理 IP,并更新 fetch_proxies 函数以使用新方法
This commit is contained in:
parent
2004aa0900
commit
38562aba60
@ -26,6 +26,7 @@ SLEEP_INTERVAL = 10
|
|||||||
|
|
||||||
db = DB.DBVidcon()
|
db = DB.DBVidcon()
|
||||||
|
|
||||||
|
|
||||||
def fetch_proxies3(region_code, n):
|
def fetch_proxies3(region_code, n):
|
||||||
url = "https://get-ip.thordata.net/api"
|
url = "https://get-ip.thordata.net/api"
|
||||||
params = {
|
params = {
|
||||||
@ -42,9 +43,29 @@ def fetch_proxies3(region_code, n):
|
|||||||
result = ["http://" + item for item in resp.text.split("\r\n")]
|
result = ["http://" + item for item in resp.text.split("\r\n")]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def fetch_proxies2(region_code, n):
|
|
||||||
url = "http://" + region_code.lower() + "-pr.thordata.net:25000"
|
# def fetch_proxies2(region_code, n):
|
||||||
return [url for _ in range(n)]
|
# url = "http://" + region_code.lower() + "-pr.thordata.net:25000"
|
||||||
|
# return [url for _ in range(n)
|
||||||
|
|
||||||
|
def fetch_proxies4(region_code, n):
|
||||||
|
url = f"http://api.proxy.roxlabs.io/getProxyIp?num={n}&return_type=json&lb=1&sb=&flow=1®ions={region_code}&protocol=http"
|
||||||
|
try:
|
||||||
|
res = requests.get(url)
|
||||||
|
data_json = res.json()
|
||||||
|
except (RequestException, ValueError):
|
||||||
|
time.sleep(1)
|
||||||
|
return fetch_proxies1(region_code, n)
|
||||||
|
arr = data_json["data"] or []
|
||||||
|
if not arr:
|
||||||
|
time.sleep(1)
|
||||||
|
return fetch_proxies1(region_code, n)
|
||||||
|
result = []
|
||||||
|
for item in arr:
|
||||||
|
ip = item["ip"]
|
||||||
|
port = item["port"]
|
||||||
|
result.append(f"http://{ip}:{port}")
|
||||||
|
return result
|
||||||
|
|
||||||
def fetch_proxies1(region_code, n):
|
def fetch_proxies1(region_code, n):
|
||||||
url = f"http://api.proxy.ipidea.io/getProxyIp?num={n}&return_type=json&lb=1&sb=0&flow=1®ions={region_code}&protocol=http"
|
url = f"http://api.proxy.ipidea.io/getProxyIp?num={n}&return_type=json&lb=1&sb=0&flow=1®ions={region_code}&protocol=http"
|
||||||
@ -65,6 +86,7 @@ def fetch_proxies1(region_code, n):
|
|||||||
result.append(f"http://{ip}:{port}")
|
result.append(f"http://{ip}:{port}")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def fetch_proxies(region_code, n):
|
def fetch_proxies(region_code, n):
|
||||||
"""
|
"""
|
||||||
从第三方一次性请求 n 条代理,返回格式化列表
|
从第三方一次性请求 n 条代理,返回格式化列表
|
||||||
@ -131,7 +153,7 @@ def refill_queue(r: redis.Redis, region_name: str, region_code: str,
|
|||||||
|
|
||||||
to_fetch = batch - length
|
to_fetch = batch - length
|
||||||
print(f"[{time.strftime('%H:%M:%S')}] {key} 长度 {length} < {low},一次性拉取 {to_fetch} 条…")
|
print(f"[{time.strftime('%H:%M:%S')}] {key} 长度 {length} < {low},一次性拉取 {to_fetch} 条…")
|
||||||
proxies = fetch_proxies1(region_code, to_fetch)
|
proxies = fetch_proxies4(region_code, to_fetch)
|
||||||
if proxies:
|
if proxies:
|
||||||
r.rpush(key, *proxies)
|
r.rpush(key, *proxies)
|
||||||
print(f"[{time.strftime('%H:%M:%S')}] 已入队 {len(proxies)} 条 → 新长度 {r.llen(key)}")
|
print(f"[{time.strftime('%H:%M:%S')}] 已入队 {len(proxies)} 条 → 新长度 {r.llen(key)}")
|
||||||
|
Loading…
x
Reference in New Issue
Block a user