feat: 添加 fetch_proxies4 方法以从新 API 获取代理 IP,并更新 fetch_proxies 函数以使用新方法

This commit is contained in:
晓丰 2025-05-28 20:57:57 +08:00
parent 2004aa0900
commit 38562aba60

View File

@ -26,6 +26,7 @@ SLEEP_INTERVAL = 10
db = DB.DBVidcon() db = DB.DBVidcon()
def fetch_proxies3(region_code, n): def fetch_proxies3(region_code, n):
url = "https://get-ip.thordata.net/api" url = "https://get-ip.thordata.net/api"
params = { params = {
@ -42,9 +43,29 @@ def fetch_proxies3(region_code, n):
result = ["http://" + item for item in resp.text.split("\r\n")] result = ["http://" + item for item in resp.text.split("\r\n")]
return result return result
def fetch_proxies2(region_code, n):
url = "http://" + region_code.lower() + "-pr.thordata.net:25000" # def fetch_proxies2(region_code, n):
return [url for _ in range(n)] # url = "http://" + region_code.lower() + "-pr.thordata.net:25000"
# return [url for _ in range(n)
def fetch_proxies4(region_code, n):
url = f"http://api.proxy.roxlabs.io/getProxyIp?num={n}&return_type=json&lb=1&sb=&flow=1&regions={region_code}&protocol=http"
try:
res = requests.get(url)
data_json = res.json()
except (RequestException, ValueError):
time.sleep(1)
return fetch_proxies1(region_code, n)
arr = data_json["data"] or []
if not arr:
time.sleep(1)
return fetch_proxies1(region_code, n)
result = []
for item in arr:
ip = item["ip"]
port = item["port"]
result.append(f"http://{ip}:{port}")
return result
def fetch_proxies1(region_code, n): def fetch_proxies1(region_code, n):
url = f"http://api.proxy.ipidea.io/getProxyIp?num={n}&return_type=json&lb=1&sb=0&flow=1&regions={region_code}&protocol=http" url = f"http://api.proxy.ipidea.io/getProxyIp?num={n}&return_type=json&lb=1&sb=0&flow=1&regions={region_code}&protocol=http"
@ -65,6 +86,7 @@ def fetch_proxies1(region_code, n):
result.append(f"http://{ip}:{port}") result.append(f"http://{ip}:{port}")
return result return result
def fetch_proxies(region_code, n): def fetch_proxies(region_code, n):
""" """
从第三方一次性请求 n 条代理返回格式化列表 从第三方一次性请求 n 条代理返回格式化列表
@ -98,7 +120,7 @@ def fetch_proxies(region_code, n):
"dl": "," "dl": ","
} }
try: try:
resp = requests.get(url,headers=headers, params=params, timeout=10) resp = requests.get(url, headers=headers, params=params, timeout=10)
resp.raise_for_status() resp.raise_for_status()
data = resp.json() data = resp.json()
except (RequestException, ValueError): except (RequestException, ValueError):
@ -113,7 +135,7 @@ def fetch_proxies(region_code, n):
result = [] result = []
for item in arr: for item in arr:
user = quote(item["username"], safe="") user = quote(item["username"], safe="")
pwd = quote(item["password"], safe="") pwd = quote(item["password"], safe="")
ip = item["ip"] ip = item["ip"]
port = item["port"] port = item["port"]
result.append(f"http://{user}:{pwd}@{ip}:{port}") result.append(f"http://{user}:{pwd}@{ip}:{port}")
@ -131,7 +153,7 @@ def refill_queue(r: redis.Redis, region_name: str, region_code: str,
to_fetch = batch - length to_fetch = batch - length
print(f"[{time.strftime('%H:%M:%S')}] {key} 长度 {length} < {low},一次性拉取 {to_fetch} 条…") print(f"[{time.strftime('%H:%M:%S')}] {key} 长度 {length} < {low},一次性拉取 {to_fetch} 条…")
proxies = fetch_proxies1(region_code, to_fetch) proxies = fetch_proxies4(region_code, to_fetch)
if proxies: if proxies:
r.rpush(key, *proxies) r.rpush(key, *proxies)
print(f"[{time.strftime('%H:%M:%S')}] 已入队 {len(proxies)} 条 → 新长度 {r.llen(key)}") print(f"[{time.strftime('%H:%M:%S')}] 已入队 {len(proxies)} 条 → 新长度 {r.llen(key)}")
@ -164,4 +186,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()