feat: 添加 fetch_proxies4 方法以从新 API 获取代理 IP,并更新 fetch_proxies 函数以使用新方法
This commit is contained in:
parent
2004aa0900
commit
38562aba60
@ -26,6 +26,7 @@ SLEEP_INTERVAL = 10
|
||||
|
||||
db = DB.DBVidcon()
|
||||
|
||||
|
||||
def fetch_proxies3(region_code, n):
|
||||
url = "https://get-ip.thordata.net/api"
|
||||
params = {
|
||||
@ -42,9 +43,29 @@ def fetch_proxies3(region_code, n):
|
||||
result = ["http://" + item for item in resp.text.split("\r\n")]
|
||||
return result
|
||||
|
||||
def fetch_proxies2(region_code, n):
|
||||
url = "http://" + region_code.lower() + "-pr.thordata.net:25000"
|
||||
return [url for _ in range(n)]
|
||||
|
||||
# def fetch_proxies2(region_code, n):
|
||||
# url = "http://" + region_code.lower() + "-pr.thordata.net:25000"
|
||||
# return [url for _ in range(n)
|
||||
|
||||
def fetch_proxies4(region_code, n):
|
||||
url = f"http://api.proxy.roxlabs.io/getProxyIp?num={n}&return_type=json&lb=1&sb=&flow=1®ions={region_code}&protocol=http"
|
||||
try:
|
||||
res = requests.get(url)
|
||||
data_json = res.json()
|
||||
except (RequestException, ValueError):
|
||||
time.sleep(1)
|
||||
return fetch_proxies1(region_code, n)
|
||||
arr = data_json["data"] or []
|
||||
if not arr:
|
||||
time.sleep(1)
|
||||
return fetch_proxies1(region_code, n)
|
||||
result = []
|
||||
for item in arr:
|
||||
ip = item["ip"]
|
||||
port = item["port"]
|
||||
result.append(f"http://{ip}:{port}")
|
||||
return result
|
||||
|
||||
def fetch_proxies1(region_code, n):
|
||||
url = f"http://api.proxy.ipidea.io/getProxyIp?num={n}&return_type=json&lb=1&sb=0&flow=1®ions={region_code}&protocol=http"
|
||||
@ -65,6 +86,7 @@ def fetch_proxies1(region_code, n):
|
||||
result.append(f"http://{ip}:{port}")
|
||||
return result
|
||||
|
||||
|
||||
def fetch_proxies(region_code, n):
|
||||
"""
|
||||
从第三方一次性请求 n 条代理,返回格式化列表
|
||||
@ -98,7 +120,7 @@ def fetch_proxies(region_code, n):
|
||||
"dl": ","
|
||||
}
|
||||
try:
|
||||
resp = requests.get(url,headers=headers, params=params, timeout=10)
|
||||
resp = requests.get(url, headers=headers, params=params, timeout=10)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except (RequestException, ValueError):
|
||||
@ -113,7 +135,7 @@ def fetch_proxies(region_code, n):
|
||||
result = []
|
||||
for item in arr:
|
||||
user = quote(item["username"], safe="")
|
||||
pwd = quote(item["password"], safe="")
|
||||
pwd = quote(item["password"], safe="")
|
||||
ip = item["ip"]
|
||||
port = item["port"]
|
||||
result.append(f"http://{user}:{pwd}@{ip}:{port}")
|
||||
@ -131,7 +153,7 @@ def refill_queue(r: redis.Redis, region_name: str, region_code: str,
|
||||
|
||||
to_fetch = batch - length
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {key} 长度 {length} < {low},一次性拉取 {to_fetch} 条…")
|
||||
proxies = fetch_proxies1(region_code, to_fetch)
|
||||
proxies = fetch_proxies4(region_code, to_fetch)
|
||||
if proxies:
|
||||
r.rpush(key, *proxies)
|
||||
print(f"[{time.strftime('%H:%M:%S')}] 已入队 {len(proxies)} 条 → 新长度 {r.llen(key)}")
|
||||
@ -164,4 +186,4 @@ def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
Loading…
x
Reference in New Issue
Block a user