diff --git a/multi_proxy_refill.py b/multi_proxy_refill.py index 98fa643..e398d6e 100644 --- a/multi_proxy_refill.py +++ b/multi_proxy_refill.py @@ -26,6 +26,7 @@ SLEEP_INTERVAL = 10 db = DB.DBVidcon() + def fetch_proxies3(region_code, n): url = "https://get-ip.thordata.net/api" params = { @@ -42,9 +43,29 @@ def fetch_proxies3(region_code, n): result = ["http://" + item for item in resp.text.split("\r\n")] return result -def fetch_proxies2(region_code, n): - url = "http://" + region_code.lower() + "-pr.thordata.net:25000" - return [url for _ in range(n)] + +# def fetch_proxies2(region_code, n): +# url = "http://" + region_code.lower() + "-pr.thordata.net:25000" +# return [url for _ in range(n) + +def fetch_proxies4(region_code, n): + url = f"http://api.proxy.roxlabs.io/getProxyIp?num={n}&return_type=json&lb=1&sb=&flow=1®ions={region_code}&protocol=http" + try: + res = requests.get(url) + data_json = res.json() + except (RequestException, ValueError): + time.sleep(1) + return fetch_proxies1(region_code, n) + arr = data_json["data"] or [] + if not arr: + time.sleep(1) + return fetch_proxies1(region_code, n) + result = [] + for item in arr: + ip = item["ip"] + port = item["port"] + result.append(f"http://{ip}:{port}") + return result def fetch_proxies1(region_code, n): url = f"http://api.proxy.ipidea.io/getProxyIp?num={n}&return_type=json&lb=1&sb=0&flow=1®ions={region_code}&protocol=http" @@ -65,6 +86,7 @@ def fetch_proxies1(region_code, n): result.append(f"http://{ip}:{port}") return result + def fetch_proxies(region_code, n): """ 从第三方一次性请求 n 条代理,返回格式化列表 @@ -98,7 +120,7 @@ def fetch_proxies(region_code, n): "dl": "," } try: - resp = requests.get(url,headers=headers, params=params, timeout=10) + resp = requests.get(url, headers=headers, params=params, timeout=10) resp.raise_for_status() data = resp.json() except (RequestException, ValueError): @@ -113,7 +135,7 @@ def fetch_proxies(region_code, n): result = [] for item in arr: user = quote(item["username"], safe="") - pwd = quote(item["password"], safe="") + pwd = quote(item["password"], safe="") ip = item["ip"] port = item["port"] result.append(f"http://{user}:{pwd}@{ip}:{port}") @@ -131,7 +153,7 @@ def refill_queue(r: redis.Redis, region_name: str, region_code: str, to_fetch = batch - length print(f"[{time.strftime('%H:%M:%S')}] {key} 长度 {length} < {low},一次性拉取 {to_fetch} 条…") - proxies = fetch_proxies1(region_code, to_fetch) + proxies = fetch_proxies4(region_code, to_fetch) if proxies: r.rpush(key, *proxies) print(f"[{time.strftime('%H:%M:%S')}] 已入队 {len(proxies)} 条 → 新长度 {r.llen(key)}") @@ -164,4 +186,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main()