feat: 修改 get_searchInfo 函数,添加代理参数以支持代理请求
This commit is contained in:
parent
91a1113d51
commit
82f134fe2f
28
main.py
28
main.py
@ -1,3 +1,4 @@
|
||||
import json
|
||||
import random
|
||||
import traceback
|
||||
from urllib.parse import quote
|
||||
@ -227,7 +228,7 @@ def gettoken(proxy, r=2):
|
||||
return copy.deepcopy(headers1)
|
||||
|
||||
|
||||
def get_searchInfo(keyword, level, headers, proxy_name):
|
||||
def get_searchInfo(keyword, level, headers, proxy_name, r=2):
|
||||
video_list = []
|
||||
max_page = 2
|
||||
limit = 30
|
||||
@ -589,12 +590,25 @@ def get_searchInfo(keyword, level, headers, proxy_name):
|
||||
|
||||
jsondata = response.json()
|
||||
try:
|
||||
resinfo = jsondata['data']['search']['stories']['edges']
|
||||
print('resinfo :', len(resinfo))
|
||||
errors = jsondata.get("errors") # GraphQL errors 数组
|
||||
stories = jsondata.get("data", {}).get("search", {}).get("stories")
|
||||
|
||||
if errors or stories is None: # 有错误 或 stories 为 null
|
||||
if r == 0:
|
||||
print("连续 3 次错误或空结果:", json.dumps(jsondata, ensure_ascii=False))
|
||||
return None
|
||||
time.sleep((3 - r) * 5)
|
||||
return get_searchInfo(keyword, level, headers, proxy_name, r - 1)
|
||||
resinfo = stories["edges"]
|
||||
print("resinfo :", len(resinfo))
|
||||
except Exception:
|
||||
resinfo = []
|
||||
print("[搜索接口]", response.text)
|
||||
if r < 0:
|
||||
print("[搜索接口] 未知:未处理", response.text)
|
||||
print("返回字段解析错误!")
|
||||
return None
|
||||
else:
|
||||
time.sleep((3 - r) * 5)
|
||||
return get_searchInfo(keyword, level, headers, proxy_name, r - 1)
|
||||
for index, iteminfo in enumerate(resinfo):
|
||||
calculated_index = index + 1 + (j - 1) * 100
|
||||
node = iteminfo['node']
|
||||
@ -636,7 +650,7 @@ def search_worker(payload, kitem, flag):
|
||||
if not v_list:
|
||||
for i in range(2):
|
||||
time.sleep(i * 5)
|
||||
v_list = get_searchInfo(kitem['keyword'], kitem['level'], header)
|
||||
v_list = get_searchInfo(kitem['keyword'], kitem['level'], header, gproxies)
|
||||
if v_list:
|
||||
break
|
||||
time.sleep(2)
|
||||
@ -649,6 +663,8 @@ def search_worker(payload, kitem, flag):
|
||||
|
||||
|
||||
executor = concurrent.futures.ThreadPoolExecutor(MAX_WORKERS)
|
||||
|
||||
|
||||
def integrate_data_parallel():
|
||||
while True:
|
||||
tasks, flag = db.item_keyword()
|
||||
|
Loading…
x
Reference in New Issue
Block a user