kali/DailyMotion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 优化并发任务处理逻辑以提高搜索效率

Browse Source
This commit is contained in:
晓丰 2025-05-31 03:43:40 +08:00
parent 4b3a8ce864
commit 4db01ec6da
1 changed files with 13 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
main.py
View File

@ -358,27 +358,20 @@ def integrate_data_parallel():
if not tasks: if not tasks:
time.sleep(10) time.sleep(10)
continue continue
for payload, kitem in tasks:
proxname = proxiesdict.get(kitem['rn'])
print(proxname)
h = gettoken(proxname)
print(h)
v_list = get_searchInfo(kitem['keyword'], kitem['level'], h, proxname)
# futures = [ futures = [
# executor.submit(search_worker, payload, kitem, flag) executor.submit(search_worker, payload, kitem, flag)
# for payload, kitem in tasks for payload, kitem in tasks
# ] ]
# # 统计回滚
# # 统计回滚 rollback = {0: [], 1: [], 2: []}
# rollback = {0: [], 1: [], 2: []}
# for fut in concurrent.futures.as_completed(futures):
# for fut in concurrent.futures.as_completed(futures): ok, f_flag, payload, kitem, v_list = fut.result()
# ok, f_flag, payload, kitem, v_list = fut.result()
# if not ok:
# if not ok: rollback[f_flag].append(payload)
# rollback[f_flag].append(payload) continue
# continue
for item in v_list: for item in v_list:
if not v_list: if not v_list: