kali/DailyMotion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 重构数据集成逻辑，支持并行处理和异常回滚机制

Browse Source
This commit is contained in:
晓丰 2025-05-20 23:16:32 +08:00
parent 456be9f8c5
commit f9e88e98c9
1 changed files with 62 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
main.py
View File

@ -1,4 +1,5 @@
import random import random
import traceback
from urllib.parse import quote from urllib.parse import quote
import argparse import argparse
import time import time
@ -627,86 +628,82 @@ def get_searchInfo(keyword, level, headers):
proxiesdict = db.get_proxy_agent_dict() proxiesdict = db.get_proxy_agent_dict()
def search_worker(payload, kitem): def search_worker(payload, kitem, flag):
try: try:
gproxies = proxiesdict[kitem['rn']] gproxies = proxiesdict[kitem['rn']]
header = gettoken(gproxies) header = gettoken(gproxies)
v_list = get_searchInfo(kitem['keyword'], kitem['level'], header) v_list = get_searchInfo(kitem['keyword'], kitem['level'], header)
if not v_list: if not v_list:
for i in range(2): for i in range(2):
time.sleep(i * 5) time.sleep(i * 5)
v_list = get_searchInfo(kitem['keyword'], kitem['level']) v_list = get_searchInfo(kitem['keyword'], kitem['level'], header)
if v_list: if v_list:
break break
time.sleep(2) time.sleep(2)
return payload, kitem, v_list return True, flag, payload, kitem, v_list # 成功
except Exception as e: except Exception as e:
raise RuntimeError(f"{kitem['keyword']} 处理失败: {e}") from e print(f"[线程异常] {kitem['keyword']} → {e}")
traceback.print_exc()
return False, flag, payload, kitem, [] # 失败
def integrate_data(): executor = concurrent.futures.ThreadPoolExecutor(MAX_WORKERS)
def integrate_data_parallel():
while True: while True:
keywords, flag = db.item_keyword() tasks, flag = db.item_keyword()
if len(keywords) < 1: if not tasks:
time.sleep(30) time.sleep(10)
else: continue
for index, (payload, kitem) in enumerate(keywords):
try:
proxiesdict = db.get_proxy_agent_dict()
global Gproxies
Gproxies = proxiesdict[kitem['rn']]
gettoken()
v_list = get_searchInfo(kitem['keyword'], kitem['level'])
if not v_list: futures = [
for i in range(3): executor.submit(search_worker, payload, kitem, flag)
time.sleep(i * 5) for payload, kitem in tasks
v_list = get_searchInfo(kitem["keyword"], kitem['level'], headers) ]
if v_list:
break
time.sleep(2)
for item in v_list: # 统计回滚
record = { rollback = {0: [], 1: [], 2: []}
"keyword": kitem.get("keyword"),
"v_name": kitem.get("v_name"), for fut in concurrent.futures.as_completed(futures):
"v_id": item.get("v_id"), ok, f_flag, payload, kitem, v_list = fut.result()
"v_xid": item.get("v_xid"),
"link": item.get("link"), if not ok:
"title": item.get("title"), rollback[f_flag].append(payload)
"duration": format_duration(item.get("duration")), continue
"fans": clean_dash_to_zero(item.get("fans", 0)),
"videos": clean_dash_to_zero(item.get("videos", 0)), # —— 写库：可按你原来的 upsert / flush 逻辑 ——
"watch_number": clean_dash_to_zero(item.get("view", 0)), for item in v_list:
"create_time": format_create_time(item.get("createtime")), record = {
"cover_pic": item.get("pic"), "keyword": kitem["keyword"],
"index": item.get("index", 0), "v_name": kitem["v_name"],
"u_id": item.get("u_id"), "v_id": item["v_id"],
"u_xid": item.get("u_xid"), "v_xid": item["v_xid"],
"u_name": item.get("u_name"), "link": item["link"],
"u_pic": item.get("u_pic"), "title": item["title"],
"rn": kitem.get("rn"), "duration": format_duration(item["duration"]),
"batch": kitem['batch'], "fans": clean_dash_to_zero(item["fans"]),
"machine_id": MACHINE_ID, "videos": clean_dash_to_zero(item["videos"]),
"level": kitem['level'], "watch_number": clean_dash_to_zero(item["view"]),
} "create_time": format_create_time(item["createtime"]),
db.upsert_video(record) "cover_pic": item["pic"],
db.flush() "index": item["index"],
except Exception as e: "u_id": item["u_id"],
print(f"[异常] {str(e.__class__.__name__)}: {str(e)}") "u_xid": item["u_xid"],
print(f"[异常] 处理关键词 {kitem['keyword']} 时发生错误，正在回滚...") "u_name": item["u_name"],
time.sleep(5) "u_pic": item["u_pic"],
remaining_payloads = [p for p, _ in keywords[index:]] "rn": kitem["rn"],
if flag == 0: "batch": kitem["batch"],
db.rollback_l0(remaining_payloads) "machine_id": MACHINE_ID,
elif flag == 1: "level": kitem["level"],
db.rollback_l1(remaining_payloads) }
elif flag == 2: db.upsert_video(record)
db.rollback_l2(remaining_payloads) db.flush()
time.sleep(5) if rollback[0]:
break db.rollback_l0(rollback[0])
if rollback[1]:
db.rollback_l1(rollback[1])
if rollback[2]:
db.rollback_l2(rollback[2])
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
@ -744,6 +741,6 @@ if __name__ == '__main__':
parse_args() parse_args()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
print(f"开始时间：{start_time.strftime('%Y-%m-%d %H:%M:%S')}") print(f"开始时间：{start_time.strftime('%Y-%m-%d %H:%M:%S')}")
integrate_data() integrate_data_parallel()
end_time = datetime.datetime.now() end_time = datetime.datetime.now()
duration = end_time - start_time duration = end_time - start_time