fix: 优化视频任务处理逻辑，移除冗余代码并确保仅添加有效视频数据
This commit is contained in:
parent
64f8ed96a1
commit
01c3576d6d
101
main.py
101
main.py
@ -579,7 +579,6 @@ def get_searchInfo(keyword, level):
|
|||||||
resinfo = []
|
resinfo = []
|
||||||
print("[搜索接口]", response.text)
|
print("[搜索接口]", response.text)
|
||||||
print("返回字段解析错误!")
|
print("返回字段解析错误!")
|
||||||
video_tasks = []
|
|
||||||
for index, iteminfo in enumerate(resinfo):
|
for index, iteminfo in enumerate(resinfo):
|
||||||
calculated_index = index + 1 + (j - 1) * 100
|
calculated_index = index + 1 + (j - 1) * 100
|
||||||
node = iteminfo['node']
|
node = iteminfo['node']
|
||||||
@ -587,31 +586,13 @@ def get_searchInfo(keyword, level):
|
|||||||
continue
|
continue
|
||||||
creator = node['creator']
|
creator = node['creator']
|
||||||
duration = node.get('duration')
|
duration = node.get('duration')
|
||||||
if duration > 300:
|
if duration <= 300:
|
||||||
video_tasks.append({
|
continue
|
||||||
|
v_data = {
|
||||||
"index": calculated_index,
|
"index": calculated_index,
|
||||||
"xid": node.get('xid'),
|
|
||||||
"node": node,
|
|
||||||
"creator": creator,
|
|
||||||
})
|
|
||||||
|
|
||||||
def safe_fetch(task, max_try=2):
|
|
||||||
attempt = 0
|
|
||||||
while attempt < max_try:
|
|
||||||
try:
|
|
||||||
return fetch_video_detail(task)
|
|
||||||
except Exception as e:
|
|
||||||
attempt += 1
|
|
||||||
print(f"[线程异常] {task['xid']} 获取失败: {str(e)}")
|
|
||||||
|
|
||||||
node = task["node"]
|
|
||||||
creator = task["creator"]
|
|
||||||
avatar = creator.get("avatar", {})
|
|
||||||
return {
|
|
||||||
"index": task["index"],
|
|
||||||
"v_id": node.get("id"),
|
"v_id": node.get("id"),
|
||||||
"v_xid": task["xid"],
|
"v_xid": node.get('xid'),
|
||||||
"link": "https://www.dailymotion.com/video/" + task["xid"],
|
"link": "https://www.dailymotion.com/video/" + xid,
|
||||||
"title": node.get("title"),
|
"title": node.get("title"),
|
||||||
"createtime": node.get("createdAt"),
|
"createtime": node.get("createdAt"),
|
||||||
"duration": node.get("duration"),
|
"duration": node.get("duration"),
|
||||||
@ -622,80 +603,12 @@ def get_searchInfo(keyword, level):
|
|||||||
"u_id": creator.get('id'),
|
"u_id": creator.get('id'),
|
||||||
"u_xid": creator.get('xid'),
|
"u_xid": creator.get('xid'),
|
||||||
"u_name": creator.get('name'),
|
"u_name": creator.get('name'),
|
||||||
"u_pic": avatar.get('url'),
|
"u_pic": node.get('thumbnail').get('url')
|
||||||
"_region": Gproxies
|
|
||||||
}
|
}
|
||||||
|
video_list.append(v_data)
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
|
||||||
results = list(executor.map(safe_fetch, video_tasks))
|
|
||||||
|
|
||||||
for result in results:
|
|
||||||
if result:
|
|
||||||
video_list.append(result)
|
|
||||||
return video_list
|
return video_list
|
||||||
|
|
||||||
|
|
||||||
def fetch_video_detail(task):
|
|
||||||
xid = task["xid"]
|
|
||||||
v_info = get_videoInfo(xid)
|
|
||||||
node = task["node"]
|
|
||||||
creator = task["creator"]
|
|
||||||
avatar = creator.get("avatar", {})
|
|
||||||
return {
|
|
||||||
"index": task["index"],
|
|
||||||
"v_id": node.get("id"),
|
|
||||||
"v_xid": xid,
|
|
||||||
"link": "https://www.dailymotion.com/video/" + xid,
|
|
||||||
"title": node.get("title"),
|
|
||||||
"createtime": node.get("createdAt"),
|
|
||||||
"duration": node.get("duration"),
|
|
||||||
"pic": node.get("thumbnail", {}).get("url"),
|
|
||||||
"view": v_info['view'],
|
|
||||||
"fans": v_info['fans'],
|
|
||||||
"videos": v_info['videos'],
|
|
||||||
"u_id": creator.get('id'),
|
|
||||||
"u_xid": creator.get('xid'),
|
|
||||||
"u_name": creator.get('name'),
|
|
||||||
"u_pic": avatar.get('url')
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def get_videoInfo(x_id, r=3):
|
|
||||||
payload = {
|
|
||||||
"operationName": "WATCHING_VIDEO",
|
|
||||||
"variables": {
|
|
||||||
"xid": x_id,
|
|
||||||
"isSEO": False
|
|
||||||
},
|
|
||||||
"query": "fragment VIDEO_FRAGMENT on Video {\n id\n xid\n isPublished\n duration\n title\n description\n thumbnailx60: thumbnailURL(size: \"x60\")\n thumbnailx120: thumbnailURL(size: \"x120\")\n thumbnailx240: thumbnailURL(size: \"x240\")\n thumbnailx360: thumbnailURL(size: \"x360\")\n thumbnailx480: thumbnailURL(size: \"x480\")\n thumbnailx720: thumbnailURL(size: \"x720\")\n thumbnailx1080: thumbnailURL(size: \"x1080\")\n aspectRatio\n category\n categories(filter: {category: {eq: CONTENT_CATEGORY}}) {\n edges {\n node { id name slug __typename }\n __typename\n }\n __typename\n }\n iab_categories: categories(\n filter: {category: {eq: IAB_CATEGORY}, percentage: {gte: 70}}\n ) {\n edges {\n node { id slug __typename }\n __typename\n }\n __typename\n }\n bestAvailableQuality\n createdAt\n viewerEngagement {\n id\n liked\n favorited\n __typename\n }\n isPrivate\n isWatched\n isCreatedForKids\n isExplicit\n canDisplayAds\n videoWidth: width\n videoHeight: height\n status\n hashtags {\n edges {\n node { id name __typename }\n __typename\n }\n __typename\n }\n stats {\n id\n views { id total __typename }\n __typename\n }\n channel {\n __typename\n id\n xid\n name\n displayName\n isArtist\n logoURLx25: logoURL(size: \"x25\")\n logoURL(size: \"x60\")\n isFollowed\n accountType\n coverURLx375: coverURL(size: \"x375\")\n stats {\n id\n views { id total __typename }\n followers { id total __typename }\n videos { id total __typename }\n __typename\n }\n country { id codeAlpha2 __typename }\n organization @skip(if: $isSEO) {\n id\n xid\n owner { id xid __typename }\n __typename\n }\n }\n language { id codeAlpha2 __typename }\n tags {\n edges {\n node { id label __typename }\n __typename\n }\n __typename\n }\n moderation { id reviewedAt __typename }\n topics(whitelistedOnly: true, first: 3, page: 1) {\n edges {\n node {\n id\n xid\n name\n names {\n edges {\n node {\n id\n name\n language { id codeAlpha2 __typename }\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n geoblockedCountries {\n id\n allowed\n denied\n __typename\n }\n transcript {\n edges {\n node { id timecode text __typename }\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment LIVE_FRAGMENT on Live {\n id\n xid\n startAt\n endAt\n isPublished\n title\n description\n thumbnailx60: thumbnailURL(size: \"x60\")\n thumbnailx120: thumbnailURL(size: \"x120\")\n thumbnailx240: thumbnailURL(size: \"x240\")\n thumbnailx360: thumbnailURL(size: \"x360\")\n thumbnailx480: thumbnailURL(size: \"x480\")\n thumbnailx720: thumbnailURL(size: \"x720\")\n thumbnailx1080: thumbnailURL(size: \"x1080\")\n aspectRatio\n category\n createdAt\n viewerEngagement { id liked favorited __typename }\n isPrivate\n isExplicit\n isCreatedForKids\n bestAvailableQuality\n canDisplayAds\n videoWidth: width\n videoHeight: height\n stats { id views { id total __typename } __typename }\n channel {\n __typename\n id\n xid\n name\n displayName\n isArtist\n logoURLx25: logoURL(size: \"x25\")\n logoURL(size: \"x60\")\n isFollowed\n accountType\n coverURLx375: coverURL(size: \"x375\")\n stats { id views { id total __typename } followers { id total __typename } videos { id total __typename } __typename }\n country { id codeAlpha2 __typename }\n organization @skip(if: $isSEO) { id xid owner { id xid __typename } __typename }\n }\n language { id codeAlpha2 __typename }\n tags { edges { node { id label __typename } __typename } __typename }\n moderation { id reviewedAt __typename }\n topics(whitelistedOnly: true, first: 3, page: 1) {\n edges { node { id xid name names { edges { node { id name language { id codeAlpha2 __typename } __typename } __typename } __typename } __typename } __typename }\n __typename\n }\n geoblockedCountries { id allowed denied __typename }\n __typename\n}\n\nquery WATCHING_VIDEO($xid: String!, $isSEO: Boolean!) {\n video: media(xid: $xid) {\n __typename\n ... on Video { id ...VIDEO_FRAGMENT __typename }\n ... on Live { id ...LIVE_FRAGMENT __typename }\n }\n}"
|
|
||||||
}
|
|
||||||
url = 'https://graphql.api.dailymotion.com/'
|
|
||||||
|
|
||||||
response = post_with_retry(
|
|
||||||
url,
|
|
||||||
json_payload=payload,
|
|
||||||
headers=headers1,
|
|
||||||
proxies=None,
|
|
||||||
)
|
|
||||||
jsondata = response.json()
|
|
||||||
try:
|
|
||||||
v_info = jsondata['data']['video']['channel']['stats']
|
|
||||||
except Exception:
|
|
||||||
if r > 0:
|
|
||||||
return get_videoInfo(x_id=x_id, r=r - 1)
|
|
||||||
else:
|
|
||||||
return {
|
|
||||||
"view": '-',
|
|
||||||
"fans": '-',
|
|
||||||
"videos": '-',
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
"view": v_info['views']['total'],
|
|
||||||
"fans": v_info['followers']['total'],
|
|
||||||
"videos": v_info['videos']['total'],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def integrate_data():
|
def integrate_data():
|
||||||
while True:
|
while True:
|
||||||
keywords, flag = db.item_keyword()
|
keywords, flag = db.item_keyword()
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user