kali/DailyMotion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 替换日志模块并添加信息日志以增强调试能力

Browse Source
This commit is contained in:
晓丰 2025-07-17 16:44:27 +08:00
parent d1307039c8
commit 58aac99a1c
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
oneget.py
View File

@ -7,7 +7,7 @@ import random
import time import time
import copy import copy
from threading import Lock from threading import Lock
import logging from logger import logger
from DB import DBVidcon, DBSA from DB import DBVidcon, DBSA
import json import json
from requests.adapters import HTTPAdapter from requests.adapters import HTTPAdapter
@ -15,7 +15,6 @@ from urllib3.util.retry import Retry
from dateutil import parser as date_parser from dateutil import parser as date_parser
MACHINE_ID = 3 MACHINE_ID = 3
logger = logging.getLogger(__name__)
db = DBVidcon() db = DBVidcon()
proxiesdict = db.get_proxy_agent_dict() proxiesdict = db.get_proxy_agent_dict()
@ -233,7 +232,7 @@ def main():
if not kwdata: if not kwdata:
logger.error("没有获取到关键词数据") logger.error("没有获取到关键词数据")
exit(1) exit(1)
logger.info(f"搜索关键词数据: {kwdata}")
kwdata = kwdata[0][1] kwdata = kwdata[0][1]
rn = kwdata['rn'] rn = kwdata['rn']
proxy_name = proxiesdict.get(rn) proxy_name = proxiesdict.get(rn)
@ -623,6 +622,7 @@ def main():
data = response.json() data = response.json()
edges = data['data']['search']['stories']['edges'] edges = data['data']['search']['stories']['edges']
edges_len = len(edges) edges_len = len(edges)
logger.info(f"第 {i} 页，关键词: {kw}，获取到 {edges_len} 条数据")
tancks = [] tancks = []
for j, edge in enumerate(edges): for j, edge in enumerate(edges):
node = edge.get("node", {}) node = edge.get("node", {})
@ -637,7 +637,6 @@ def main():
"level": 0, "level": 0,
} }
tancks.append(s_data) tancks.append(s_data)
# 我想在这加入20 个线程池
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
executor.map(dmvideo_info.get_video_info, tancks) executor.map(dmvideo_info.get_video_info, tancks)
if edges_len < 20: if edges_len < 20: