kali/DailyMotion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 调整分页和限制参数以优化数据检索性能

Browse Source
This commit is contained in:
晓丰 2025-05-30 22:35:54 +08:00
parent 0468bcea03
commit 5b1c6bc358
1 changed files with 11 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
main.py
View File

@ -254,11 +254,11 @@ def get_searchInfo(keyword, level, headers, proxy_name, r=2):
if r == 2: if r == 2:
logger.info(f"NET处理->{keyword},\trn->{proxy_name},\tlevel->{level}") logger.info(f"NET处理->{keyword},\trn->{proxy_name},\tlevel->{level}")
video_list = [] video_list = []
max_page = 2 max_page = 4
limit = 30 limit = 10
if level == 0 or level == 1: if level == 0 or level == 1:
max_page = 3 max_page = 10
limit = 100 limit = 20
for j in range(1, max_page): for j in range(1, max_page):
# 别展开 = = ! # 别展开 = = !
data = ( data = (
@ -295,7 +295,7 @@ def get_searchInfo(keyword, level, headers, proxy_name, r=2):
time.sleep((3 - r) * 5) time.sleep((3 - r) * 5)
return get_searchInfo(keyword, level, headers, proxy_name, r - 1) return get_searchInfo(keyword, level, headers, proxy_name, r - 1)
for index, iteminfo in enumerate(resinfo): for index, iteminfo in enumerate(resinfo):
calculated_index = index + 1 + (j - 1) * 100 calculated_index = index + 1 + (j - 1) * limit
node = iteminfo['node'] node = iteminfo['node']
if node['__typename'] != "Video": if node['__typename'] != "Video":
continue continue
@ -443,15 +443,9 @@ def parse_args() -> argparse.Namespace:
if __name__ == '__main__': if __name__ == '__main__':
# parse_args() parse_args()
# start_time = datetime.datetime.now() start_time = datetime.datetime.now()
# logger.info(f"开始时间：{start_time.strftime('%Y-%m-%d %H:%M:%S')}") logger.info(f"开始时间：{start_time.strftime('%Y-%m-%d %H:%M:%S')}")
# integrate_data_parallel() integrate_data_parallel()
# end_time = datetime.datetime.now() end_time = datetime.datetime.now()
# duration = end_time - start_time duration = end_time - start_time
keyword = '戀愛學園'
level = 0
headers = gettoken('TW')
proxy_name = "tw"
print(db.get_proxy(proxy))
# print(get_searchInfo(keyword, level, headers, proxy_name, r=2))