feat: 调整分页和限制参数以优化数据检索性能

main.py

@@ -254,11 +254,11 @@ def get_searchInfo(keyword, level, headers, proxy_name, r=2):
     if r == 2:
         logger.info(f"NET处理->{keyword},\trn->{proxy_name},\tlevel->{level}")
     video_list = []
-    max_page = 2
-    limit = 30
+    max_page = 4
+    limit = 10
     if level == 0 or level == 1:
-        max_page = 3
-        limit = 100
+        max_page = 10
+        limit = 20
     for j in range(1, max_page):
         # 别展开 = = !
         data = (
@@ -295,7 +295,7 @@ def get_searchInfo(keyword, level, headers, proxy_name, r=2):
                 time.sleep((3 - r) * 5)
                 return get_searchInfo(keyword, level, headers, proxy_name, r - 1)
         for index, iteminfo in enumerate(resinfo):
-            calculated_index = index + 1 + (j - 1) * 100
+            calculated_index = index + 1 + (j - 1) * limit
             node = iteminfo['node']
             if node['__typename'] != "Video":
                 continue
@@ -443,15 +443,9 @@ def parse_args() -> argparse.Namespace:
 
 
 if __name__ == '__main__':
-    # parse_args()
-    # start_time = datetime.datetime.now()
-    # logger.info(f"开始时间：{start_time.strftime('%Y-%m-%d %H:%M:%S')}")
-    # integrate_data_parallel()
-    # end_time = datetime.datetime.now()
-    # duration = end_time - start_time
-    keyword = '戀愛學園'
-    level = 0
-    headers = gettoken('TW')
-    proxy_name = "tw"
-    print(db.get_proxy(proxy))
-    # print(get_searchInfo(keyword, level, headers, proxy_name, r=2))
+    parse_args()
+    start_time = datetime.datetime.now()
+    logger.info(f"开始时间：{start_time.strftime('%Y-%m-%d %H:%M:%S')}")
+    integrate_data_parallel()
+    end_time = datetime.datetime.now()
+    duration = end_time - start_time