58 lines
1.5 KiB
Python
58 lines
1.5 KiB
Python
import json, time
|
||
import argparse
|
||
from DB import DBVidcon, DBSA
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(
|
||
description="Dump keyword/title rows into Redis list."
|
||
)
|
||
parser.add_argument("-l", "--level", type=int, default=99,
|
||
help="value for t.level (default: 99)")
|
||
return parser.parse_args()
|
||
|
||
def main():
|
||
args = parse_args()
|
||
batch = int(time.time())
|
||
db = DBVidcon()
|
||
|
||
for chunk in DBSA.stream_video_keys(chunk_size=10_000):
|
||
db.cache_video_keys_bulk(chunk)
|
||
print(f"同步Redis=={len(chunk)}")
|
||
|
||
if args.level == 0:
|
||
push = db.push_l0
|
||
empty = db.l0_empty
|
||
elif args.level == 1:
|
||
push = db.push_l1
|
||
empty = db.l1_empty
|
||
elif args.level == 9:
|
||
push = db.push_l2
|
||
empty = db.l2_empty
|
||
else:
|
||
return
|
||
|
||
if empty():
|
||
rows = db.fetch_keyword_title(level=args.level)
|
||
payload_list = []
|
||
|
||
|
||
for row in rows:
|
||
payload_list.append(json.dumps({**row, "batch": batch}, ensure_ascii=False))
|
||
if len(payload_list) >= 10000:
|
||
push(payload_list)
|
||
payload_list.clear()
|
||
if payload_list: # 收尾
|
||
push(payload_list)
|
||
|
||
data = {
|
||
"level": args.level,
|
||
"batch": batch,
|
||
"count": len(rows),
|
||
}
|
||
db.log_batch_start(data)
|
||
print(f"✔ 推送 {len(rows)} 行（batch={batch}）到 {push.__name__}队列完毕")
|
||
db.close()
|
||
|
||
if __name__ == "__main__":
|
||
main()
|