This commit is contained in:
2026-01-04 16:30:57 +08:00
parent 51a483ce22
commit 2950e9d2ce
134 changed files with 1180 additions and 47 deletions
+38 -20
View File
@@ -22,7 +22,7 @@ SILICONFLOW_BASE_URL = "https://api.siliconflow.cn/v1"
MODELS = [
'Pro/deepseek-ai/DeepSeek-V3.1-Terminus',
# 'deepseek-ai/DeepSeek-V3.2-Exp',
# 'Qwen/Qwen3-VL-235B-A22B-Instruct',
'Qwen/Qwen3-VL-235B-A22B-Instruct',
# 'MiniMaxAI/MiniMax-M2',
# 'zai-org/GLM-4.6',
# 'inclusionAI/Ring-flash-2.0',
@@ -31,12 +31,13 @@ MODELS = [
# 'inclusionAI/Ring-flash-2.0',
]
def process_text(text):
filter_list = ['\n', '\t', '\r', '\b', '\f', '\v', ':', '的', '或', '10', '天', '了', '可', '是', '该', ',', ' ', '、', '让', '和', '集',
'/', '日', '在', '(', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', ')', '...', ';', '%', '&', '+', ',', '.',
'/', '日', '在', '(', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', ')', '...', ';', '%', '&', '+', ',', '.',
':', ';', '<', '=', '>', '?', '[', ']', '|', '—', '。'
]
]
text_list = jieba.lcut(text)
results = []
for tl in text_list:
@@ -53,6 +54,7 @@ def process_text(text):
else:
return None
def load_keys_text():
if not os.path.exists(KEYS_TEXT):
print(f"文件不存在: {KEYS_TEXT}")
@@ -69,6 +71,7 @@ def load_keys_text():
return result_str
def txtFileLoader(file_path):
if not os.path.exists(file_path):
print(f"文件不存在: {file_path}")
@@ -81,9 +84,9 @@ def csvFileLoader(file_path, keys_text):
if not os.path.exists(file_path):
print(f"文件不存在: {file_path}")
exit(1)
data_dict = {} # 使用字典来存储,以id为键
with open(file_path, 'r', encoding='utf-8') as f:
reader = csv.reader(f)
for row in reader:
@@ -98,7 +101,7 @@ def csvFileLoader(file_path, keys_text):
'description': row[2],
'description_cn': row[11],
}
# 将字典的值转换为列表
return list(data_dict.values())
@@ -131,7 +134,20 @@ def create_result_folder():
folder_name = "generated_alpha"
if not os.path.exists(folder_name):
os.makedirs(folder_name)
return folder_name
now = datetime.now()
year_folder = os.path.join(folder_name, str(now.year))
month_folder = os.path.join(year_folder, f"{now.month:02d}")
day_folder = os.path.join(month_folder, f"{now.day:02d}")
if not os.path.exists(year_folder):
os.makedirs(year_folder)
if not os.path.exists(month_folder):
os.makedirs(month_folder)
if not os.path.exists(day_folder):
os.makedirs(day_folder)
return day_folder
def call_siliconflow(prompt, model):
@@ -161,16 +177,10 @@ def call_siliconflow(prompt, model):
def save_result(result, folder):
now = datetime.now()
date_folder = now.strftime("%Y-%m-%d")
time_filename = now.strftime("%H%M%S")
full_folder_path = os.path.join(folder, date_folder)
if not os.path.exists(full_folder_path):
os.makedirs(full_folder_path)
print(f"创建文件夹: {full_folder_path}")
filename = f"{time_filename}.txt"
filepath = os.path.join(full_folder_path, filename)
filepath = os.path.join(folder, filename)
with open(filepath, 'w', encoding='utf-8') as f:
f.write(result)
@@ -193,12 +203,22 @@ def manual_prompt(prompt):
if not os.path.exists(manual_prompt_path):
os.makedirs(manual_prompt_path)
print(f"创建文件夹: {manual_prompt_path}")
now = datetime.now()
year_folder = os.path.join(manual_prompt_path, str(now.year))
month_folder = os.path.join(year_folder, f"{now.month:02d}")
day_folder = os.path.join(month_folder, f"{now.day:02d}")
if not os.path.exists(year_folder):
os.makedirs(year_folder)
if not os.path.exists(month_folder):
os.makedirs(month_folder)
if not os.path.exists(day_folder):
os.makedirs(day_folder)
# 文件名后添加保存时间
now = datetime.now()
filename = f"manual_prompt_{now.strftime('%Y%m%d%H%M%S')}.txt"
filepath = os.path.join(manual_prompt_path, filename)
filepath = os.path.join(day_folder, filename)
with open(filepath, 'w', encoding='utf-8') as f:
f.write(prompt)
@@ -264,7 +284,6 @@ def main():
print(f'搜索数据集为空, 程序退出')
exit(1)
data_sets = 0
print(f'从数据集中提取了 {len(result_data_sets)} 条数据')
if len(result_data_sets) > 500:
@@ -272,7 +291,6 @@ def main():
else:
data_sets = result_data_sets
# 组合提示词
prompt = prepare_prompt(data_sets)