++
This commit is contained in:
@@ -22,7 +22,7 @@ SILICONFLOW_BASE_URL = "https://api.siliconflow.cn/v1"
|
||||
MODELS = [
|
||||
'Pro/deepseek-ai/DeepSeek-V3.1-Terminus',
|
||||
# 'deepseek-ai/DeepSeek-V3.2-Exp',
|
||||
# 'Qwen/Qwen3-VL-235B-A22B-Instruct',
|
||||
'Qwen/Qwen3-VL-235B-A22B-Instruct',
|
||||
# 'MiniMaxAI/MiniMax-M2',
|
||||
# 'zai-org/GLM-4.6',
|
||||
# 'inclusionAI/Ring-flash-2.0',
|
||||
@@ -31,12 +31,13 @@ MODELS = [
|
||||
# 'inclusionAI/Ring-flash-2.0',
|
||||
]
|
||||
|
||||
|
||||
def process_text(text):
|
||||
filter_list = ['\n', '\t', '\r', '\b', '\f', '\v', ':', '的', '或', '10', '天', '了', '可', '是', '该', ',', ' ', '、', '让', '和', '集',
|
||||
'/', '日', '在', '(', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', ')', '...', ';', '%', '&', '+', ',', '.',
|
||||
'/', '日', '在', '(', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', ')', '...', ';', '%', '&', '+', ',', '.',
|
||||
':', ';', '<', '=', '>', '?', '[', ']', '|', '—', '。'
|
||||
]
|
||||
|
||||
]
|
||||
|
||||
text_list = jieba.lcut(text)
|
||||
results = []
|
||||
for tl in text_list:
|
||||
@@ -53,6 +54,7 @@ def process_text(text):
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def load_keys_text():
|
||||
if not os.path.exists(KEYS_TEXT):
|
||||
print(f"文件不存在: {KEYS_TEXT}")
|
||||
@@ -69,6 +71,7 @@ def load_keys_text():
|
||||
|
||||
return result_str
|
||||
|
||||
|
||||
def txtFileLoader(file_path):
|
||||
if not os.path.exists(file_path):
|
||||
print(f"文件不存在: {file_path}")
|
||||
@@ -81,9 +84,9 @@ def csvFileLoader(file_path, keys_text):
|
||||
if not os.path.exists(file_path):
|
||||
print(f"文件不存在: {file_path}")
|
||||
exit(1)
|
||||
|
||||
|
||||
data_dict = {} # 使用字典来存储,以id为键
|
||||
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
reader = csv.reader(f)
|
||||
for row in reader:
|
||||
@@ -98,7 +101,7 @@ def csvFileLoader(file_path, keys_text):
|
||||
'description': row[2],
|
||||
'description_cn': row[11],
|
||||
}
|
||||
|
||||
|
||||
# 将字典的值转换为列表
|
||||
return list(data_dict.values())
|
||||
|
||||
@@ -131,7 +134,20 @@ def create_result_folder():
|
||||
folder_name = "generated_alpha"
|
||||
if not os.path.exists(folder_name):
|
||||
os.makedirs(folder_name)
|
||||
return folder_name
|
||||
|
||||
now = datetime.now()
|
||||
year_folder = os.path.join(folder_name, str(now.year))
|
||||
month_folder = os.path.join(year_folder, f"{now.month:02d}")
|
||||
day_folder = os.path.join(month_folder, f"{now.day:02d}")
|
||||
|
||||
if not os.path.exists(year_folder):
|
||||
os.makedirs(year_folder)
|
||||
if not os.path.exists(month_folder):
|
||||
os.makedirs(month_folder)
|
||||
if not os.path.exists(day_folder):
|
||||
os.makedirs(day_folder)
|
||||
|
||||
return day_folder
|
||||
|
||||
|
||||
def call_siliconflow(prompt, model):
|
||||
@@ -161,16 +177,10 @@ def call_siliconflow(prompt, model):
|
||||
|
||||
def save_result(result, folder):
|
||||
now = datetime.now()
|
||||
date_folder = now.strftime("%Y-%m-%d")
|
||||
time_filename = now.strftime("%H%M%S")
|
||||
full_folder_path = os.path.join(folder, date_folder)
|
||||
|
||||
if not os.path.exists(full_folder_path):
|
||||
os.makedirs(full_folder_path)
|
||||
print(f"创建文件夹: {full_folder_path}")
|
||||
|
||||
filename = f"{time_filename}.txt"
|
||||
filepath = os.path.join(full_folder_path, filename)
|
||||
filepath = os.path.join(folder, filename)
|
||||
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(result)
|
||||
@@ -193,12 +203,22 @@ def manual_prompt(prompt):
|
||||
|
||||
if not os.path.exists(manual_prompt_path):
|
||||
os.makedirs(manual_prompt_path)
|
||||
print(f"创建文件夹: {manual_prompt_path}")
|
||||
|
||||
now = datetime.now()
|
||||
year_folder = os.path.join(manual_prompt_path, str(now.year))
|
||||
month_folder = os.path.join(year_folder, f"{now.month:02d}")
|
||||
day_folder = os.path.join(month_folder, f"{now.day:02d}")
|
||||
|
||||
if not os.path.exists(year_folder):
|
||||
os.makedirs(year_folder)
|
||||
if not os.path.exists(month_folder):
|
||||
os.makedirs(month_folder)
|
||||
if not os.path.exists(day_folder):
|
||||
os.makedirs(day_folder)
|
||||
|
||||
# 文件名后添加保存时间
|
||||
now = datetime.now()
|
||||
filename = f"manual_prompt_{now.strftime('%Y%m%d%H%M%S')}.txt"
|
||||
filepath = os.path.join(manual_prompt_path, filename)
|
||||
filepath = os.path.join(day_folder, filename)
|
||||
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(prompt)
|
||||
@@ -264,7 +284,6 @@ def main():
|
||||
print(f'搜索数据集为空, 程序退出')
|
||||
exit(1)
|
||||
|
||||
|
||||
data_sets = 0
|
||||
print(f'从数据集中提取了 {len(result_data_sets)} 条数据')
|
||||
if len(result_data_sets) > 500:
|
||||
@@ -272,7 +291,6 @@ def main():
|
||||
else:
|
||||
data_sets = result_data_sets
|
||||
|
||||
|
||||
# 组合提示词
|
||||
prompt = prepare_prompt(data_sets)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user