++
This commit is contained in:
jack
@@ -1,39 +1,26 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
使用 AI 总结数据集名称以及中英文描述, 生成出 tags
|
||||
传入金融逻辑描述
|
||||
1, 拆分词条
|
||||
2, 将词条作为 key, 在数据库查找, 获取合适使用的数据集
|
||||
'''
|
||||
import psycopg2
|
||||
import jieba
|
||||
|
||||
# 数据库连接参数
|
||||
db_config = {
|
||||
'host': '192.168.31.201',
|
||||
'port': 5432,
|
||||
'database': 'alpha',
|
||||
'user': 'jack',
|
||||
'password': 'aaaAAA111'
|
||||
}
|
||||
|
||||
def process_text(text):
|
||||
filter_list = ['\n', '\t', '\r', '\b', '\f', '\v', '：', '的', '或', '10', '天', '了', '可', '是', '该', '，', ' ', '、', '让', '和', '集',
|
||||
'/', '日', '在', '（', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', '）', '...', '；', '%', '&', '+', ',', '.',
|
||||
':', ';', '<', '=', '>', '?', '[', ']', '|', '—', '。'
|
||||
'/', '日', '在', '（', '_', '-', ')', '(', '上', '距', '与', '比', '下', '及', '）', '...', '；', '%', '&', '+', ',', '.',
|
||||
':', ';', '<', '=', '>', '?', '[', ']', '|', '—', '。'
|
||||
]
|
||||
|
||||
# 使用 jieba 进行分词
|
||||
|
||||
text_list = jieba.lcut(text)
|
||||
|
||||
# 过滤掉包含 filter_list 中任何字符的元素
|
||||
results = []
|
||||
for tl in text_list:
|
||||
# 检查当前元素是否包含 filter_list 中的任何字符
|
||||
should_include = True
|
||||
for fl in filter_list:
|
||||
if fl == tl:
|
||||
should_include = False
|
||||
break
|
||||
|
||||
# 如果不包含任何 filter_list 中的字符，则添加到结果
|
||||
if should_include:
|
||||
results.append(tl)
|
||||
|
||||
@@ -42,38 +29,78 @@ def process_text(text):
|
||||
else:
|
||||
return None
|
||||
|
||||
def query_datasets_by_text(text):
|
||||
db_config = {
|
||||
'host': '192.168.31.201',
|
||||
'port': 5432,
|
||||
'database': 'alpha',
|
||||
'user': 'jack',
|
||||
'password': 'aaaAAA111'
|
||||
}
|
||||
|
||||
results = []
|
||||
f_list = []
|
||||
try:
|
||||
# 连接数据库
|
||||
conn = psycopg2.connect(**db_config)
|
||||
# 创建游标
|
||||
cur = conn.cursor()
|
||||
|
||||
# SQL 查询语句
|
||||
sql = """select * from data_sets order by id asc"""
|
||||
|
||||
# 执行查询
|
||||
cur.execute(sql)
|
||||
keywords_list = process_text(text)
|
||||
|
||||
# 获取所有结果
|
||||
rows = cur.fetchall()
|
||||
if not keywords_list:
|
||||
print("未提取到有效关键词")
|
||||
return []
|
||||
|
||||
print(f"提取到的关键词: {keywords_list}")
|
||||
|
||||
return query_datasets_by_keywords(keywords_list, db_config)
|
||||
|
||||
# 将每一行转换为字典
|
||||
for row in rows:
|
||||
result = process_text(row[11])
|
||||
def query_datasets_by_keywords(keywords_list, db_config):
|
||||
if not keywords_list:
|
||||
return []
|
||||
|
||||
results = []
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(**db_config)
|
||||
cur = conn.cursor()
|
||||
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
for keyword in keywords_list:
|
||||
conditions.append("name LIKE %s")
|
||||
params.append(f'%{keyword}%')
|
||||
|
||||
sql = f"""
|
||||
SELECT * FROM data_sets
|
||||
WHERE {' OR '.join(conditions)}
|
||||
ORDER BY id ASC
|
||||
"""
|
||||
|
||||
cur.execute(sql, params)
|
||||
rows = cur.fetchall()
|
||||
|
||||
for row in rows:
|
||||
results.append({
|
||||
"id": row[0],
|
||||
"name": row[1],
|
||||
"description": row[2],
|
||||
})
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"查询过程出错: {e}")
|
||||
raise
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# 关闭游标和连接
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
print("数据库连接或查询出错:", e)
|
||||
|
||||
|
||||
for result in results:
|
||||
print(result)
|
||||
|
||||
print(f"本次搜索共 {len(results)} 条数据")
|
||||
if __name__ == "__main__":
|
||||
test_text = """
|
||||
盈利
|
||||
"""
|
||||
|
||||
try:
|
||||
results = query_datasets_by_text(test_text)
|
||||
print(f"\n查询结果 ({len(results)} 条):")
|
||||
for result in results:
|
||||
print(result)
|
||||
|
||||
except Exception as e:
|
||||
print(f"程序执行出错: {e}")
|
||||
@@ -0,0 +1,55 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import psycopg2
|
||||
|
||||
# 数据库连接参数
|
||||
db_config = {
|
||||
'host': '192.168.31.201',
|
||||
'port': 5432,
|
||||
'database': 'alpha',
|
||||
'user': 'jack',
|
||||
'password': 'aaaAAA111'
|
||||
}
|
||||
|
||||
results = []
|
||||
|
||||
keys_list = []
|
||||
|
||||
try:
|
||||
# 连接数据库
|
||||
conn = psycopg2.connect(**db_config)
|
||||
# 创建游标
|
||||
cur = conn.cursor()
|
||||
|
||||
keys = [
|
||||
'占位'
|
||||
]
|
||||
|
||||
for key in keys:
|
||||
# SQL 查询语句
|
||||
sql = """
|
||||
select * from data_sets
|
||||
where name like '%{}%'
|
||||
""".format(key)
|
||||
|
||||
# 执行查询
|
||||
cur.execute(sql)
|
||||
|
||||
# 获取所有结果
|
||||
rows = cur.fetchall()
|
||||
|
||||
# 打印结果
|
||||
for row in rows:
|
||||
results.append({
|
||||
"name": row[1],
|
||||
"description": row[2],
|
||||
})
|
||||
|
||||
# 关闭游标和连接
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
print("数据库连接或查询出错:", e)
|
||||
|
||||
for result in results:
|
||||
print(result)
|
||||
@@ -0,0 +1,133 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
使用ollama翻译 data-sets description
|
||||
'''
|
||||
|
||||
import psycopg2
|
||||
import httpx
|
||||
import time
|
||||
import sys
|
||||
|
||||
def translate_with_ollama(text, model="qwen2.5:7b"):
|
||||
if not text or not text.strip():
|
||||
return None
|
||||
|
||||
url = "http://localhost:11434/api/generate"
|
||||
prompt = f"""你是一个金融数据专家，负责翻译期权交易领域的专业字段名。
|
||||
规则：
|
||||
1. 在此上下文中，“call” 指“看涨期权”，切勿翻译为“呼叫”。
|
||||
2. "put" 指"看跌期权"。
|
||||
3. “breakeven” 统一翻译为“盈亏平衡点”。
|
||||
4. 整体翻译需简洁，符合数据库字段名的命名习惯。
|
||||
5. 输出仅返回翻译后的中文，不要任何解释。
|
||||
|
||||
请翻译：{text}"""
|
||||
|
||||
payload = {"model": model, "prompt": prompt, "stream": False}
|
||||
|
||||
try:
|
||||
response = httpx.post(url, json=payload, timeout=180.0)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
return result['response'].strip()
|
||||
except httpx.TimeoutException:
|
||||
print(f"\n[超时] {text[:60]}...")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"\n[请求失败] {text[:30]}... - {e}")
|
||||
return None
|
||||
|
||||
db_config = {
|
||||
'host': '192.168.31.201',
|
||||
'port': 5432,
|
||||
'database': 'alpha',
|
||||
'user': 'jack',
|
||||
'password': 'aaaAAA111'
|
||||
}
|
||||
|
||||
start_from_id = 0
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(**db_config)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("""
|
||||
SELECT id, description
|
||||
FROM operator
|
||||
WHERE description IS NOT NULL
|
||||
AND description_cn IS NULL
|
||||
AND id >= %s
|
||||
ORDER BY id ASC
|
||||
""", (start_from_id,))
|
||||
|
||||
rows = cur.fetchall()
|
||||
total = len(rows)
|
||||
|
||||
if total == 0:
|
||||
print("没有需要翻译的记录！")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"找到 {total} 条待翻译记录 (从ID {start_from_id} 开始)")
|
||||
print("=" * 60)
|
||||
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
last_success_id = start_from_id
|
||||
|
||||
for idx, (row_id, description) in enumerate(rows, 1):
|
||||
percent = (idx / total) * 100
|
||||
sys.stdout.write(f"\r处理进度: {idx}/{total} ({percent:.1f}%) | 成功: {success_count} | 失败: {fail_count}")
|
||||
sys.stdout.flush()
|
||||
|
||||
max_retries = 2
|
||||
translate_text = None
|
||||
|
||||
for attempt in range(max_retries):
|
||||
translate_text = translate_with_ollama(description)
|
||||
if translate_text:
|
||||
break
|
||||
elif attempt < max_retries - 1:
|
||||
time.sleep(3)
|
||||
|
||||
if translate_text:
|
||||
print(translate_text)
|
||||
try:
|
||||
cur.execute(
|
||||
"UPDATE operator SET description_cn = %s WHERE id = %s",
|
||||
(translate_text, row_id)
|
||||
)
|
||||
success_count += 1
|
||||
last_success_id = row_id
|
||||
except Exception as e:
|
||||
print(f"\n[更新失败] ID:{row_id} - {e}")
|
||||
fail_count += 1
|
||||
else:
|
||||
fail_count += 1
|
||||
|
||||
if idx % 10 == 0:
|
||||
conn.commit()
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
conn.commit()
|
||||
|
||||
print(f"\n" + "=" * 60)
|
||||
print("翻译任务完成！")
|
||||
print(f"✓ 成功翻译: {success_count} 条")
|
||||
print(f"✗ 翻译失败: {fail_count} 条")
|
||||
print(f"最后成功处理的ID: {last_success_id}")
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
except psycopg2.Error as e:
|
||||
print(f"\n[数据库错误] {e}")
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n\n用户中断程序")
|
||||
print(f"已处理到ID: {last_success_id}")
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
print(f"\n[程序错误] {e}")
|
||||
finally:
|
||||
if 'conn' in locals() and conn:
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user