first commit
This commit is contained in:
jack
@@ -0,0 +1,87 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
设置每天 00:00:00 新建一个日志记录
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
|
||||
|
||||
import time
|
||||
from datetime import datetime
|
||||
import pymongo
|
||||
from base.base_load_config import load_config, get_base_path
|
||||
|
||||
config_json = load_config()
|
||||
base_project = get_base_path()
|
||||
|
||||
PROJECT_NAME = config_json.get('PROJECT_NAME')
|
||||
DB_USER = config_json.get('DB_USER')
|
||||
DB_PASSWORD = config_json.get('DB_PASSWORD')
|
||||
DB_IP = config_json.get('DB_IP')
|
||||
DB_PORT = config_json.get('DB_PORT')
|
||||
MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
|
||||
MAIL_HOST = config_json.get('MAIL_HOST')
|
||||
MAIL_USER = config_json.get('MAIL_USER')
|
||||
MAIL_PASS = config_json.get('MAIL_PASS')
|
||||
MAIL_SENDER = config_json.get('MAIL_SENDER')
|
||||
MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
|
||||
|
||||
now_day = time.strftime('%Y-%m-%d', time.localtime())
|
||||
rss_base_url = 'http://home.erhe.link:20002/xmlfile/'
|
||||
|
||||
|
||||
class LogsHandle(object):
|
||||
def __init__(self):
|
||||
self.now_day = time.strftime('%Y-%m-%d', time.localtime())
|
||||
db = 'logs'
|
||||
collection = 'logs_' + self.now_day
|
||||
self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
|
||||
|
||||
def logs_generate(self):
|
||||
data_to_insert = {
|
||||
"title": "logs",
|
||||
"context": 'generate message logs',
|
||||
"state": "create",
|
||||
"create_time": int(time.time()),
|
||||
"create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
}
|
||||
|
||||
self.mongo.collection.insert_one(data_to_insert)
|
||||
|
||||
|
||||
class MongoHandle(object):
|
||||
def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0):
|
||||
self.client = pymongo.MongoClient(MONGO_LINK)
|
||||
self.db = db
|
||||
self.collection = collection
|
||||
|
||||
if del_db and db:
|
||||
# 检查数据库是否存在
|
||||
if db in self.client.list_database_names():
|
||||
# 删除数据库
|
||||
self.client.drop_database(db)
|
||||
self.db = self.client[db]
|
||||
|
||||
if del_collection and self.collection:
|
||||
# 检查集合是否存在
|
||||
if self.collection in self.db.list_collection_names():
|
||||
# 删除集合
|
||||
self.db.drop_collection(collection)
|
||||
self.collection = self.db[collection]
|
||||
|
||||
if auto_remove:
|
||||
self.auto_remove_data(auto_remove)
|
||||
|
||||
def write_data(self, data):
|
||||
self.collection.insert_one(data)
|
||||
|
||||
def auto_remove_data(self, day):
|
||||
for data in self.collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}):
|
||||
self.collection.delete_one({'_id': data['_id']})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("新建当天日志记录...")
|
||||
LogsHandle().logs_generate()
|
||||
print("当天日志记录已创建...")
|
||||
@@ -0,0 +1,169 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
设置每天 23:59 执行, 读取当天数据库中, 所有日志, 发送到指定邮箱
|
||||
'''
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
import httpx
|
||||
|
||||
sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
|
||||
|
||||
import pymongo
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
from email.header import Header
|
||||
from base.base_load_config import load_config, get_base_path
|
||||
|
||||
config_json = load_config()
|
||||
base_project = get_base_path()
|
||||
|
||||
PROJECT_NAME = config_json.get('PROJECT_NAME')
|
||||
DB_USER = config_json.get('DB_USER')
|
||||
DB_PASSWORD = config_json.get('DB_PASSWORD')
|
||||
DB_IP = config_json.get('DB_IP')
|
||||
DB_PORT = config_json.get('DB_PORT')
|
||||
MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
|
||||
MAIL_HOST = config_json.get('MAIL_HOST')
|
||||
MAIL_USER = config_json.get('MAIL_USER')
|
||||
MAIL_PASS = config_json.get('MAIL_PASS')
|
||||
MAIL_SENDER = config_json.get('MAIL_SENDER')
|
||||
MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
|
||||
|
||||
now_day = time.strftime('%Y-%m-%d', time.localtime())
|
||||
|
||||
|
||||
class LogsHandle(object):
|
||||
def __init__(self):
|
||||
self.now_day = time.strftime('%Y-%m-%d', time.localtime())
|
||||
db = 'logs'
|
||||
collection = 'logs_' + self.now_day
|
||||
self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
|
||||
|
||||
def logs_send(self):
|
||||
subject = 'auto message logs'
|
||||
title = 'auto message - daily logs: {}'.format(self.now_day)
|
||||
text = ''
|
||||
|
||||
# TODO
|
||||
# 从 mongodb 读取日志, 拼接 text, 发送邮件
|
||||
# 查询所有文档
|
||||
query = {'state': 'error'}
|
||||
cursor = self.mongo.collection.find(query)
|
||||
# 遍历结果集
|
||||
for record in cursor:
|
||||
text += "logs_source: {}, logs_detail: {}, state: {} logs_create_time: {}\n\n".format(
|
||||
record.setdefault('title'),
|
||||
record.setdefault('content'),
|
||||
record.setdefault('state'),
|
||||
record.setdefault('create_datetime'),
|
||||
)
|
||||
|
||||
if text:
|
||||
S = SendEmail(subject=subject, title=title, text=text)
|
||||
S.send()
|
||||
|
||||
G = GotifyNotifier(title=title, message=text, token='base')
|
||||
G.send_message()
|
||||
else:
|
||||
print("No error logs found for today.")
|
||||
|
||||
|
||||
class MongoHandle(object):
|
||||
def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0):
|
||||
self.client = pymongo.MongoClient(MONGO_LINK)
|
||||
self.db = db
|
||||
self.collection = collection
|
||||
|
||||
if del_db and db:
|
||||
# 检查数据库是否存在
|
||||
if db in self.client.list_database_names():
|
||||
# 删除数据库
|
||||
self.client.drop_database(db)
|
||||
self.db = self.client[db]
|
||||
|
||||
if del_collection and self.collection:
|
||||
# 检查集合是否存在
|
||||
if self.collection in self.db.list_collection_names():
|
||||
# 删除集合
|
||||
self.db.drop_collection(collection)
|
||||
self.collection = self.db[collection]
|
||||
|
||||
if auto_remove:
|
||||
self.auto_remove_data(auto_remove)
|
||||
|
||||
def write_data(self, data):
|
||||
self.collection.insert_one(data)
|
||||
|
||||
def auto_remove_data(self, day):
|
||||
for data in self.collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}):
|
||||
self.collection.delete_one({'_id': data['_id']})
|
||||
|
||||
|
||||
class SendEmail(object):
|
||||
def __init__(self, subject='auto subject', title='auto title', text='auto text') -> None:
|
||||
# 第三方 SMTP 服务
|
||||
self.mail_host = MAIL_HOST # 设置服务器
|
||||
self.mail_user = MAIL_USER # 用户名
|
||||
self.mail_pass = MAIL_PASS # 口令
|
||||
|
||||
self.sender = MAIL_SENDER
|
||||
self.receivers = [MAIL_RECEIVERS]
|
||||
|
||||
self.subject = subject
|
||||
self.title = title
|
||||
self.text = text
|
||||
|
||||
def send(self):
|
||||
message = MIMEText(self.text, 'plain', 'utf-8')
|
||||
message['From'] = Header(self.title, 'utf-8')
|
||||
message['To'] = Header("auto", 'utf-8')
|
||||
message['Subject'] = Header(self.subject, 'utf-8')
|
||||
|
||||
try:
|
||||
smtpObj = smtplib.SMTP_SSL(self.mail_host)
|
||||
smtpObj.login(self.mail_user, self.mail_pass)
|
||||
smtpObj.sendmail(self.sender, self.receivers, message.as_string())
|
||||
print("邮件发送成功")
|
||||
except smtplib.SMTPException as e:
|
||||
print("Error: 无法发送邮件", e)
|
||||
|
||||
|
||||
class GotifyNotifier:
|
||||
def __init__(self, title, message, token='A8EVb0Cmxnb2vfk'):
|
||||
self.gotify_url = 'https://gotify.erhe.top'
|
||||
self.app_token = token
|
||||
self.title = title
|
||||
self.message = message
|
||||
|
||||
def send_message(self):
|
||||
# 构建POST请求的headers
|
||||
headers = {
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
# 构建POST请求的body
|
||||
body = {
|
||||
'title': self.title,
|
||||
'message': self.message
|
||||
}
|
||||
|
||||
# 发送POST请求
|
||||
with httpx.Client() as client:
|
||||
response = client.post(
|
||||
url=f"{self.gotify_url}/message?token={self.app_token}",
|
||||
headers=headers,
|
||||
json=body
|
||||
)
|
||||
|
||||
# 检查响应状态码
|
||||
if response.status_code == 200:
|
||||
print('Gotify Message sent successfully!')
|
||||
else:
|
||||
print('Failed to send message:', response.text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("开始执行日志处理")
|
||||
LogsHandle().logs_send()
|
||||
print("处理日志程序执行完毕")
|
||||
@@ -0,0 +1,34 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
用于读取config.json
|
||||
无需定时
|
||||
'''
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
|
||||
|
||||
def load_config():
|
||||
try:
|
||||
sys.path.append(os.path.join(os.getcwd().split('auto')[0], 'auto'))
|
||||
base_project = os.path.join(os.getcwd().split('auto')[0], 'auto')
|
||||
|
||||
config_path = os.path.join(base_project, 'config.json')
|
||||
config_json = {}
|
||||
with open(config_path, 'r') as f:
|
||||
config_json = json.load(f)
|
||||
|
||||
if not config_json:
|
||||
print('No config file found')
|
||||
exit(0)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
exit(0)
|
||||
|
||||
return config_json
|
||||
|
||||
|
||||
def get_base_path():
|
||||
return os.path.join(os.getcwd().split('auto')[0], 'auto')
|
||||
@@ -0,0 +1,236 @@
|
||||
'''
|
||||
每日从 mongo 数据库, 做新闻汇总,发送到邮箱
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
|
||||
|
||||
from pymongo import MongoClient
|
||||
from datetime import datetime, timedelta
|
||||
import re
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
from email.header import Header
|
||||
from base.base_load_config import load_config, get_base_path
|
||||
|
||||
config_json = load_config()
|
||||
base_project = get_base_path()
|
||||
|
||||
PROJECT_NAME = config_json.get('PROJECT_NAME')
|
||||
DB_USER = config_json.get('DB_USER')
|
||||
DB_PASSWORD = config_json.get('DB_PASSWORD')
|
||||
DB_IP = config_json.get('DB_IP')
|
||||
DB_PORT = config_json.get('DB_PORT')
|
||||
MAIL_HOST = config_json.get('MAIL_HOST')
|
||||
MAIL_USER = config_json.get('MAIL_USER')
|
||||
MAIL_PASS = config_json.get('MAIL_PASS')
|
||||
MAIL_SENDER = config_json.get('MAIL_SENDER')
|
||||
MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
|
||||
DB_NAME = config_json.get('DB_NAME') # 确保配置文件中有这个键
|
||||
MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'.format(**config_json)
|
||||
now_day = datetime.now().strftime('%Y-%m-%d') # 获取今天的日期
|
||||
filter_days = config_json.get('FILTER_DAYS')
|
||||
filter_keys = config_json.get('FILTER_KEYS')
|
||||
filter_switch = True
|
||||
|
||||
|
||||
class NewsDataCollation(object):
|
||||
def __init__(self):
|
||||
# 第三方 SMTP 服务
|
||||
self.mail_host = MAIL_HOST # 设置服务器
|
||||
self.mail_user = MAIL_USER # 用户名
|
||||
self.mail_pass = MAIL_PASS # 口令
|
||||
|
||||
self.sender = MAIL_SENDER
|
||||
self.receivers = [MAIL_RECEIVERS]
|
||||
|
||||
self.processed_data = []
|
||||
|
||||
def load_data(self):
|
||||
processed_data = []
|
||||
# 读取数据
|
||||
print('程序正在读取数据')
|
||||
client = MongoClient(MONGO_LINK)
|
||||
db = client['NEWS']
|
||||
|
||||
# 根据 self.days 获取日期范围
|
||||
start_date = (datetime.now() - timedelta(days=filter_days - 1)).strftime('%Y-%m-%d')
|
||||
end_date = datetime.now().strftime('%Y-%m-%d')
|
||||
|
||||
# 构造查询条件，匹配日期范围内的日期
|
||||
query = {
|
||||
"create_datetime": {
|
||||
"$regex": f"^{start_date}|{end_date}",
|
||||
"$options": "i" # 使用不区分大小写的匹配
|
||||
}
|
||||
}
|
||||
|
||||
# 遍历数据库中的所有集合
|
||||
for collection_name in db.list_collection_names():
|
||||
print(collection_name)
|
||||
collection = db[collection_name]
|
||||
cursor = collection.find(query)
|
||||
for document in cursor:
|
||||
if not document.get('title'):
|
||||
continue
|
||||
|
||||
# 检查 'repush_times' 字段是否存在，如果不存在则默认为 5
|
||||
repush_times = document.get('repush_times', 5)
|
||||
|
||||
# 减少 repush_times 的值
|
||||
new_repush_times = repush_times - 1
|
||||
|
||||
# 更新数据库中的 repush_times 字段
|
||||
collection.update_one(
|
||||
{"_id": document['_id']}, # 假设文档中有 _id 字段作为唯一标识
|
||||
{"$set": {"repush_times": new_repush_times}}
|
||||
)
|
||||
|
||||
data = self.process_data(document)
|
||||
if data:
|
||||
processed_data.append(data)
|
||||
|
||||
# 关闭MongoDB连接
|
||||
client.close()
|
||||
return processed_data
|
||||
|
||||
def process_data(self, document):
|
||||
# 处理数据
|
||||
data = {
|
||||
"title": document.get('title') or '',
|
||||
"context": document.get('context') or '',
|
||||
"source_url": document.get('source_url') or '',
|
||||
'link': document.get('link') or '',
|
||||
"article_type": document.get('article_type') or '',
|
||||
"article_source": document.get('article_source') or '',
|
||||
"img_url": document.get('img_url') or '',
|
||||
'keyword': document.get('keyword') or '',
|
||||
"posted_date": document.get('posted_date') or '',
|
||||
"create_time": document.get('create_time') or '',
|
||||
"create_datetime": document.get('create_datetime') or '',
|
||||
"repush_times": document.get('repush_times', 5) - 1
|
||||
}
|
||||
|
||||
data['title'] = self.clean_string(data['title'], 'title')
|
||||
data['context'] = self.clean_string(data['context'], 'context')
|
||||
|
||||
return data
|
||||
|
||||
def clean_string(self, input_string, text_type):
|
||||
# 清除 title 和 context 中的换行符和制表符
|
||||
if not isinstance(input_string, str):
|
||||
return ''
|
||||
|
||||
# 清除所有空白字符（包括空格、制表符、换行符等）
|
||||
cleaned_string = re.sub(r'\s+', '', input_string)
|
||||
|
||||
if len(cleaned_string) > 100:
|
||||
cleaned_string = cleaned_string[:100] + '...'
|
||||
|
||||
if text_type == 'context':
|
||||
pass
|
||||
|
||||
return cleaned_string
|
||||
|
||||
def send_email(self, processed_data):
|
||||
# 发送邮件
|
||||
print('准备发送邮件')
|
||||
subject = '新闻汇总sub'
|
||||
title = '新闻汇总title'
|
||||
text = '********************************************************\n'
|
||||
for data in processed_data:
|
||||
text += '标题: {}\n'.format(data['title'])
|
||||
text += '正文: {}\n'.format(data['context'])
|
||||
text += '文章地址: {}\n'.format(data['link'])
|
||||
text += '类型: {}\n'.format(data['article_type'])
|
||||
text += '板块: {}\n'.format(data['article_source'])
|
||||
text += '文章时间: {}\n'.format(data['posted_date'])
|
||||
text += '获取时间: {}\n'.format(data['create_datetime'])
|
||||
text += '********************************************************\n\n'
|
||||
|
||||
message = MIMEText(text, 'plain', 'utf-8')
|
||||
message['From'] = Header(title, 'utf-8')
|
||||
message['To'] = Header("auto", 'utf-8')
|
||||
message['Subject'] = Header(subject, 'utf-8')
|
||||
|
||||
try:
|
||||
smtpObj = smtplib.SMTP_SSL(self.mail_host)
|
||||
smtpObj.login(self.mail_user, self.mail_pass)
|
||||
smtpObj.sendmail(self.sender, self.receivers, message.as_string())
|
||||
print("邮件发送成功")
|
||||
except smtplib.SMTPException as e:
|
||||
print("Error: 无法发送邮件", e)
|
||||
|
||||
def send_email_with_keyword(self, series, keys, processed_data):
|
||||
process_send_data = {}
|
||||
keys = keys.split('|')
|
||||
have_data_keys = []
|
||||
for key in keys:
|
||||
# print(f'通过关键字: {key} 过滤') # 用来调试 key 是否正确
|
||||
for data in processed_data:
|
||||
if key in data['title'] or key in data['context']:
|
||||
# 如果数据里面无 keyword, 用当前 key 替换一下
|
||||
if not data.get('keyword'):
|
||||
data['keyword'] = key
|
||||
|
||||
if series not in process_send_data:
|
||||
process_send_data[series] = [data]
|
||||
else:
|
||||
process_send_data[series].append(data)
|
||||
|
||||
# 储存一下有数据的 key, 输出用
|
||||
have_data_keys.append(key)
|
||||
|
||||
if process_send_data:
|
||||
print('{}系列, 以下关键字有数据\n{}'.format(series, list(set(have_data_keys))))
|
||||
# 发送邮件
|
||||
print('程序正在准备发送邮件的数据')
|
||||
for key in process_send_data:
|
||||
subject = '新闻汇总sub - {}'.format(series)
|
||||
title = '新闻汇总title - {}'.format(series)
|
||||
text = '********************************************************\n'
|
||||
for data in process_send_data[key]:
|
||||
text += '标题: {}\n'.format(data['title'])
|
||||
text += '正文: {}\n'.format(data['context'])
|
||||
text += '文章地址: {}\n'.format(data['link'])
|
||||
text += '类型: {}\n'.format(data['article_type'])
|
||||
text += '板块: {}\n'.format(data['article_source'])
|
||||
text += '关键词: {}\n'.format(key)
|
||||
text += '文章时间: {}\n'.format(data['posted_date'])
|
||||
text += '获取时间: {}\n'.format(data['create_datetime'])
|
||||
text += '********************************************************\n\n'
|
||||
|
||||
message = MIMEText(text, 'plain', 'utf-8')
|
||||
message['From'] = Header(title, 'utf-8')
|
||||
message['To'] = Header("auto", 'utf-8')
|
||||
message['Subject'] = Header(subject, 'utf-8')
|
||||
|
||||
try:
|
||||
smtpObj = smtplib.SMTP_SSL(self.mail_host)
|
||||
smtpObj.login(self.mail_user, self.mail_pass)
|
||||
smtpObj.sendmail(self.sender, self.receivers, message.as_string())
|
||||
print("关键字: {} 的邮件发送成功".format(series))
|
||||
except smtplib.SMTPException as e:
|
||||
print("Error: 无法发送邮件", e)
|
||||
|
||||
def main(self):
|
||||
# 加载指定天数的所有数据
|
||||
processed_data = self.load_data()
|
||||
|
||||
# 如果无数据, 则退出
|
||||
if not processed_data:
|
||||
print("没有找到任何数据")
|
||||
exit(0)
|
||||
|
||||
# 发送一次所有数据的邮件
|
||||
# self.send_email(processed_data)
|
||||
|
||||
# # 这里是通过关键词过滤然后再发送邮件
|
||||
if filter_switch and filter_keys:
|
||||
for series, keys in filter_keys.items():
|
||||
self.send_email_with_keyword(series, keys, processed_data)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
NewsDataCollation().main()
|
||||
@@ -0,0 +1,193 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
自动清除大于指定天数的数据
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime
|
||||
import pymongo
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
from email.header import Header
|
||||
|
||||
sys.path.append(os.path.join(os.path.abspath(__file__).split('auto')[0] + 'auto'))
|
||||
base_project = os.path.join(os.getcwd().split('auto')[0], 'auto')
|
||||
import json
|
||||
|
||||
config_path = os.path.join(base_project, 'config.json')
|
||||
with open(config_path, 'r') as f:
|
||||
config_json = json.load(f)
|
||||
|
||||
if not config_json:
|
||||
print('No config file found')
|
||||
exit(0)
|
||||
|
||||
PROJECT_NAME = config_json.get('PROJECT_NAME')
|
||||
DB_USER = config_json.get('DB_USER')
|
||||
DB_PASSWORD = config_json.get('DB_PASSWORD')
|
||||
DB_IP = config_json.get('DB_IP')
|
||||
DB_PORT = config_json.get('DB_PORT')
|
||||
MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/'
|
||||
MAIL_HOST = config_json.get('MAIL_HOST')
|
||||
MAIL_USER = config_json.get('MAIL_USER')
|
||||
MAIL_PASS = config_json.get('MAIL_PASS')
|
||||
MAIL_SENDER = config_json.get('MAIL_SENDER')
|
||||
MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS')
|
||||
|
||||
now_day = time.strftime('%Y-%m-%d', time.localtime())
|
||||
rss_base_url = 'http://home.erhe.link:20002/xmlfile/'
|
||||
base_project = os.path.join(os.getcwd().split(PROJECT_NAME)[0], PROJECT_NAME)
|
||||
|
||||
|
||||
class MongoHandle(object):
|
||||
def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0):
|
||||
self.client = pymongo.MongoClient(MONGO_LINK)
|
||||
self.db = db
|
||||
self.collection = collection
|
||||
|
||||
if del_db and db:
|
||||
# 检查数据库是否存在
|
||||
if db in self.client.list_database_names():
|
||||
# 删除数据库
|
||||
self.client.drop_database(db)
|
||||
self.db = self.client[db]
|
||||
|
||||
if del_collection and self.collection:
|
||||
# 检查集合是否存在
|
||||
if self.collection in self.db.list_collection_names():
|
||||
# 删除集合
|
||||
self.db.drop_collection(collection)
|
||||
self.collection = self.db[collection]
|
||||
|
||||
if auto_remove:
|
||||
self.auto_remove_data(auto_remove)
|
||||
|
||||
def write_data(self, data):
|
||||
self.collection.insert_one(data)
|
||||
|
||||
def auto_remove_data(self, day):
|
||||
for data in self.collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}):
|
||||
self.collection.delete_one({'_id': data['_id']})
|
||||
|
||||
|
||||
class SendEmail(object):
|
||||
def __init__(self, subject='auto subject', title='auto title', text='auto text') -> None:
|
||||
# 第三方 SMTP 服务
|
||||
self.mail_host = MAIL_HOST # 设置服务器
|
||||
self.mail_user = MAIL_USER # 用户名
|
||||
self.mail_pass = MAIL_PASS # 口令
|
||||
|
||||
self.sender = MAIL_SENDER
|
||||
self.receivers = [MAIL_RECEIVERS]
|
||||
|
||||
self.subject = subject
|
||||
self.title = title
|
||||
self.text = text
|
||||
|
||||
def send(self):
|
||||
message = MIMEText(self.text, 'plain', 'utf-8')
|
||||
message['From'] = Header(self.title, 'utf-8')
|
||||
message['To'] = Header("auto", 'utf-8')
|
||||
message['Subject'] = Header(self.subject, 'utf-8')
|
||||
|
||||
try:
|
||||
smtpObj = smtplib.SMTP_SSL(self.mail_host)
|
||||
smtpObj.login(self.mail_user, self.mail_pass)
|
||||
smtpObj.sendmail(self.sender, self.receivers, message.as_string())
|
||||
print("邮件发送成功")
|
||||
except smtplib.SMTPException as e:
|
||||
print("Error: 无法发送邮件", e)
|
||||
|
||||
|
||||
class LogsHandle(object):
|
||||
def __init__(self):
|
||||
self.now_day = time.strftime('%Y-%m-%d', time.localtime())
|
||||
db = 'logs'
|
||||
collection = 'logs_' + self.now_day
|
||||
self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0)
|
||||
|
||||
def logs_write(self, title_source=None, content=None, state=None, send_now=False):
|
||||
data_to_insert = {
|
||||
"title": title_source,
|
||||
"context": content,
|
||||
"state": state,
|
||||
"create_time": int(time.time()),
|
||||
"create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
}
|
||||
|
||||
self.mongo.collection.insert_one(data_to_insert)
|
||||
|
||||
if send_now:
|
||||
subject = 'auto collection'
|
||||
title = 'auto collection - running logs: {}'.format(self.now_day)
|
||||
text = 'logs_source: {}, logs_detail: {}, state: {} logs_create_time: {}'.format(
|
||||
data_to_insert.setdefault('title'),
|
||||
data_to_insert.setdefault('content'),
|
||||
data_to_insert.setdefault('state'),
|
||||
data_to_insert.setdefault('create_datetime'),
|
||||
)
|
||||
|
||||
Send = SendEmail(subject=subject, title=title, text=text)
|
||||
Send.send()
|
||||
|
||||
|
||||
class AutoRemoveData(object):
|
||||
def __init__(self):
|
||||
self.databases = [
|
||||
'spider_news',
|
||||
'apprcn',
|
||||
'HelloGithub'
|
||||
]
|
||||
|
||||
self.day = 60
|
||||
|
||||
self.client = pymongo.MongoClient(MONGO_LINK)
|
||||
|
||||
self.logs = LogsHandle()
|
||||
|
||||
self.all_delete_count = 0
|
||||
|
||||
def auto_remove_data(self, db_name, day):
|
||||
print(f'准备删除时间大于: {self.day} 数据')
|
||||
if db_name not in self.client.list_database_names():
|
||||
return
|
||||
|
||||
deleted_count = 0
|
||||
|
||||
db = self.client[db_name]
|
||||
|
||||
for collection_name in db.list_collection_names():
|
||||
collection = db[collection_name]
|
||||
for data in collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}):
|
||||
collection.delete_one({'_id': data['_id']})
|
||||
deleted_count += 1
|
||||
|
||||
self.all_delete_count += deleted_count
|
||||
|
||||
msg = f"删除 {db_name} 库 {self.day} 天以上数据 {deleted_count} 条"
|
||||
if deleted_count:
|
||||
print(msg)
|
||||
self.logs.logs_write(f'自动删除 {self.day} 天以上数据', msg, 'delete', False)
|
||||
|
||||
def main(self):
|
||||
self.logs.logs_write(f'自动删除 {self.day} 天以上数据', f'开始自动删除 {self.day} 天以上数据', 'start', False)
|
||||
threads = []
|
||||
|
||||
for db_name in self.databases:
|
||||
thread = threading.Thread(target=self.auto_remove_data, args=(db_name, self.day))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
print(f'删除时间大于: {self.day} 数据, 已完成')
|
||||
print(f'本次运行共删除: {self.all_delete_count} 条数据')
|
||||
self.logs.logs_write(f'自动删除 {self.day} 天以上数据', f'自动删除 {self.day} 天数以上数据完成', 'done', False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
A = AutoRemoveData()
|
||||
A.main()
|
||||
@@ -0,0 +1,140 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
from playwright.sync_api import sync_playwright
|
||||
import psycopg2
|
||||
|
||||
|
||||
def run(playwright):
|
||||
browser = playwright.webkit.launch(headless=True)
|
||||
|
||||
context = browser.new_context()
|
||||
|
||||
page = context.new_page()
|
||||
|
||||
page.goto('https://rss.erhe.top/')
|
||||
|
||||
page.fill('#username', 'toor')
|
||||
|
||||
page.fill('#passwordPlain', '!QAZ2wsx+0913')
|
||||
|
||||
page.click('#loginButton')
|
||||
|
||||
time.sleep(4)
|
||||
|
||||
try:
|
||||
page.goto('https://rss.erhe.top/i/?c=feed&a=actualize')
|
||||
title = page.title()
|
||||
print(f'{title} 更新成功')
|
||||
except Exception as e:
|
||||
print(f'更新失败 {str(e)}')
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
context.close()
|
||||
browser.close()
|
||||
print('rss订阅源更新完成')
|
||||
|
||||
|
||||
def check_data_count():
|
||||
# 数据库连接参数
|
||||
db_params = {
|
||||
'dbname': 'freshrss',
|
||||
'user': 'freshrss',
|
||||
'password': 'freshrss',
|
||||
'host': 'erhe.top',
|
||||
'port': '20788'
|
||||
}
|
||||
|
||||
# SQL 查询语句
|
||||
query = 'SELECT COUNT(*) FROM freshrss_toor_entry;'
|
||||
|
||||
try:
|
||||
# 建立数据库连接
|
||||
conn = psycopg2.connect(**db_params)
|
||||
|
||||
# 创建 cursor 对象
|
||||
cur = conn.cursor()
|
||||
|
||||
# 执行 SQL 查询
|
||||
cur.execute(query)
|
||||
|
||||
# 获取查询结果
|
||||
records = cur.fetchall()
|
||||
|
||||
count_num = records[0][0]
|
||||
print(f'\n\nfreshrss_toor_entry 表共有数据数量为: {count_num}')
|
||||
|
||||
except psycopg2.Error as e:
|
||||
print(f"Database error: {e}")
|
||||
conn.rollback()
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
finally:
|
||||
# 关闭 cursor 和连接
|
||||
if 'cur' in locals():
|
||||
cur.close()
|
||||
if 'conn' in locals():
|
||||
conn.close()
|
||||
|
||||
|
||||
def read_ids():
|
||||
ids = []
|
||||
# 数据库连接参数
|
||||
db_params = {
|
||||
'dbname': 'freshrss',
|
||||
'user': 'freshrss',
|
||||
'password': 'freshrss',
|
||||
'host': 'erhe.top',
|
||||
'port': '20788'
|
||||
}
|
||||
|
||||
# SQL 查询语句
|
||||
query = 'SELECT id, url FROM freshrss_toor_feed order by id ASC;'
|
||||
|
||||
try:
|
||||
# 建立数据库连接
|
||||
conn = psycopg2.connect(**db_params)
|
||||
|
||||
# 创建 cursor 对象
|
||||
cur = conn.cursor()
|
||||
|
||||
# 执行 SQL 查询
|
||||
cur.execute(query)
|
||||
|
||||
# 获取查询结果
|
||||
records = cur.fetchall()
|
||||
|
||||
for record in records:
|
||||
id, url = record
|
||||
print(id, url)
|
||||
ids.append(id)
|
||||
|
||||
except psycopg2.Error as e:
|
||||
print(f"Database error: {e}")
|
||||
conn.rollback()
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
finally:
|
||||
# 关闭 cursor 和连接
|
||||
if 'cur' in locals():
|
||||
cur.close()
|
||||
if 'conn' in locals():
|
||||
conn.close()
|
||||
|
||||
if ids:
|
||||
return ids
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ids = read_ids()
|
||||
|
||||
if not ids:
|
||||
print('未获取到订阅源id, 程序退出')
|
||||
exit(0)
|
||||
# with sync_playwright() as playwright:
|
||||
# run(playwright)
|
||||
#
|
||||
# check_data_count()
|
||||
Reference in New Issue
Block a user