# -*- coding: utf-8 -*- """ 自动清除大于指定天数的数据 """ import threading import time import sys import os from datetime import datetime import pymongo sys.path.append(os.path.join(os.path.abspath(__file__).split('AutoInfo')[0] + 'AutoInfo')) base_project = os.path.join(os.getcwd().split('AutoInfo')[0], 'AutoInfo') from utils.utils import LoadConfig, GotifyNotifier config_json = LoadConfig().load_config() base_project = LoadConfig().get_base_path() PROJECT_NAME = config_json.get('PROJECT_NAME') DB_USER = config_json.get('DB_USER') DB_PASSWORD = config_json.get('DB_PASSWORD') DB_IP = config_json.get('DB_IP') DB_PORT = config_json.get('DB_PORT') MONGO_LINK = f'mongodb://{DB_USER}:{DB_PASSWORD}@{DB_IP}:{DB_PORT}/' MAIL_HOST = config_json.get('MAIL_HOST') MAIL_USER = config_json.get('MAIL_USER') MAIL_PASS = config_json.get('MAIL_PASS') MAIL_SENDER = config_json.get('MAIL_SENDER') MAIL_RECEIVERS = config_json.get('MAIL_RECEIVERS') now_day = time.strftime('%Y-%m-%d', time.localtime()) class MongoHandle(object): def __init__(self, db, collection, del_db=False, del_collection=False, auto_remove=0): self.client = pymongo.MongoClient(MONGO_LINK) self.db = db self.collection = collection if del_db and db: # 检查数据库是否存在 if db in self.client.list_database_names(): # 删除数据库 self.client.drop_database(db) self.db = self.client[db] if del_collection and self.collection: # 检查集合是否存在 if self.collection in self.db.list_collection_names(): # 删除集合 self.db.drop_collection(collection) self.collection = self.db[collection] if auto_remove: self.auto_remove_data(auto_remove) def write_data(self, data): self.collection.insert_one(data) def auto_remove_data(self, day): for data in self.collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}): self.collection.delete_one({'_id': data['_id']}) class SendEmail(object): def __init__(self, subject='AutoInfo subject', title='AutoInfo title', text='AutoInfo text') -> None: # 第三方 SMTP 服务 self.mail_host = MAIL_HOST # 设置服务器 self.mail_user = MAIL_USER # 用户名 self.mail_pass = MAIL_PASS # 口令 self.sender = MAIL_SENDER self.receivers = [MAIL_RECEIVERS] self.subject = subject self.title = title self.text = text def send(self): if self.title: G = GotifyNotifier(title=self.title, message=self.subject) G.send_message() else: print("No error logs found for today.") class LogsHandle(object): def __init__(self): self.now_day = time.strftime('%Y-%m-%d', time.localtime()) db = 'logs' collection = 'logs_' + self.now_day self.mongo = MongoHandle(db=db, collection=collection, del_db=False, del_collection=False, auto_remove=0) def logs_write(self, title_source=None, content=None, state=None, send_now=False): data_to_insert = { "title": title_source, "context": content, "state": state, "create_time": int(time.time()), "create_datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S") } self.mongo.collection.insert_one(data_to_insert) if send_now: subject = 'auto collection' title = 'auto collection - running logs: {}'.format(self.now_day) text = 'logs_source: {}, logs_detail: {}, state: {} logs_create_time: {}'.format( data_to_insert.setdefault('title'), data_to_insert.setdefault('content'), data_to_insert.setdefault('state'), data_to_insert.setdefault('create_datetime'), ) Send = SendEmail(subject=subject, title=title, text=text) Send.send() class AutoRemoveData(object): def __init__(self): self.databases = [ 'spider_news', 'apprcn', 'HelloGithub' ] self.day = 60 self.client = pymongo.MongoClient(MONGO_LINK) self.logs = LogsHandle() self.all_delete_count = 0 def auto_remove_data(self, db_name, day): print(f'准备删除时间大于: {self.day} 数据') if db_name not in self.client.list_database_names(): return deleted_count = 0 db = self.client[db_name] for collection_name in db.list_collection_names(): collection = db[collection_name] for data in collection.find({'create_time': {'$lt': int(time.time()) - day * 24 * 60 * 60}}): collection.delete_one({'_id': data['_id']}) deleted_count += 1 self.all_delete_count += deleted_count msg = f"删除 {db_name} 库 {self.day} 天以上数据 {deleted_count} 条" if deleted_count: print(msg) self.logs.logs_write(f'自动删除 {self.day} 天以上数据', msg, 'delete', False) def main(self): self.logs.logs_write(f'自动删除 {self.day} 天以上数据', f'开始自动删除 {self.day} 天以上数据', 'start', False) threads = [] for db_name in self.databases: thread = threading.Thread(target=self.auto_remove_data, args=(db_name, self.day)) threads.append(thread) thread.start() for thread in threads: thread.join() print(f'删除时间大于: {self.day} 数据, 已完成') print(f'本次运行共删除: {self.all_delete_count} 条数据') self.logs.logs_write(f'自动删除 {self.day} 天以上数据', f'自动删除 {self.day} 天数以上数据完成', 'done', False) if __name__ == "__main__": A = AutoRemoveData() A.main()