diff --git a/events/db.py b/events/db.py index af8f4fac..07fee509 100644 --- a/events/db.py +++ b/events/db.py @@ -5,11 +5,11 @@ from datetime import timedelta import hashlib -from sqlalchemy import desc, select, update, func, text, and_ +from sqlalchemy import desc, select, update, func, text, and_, delete from sqlalchemy.sql import exists from .models import FileAudit, FileUpdate, PermAudit, \ - Activity, UserActivity, FileHistory + Activity, UserActivity, FileHistory, TrashRecord logger = logging.getLogger('seafevents') @@ -235,6 +235,31 @@ def save_user_activity(session, record): session.add(user_activity) session.commit() +def save_repo_trash(session, record): + repo_trash = TrashRecord(record) + session.add(repo_trash) + session.commit() + +def restore_repo_trash(session, record): + stmt = delete(TrashRecord).where(TrashRecord.repo_id == record['repo_id'], TrashRecord.obj_name == record['obj_name'], + TrashRecord.path == record['path']) + session.execute(stmt) + session.commit() + +def clean_up_repo_trash(session, repo_id, keep_days): + if keep_days == 0: + stmt = delete(TrashRecord).where(TrashRecord.repo_id == repo_id) + session.execute(stmt) + session.commit() + else: + _timestamp = datetime.datetime.now() - timedelta(days=keep_days) + logger.warning(_timestamp) + logger.warning(TrashRecord.delete_time) + stmt = delete(TrashRecord).where(TrashRecord.repo_id == repo_id, TrashRecord.delete_time < _timestamp) + session.execute(stmt) + session.commit() + + def update_user_activity_timestamp(session, activity_id, record): activity_stmt = update(Activity).where(Activity.id == activity_id).\ values(timestamp=record["timestamp"]) diff --git a/events/handlers.py b/events/handlers.py index 35d43007..1387a25d 100644 --- a/events/handlers.py +++ b/events/handlers.py @@ -18,12 +18,13 @@ from seafobj import CommitDiffer, commit_mgr, fs_mgr from seafobj.commit_differ import DiffEntry from seafevents.events.db import save_file_audit_event, save_file_update_event, \ - save_perm_audit_event, save_user_activity, save_filehistory, update_user_activity_timestamp + save_perm_audit_event, save_user_activity, save_filehistory, update_user_activity_timestamp, \ + save_repo_trash, restore_repo_trash from seafevents.app.config import TIME_ZONE from seafevents.utils import get_opt_from_conf_or_env from .change_file_path import ChangeFilePathHandler from .change_extended_props import ChangeExtendedPropsHandler -from .models import Activity +from .models import Activity, TrashRecord from seafevents.batch_delete_files_notice.utils import get_deleted_files_count, save_deleted_files_msg from seafevents.batch_delete_files_notice.db import get_deleted_files_total_count, save_deleted_files_count @@ -102,13 +103,13 @@ def RepoUpdateEventHandler(config, session, msg): parent, time) save_file_histories(config, session, records) - records = generate_activity_records(added_files, deleted_files, + records, trash_records = generate_activity_records(added_files, deleted_files, added_dirs, deleted_dirs, modified_files, renamed_files, moved_files, renamed_dirs, moved_dirs, commit, repo_id, parent, users, time) save_user_activities(session, records) - + save_repo_trashs(session, trash_records) # save repo monitor recodes records = generate_repo_monitor_records(repo_id, commit, added_files, deleted_files, @@ -118,6 +119,7 @@ def RepoUpdateEventHandler(config, session, msg): modified_files) save_message_to_user_notification(session, records) + enable_collab_server = False if config.has_option('COLLAB_SERVER', 'enabled'): enable_collab_server = config.getboolean('COLLAB_SERVER', 'enabled') @@ -262,7 +264,6 @@ def generate_repo_monitor_records(repo_id, commit, records.append(added_files_record) if deleted_files: - deleted_files_record = copy.copy(base_record) deleted_files_record["commit_diff"] = [] @@ -559,6 +560,34 @@ def save_user_activities(session, records): for record in records: save_user_activity(session, record) + +def save_repo_trashs(session, records): + for record in records: + if record['op_type'] == 'delete': + save_repo_trash(session, record) + if record['op_type'] == 'recover': + restore_repo_trash(session, record) + + +def get_delete_records(session, repo_id, show_day, path): + if path != '/': + path = path + '/%' + else: + path = '/%' + if show_day == 0: + return + elif show_day == -1: + stmt = select(TrashRecord).where(TrashRecord.repo_id == repo_id, + TrashRecord.path.like(path)) + else: + _timestamp = datetime.datetime.now() - timedelta(days=show_day) + stmt = select(TrashRecord).where(TrashRecord.delete_time > _timestamp, + TrashRecord.repo_id == repo_id, + TrashRecord.path.like(path)) + res = session.scalars(stmt).all() + + return res + def generate_activity_records(added_files, deleted_files, added_dirs, deleted_dirs, modified_files, renamed_files, moved_files, renamed_dirs, moved_dirs, commit, repo_id, parent, related_users, time): @@ -582,13 +611,27 @@ def generate_activity_records(added_files, deleted_files, added_dirs, 'op_user': getattr(commit, 'creator_name', ''), 'repo_name': repo.repo_name } + base_trash_record = { + 'commit_id': commit.parent_id, + 'op_user': getattr(commit, 'creator_name', ''), + 'timestamp': time, + 'repo_id': repo_id, + } records = [] + trash_records = [] for de in added_files: record = copy.copy(base_record) + trash_record = copy.copy(base_trash_record) op_type = '' if commit.description.startswith('Reverted'): op_type = OP_RECOVER + trash_record['op_type'] = OP_RECOVER + trash_record['obj_id'] = de.obj_id + trash_record['obj_name'] = os.path.basename(de.path) + trash_record['path'] = '/' if os.path.dirname(de.path) == '/' else os.path.dirname(de.path) + '/' + trash_record['size'] = de.size + trash_records.append(trash_record) else: op_type = OP_CREATE record['op_type'] = op_type @@ -607,11 +650,26 @@ def generate_activity_records(added_files, deleted_files, added_dirs, record['path'] = de.path records.append(record) + trash_record = copy.copy(base_trash_record) + trash_record['obj_type'] = 'file' + trash_record['op_type'] = OP_DELETE + trash_record['obj_id'] = de.obj_id + trash_record['obj_name'] = os.path.basename(de.path) + trash_record['path'] = '/' if os.path.dirname(de.path) == '/' else os.path.dirname(de.path) + '/' + trash_record['size'] = de.size + trash_records.append(trash_record) for de in added_dirs: record = copy.copy(base_record) + trash_record = copy.copy(base_trash_record) op_type = '' if commit.description.startswith('Recovered'): op_type = OP_RECOVER + trash_record['op_type'] = OP_RECOVER + trash_record['obj_id'] = de.obj_id + trash_record['obj_name'] = os.path.basename(de.path) + trash_record['path'] = '/' if os.path.dirname(de.path) == '/' else os.path.dirname(de.path) + '/' + trash_record['size'] = de.size + trash_records.append(trash_record) else: op_type = OP_CREATE record['op_type'] = op_type @@ -628,6 +686,14 @@ def generate_activity_records(added_files, deleted_files, added_dirs, record['path'] = de.path records.append(record) + trash_record = copy.copy(base_trash_record) + trash_record['obj_type'] = 'dir' + trash_record['op_type'] = OP_DELETE + trash_record['obj_id'] = de.obj_id + trash_record['obj_name'] = os.path.basename(de.path) + trash_record['path'] = '/' if os.path.dirname(de.path) == '/' else os.path.dirname(de.path) + '/' + trash_record['size'] = de.size + trash_records.append(trash_record) for de in modified_files: record = copy.copy(base_record) op_type = '' @@ -694,8 +760,7 @@ def generate_activity_records(added_files, deleted_files, added_dirs, record['old_path'] = record['old_path'].rstrip('/') record['path'] = record['path'].rstrip('/') if record['path'] != '/' else '/' filtered_records.append(record) - - return filtered_records + return filtered_records, trash_records def list_file_in_dir(repo_id, dirents, op_type): _dirents = copy.copy(dirents) diff --git a/events/models.py b/events/models.py index 5ec544e3..23c914bd 100644 --- a/events/models.py +++ b/events/models.py @@ -227,3 +227,35 @@ def __str__(self): RepoID = %s, FilePath = %s, Permission = %s>" % \ (self.etype, self.from_user, self.to, self.repo_id, self.file_path, self.permission) + +class TrashRecord(Base): + __tablename__ = 'TrashRecord' + + id = mapped_column(Integer, primary_key=True, autoincrement=True) + user = mapped_column(String(length=255), nullable=False) + obj_type = mapped_column(String(length=128), nullable=False) + obj_id = mapped_column(String(length=40), nullable=False) + obj_name = mapped_column(String(length=255), nullable=False) + delete_time = mapped_column(DateTime, nullable=False, index=True) + + repo_id = mapped_column(String(length=36), nullable=False) + commit_id = mapped_column(String(length=40)) + path = mapped_column(Text, nullable=False) + size = mapped_column(BigInteger, nullable=False) + + def __init__(self, record): + super().__init__() + self.user = record['op_user'] + self.obj_type = record['obj_type'] + self.obj_id = record.get('obj_id', "") + self.obj_name = record['obj_name'] + self.delete_time = record['timestamp'] + self.repo_id = record['repo_id'] + self.path =record['path'] + self.commit_id = record.get('commit_id', None) + self.size = record.get('size', 0) + + + def __str__(self): + return 'TrashRecord' % \ + (self.id, self.obj_type, self.repo_id) \ No newline at end of file diff --git a/seafevents_api.py b/seafevents_api.py index b6888a9f..c5353b2c 100644 --- a/seafevents_api.py +++ b/seafevents_api.py @@ -1,6 +1,7 @@ from .db import init_db_session_class from .statistics.db import * from .events.db import * +from .events.handlers import get_delete_records from .content_scanner.db import * from .virus_scanner.db_oper import * from .app.config import is_repo_auto_del_enabled, is_search_enabled, is_audit_enabled