commit ee557dc1a5fcf9ced7d6909a3fd042967727f875 Author: Backup System Date: Mon Mar 23 14:45:12 2026 +0800 修复 WebDAV 过期备份清理功能 - 路径构造错误 问题:cleanup_old_backups 和 _delete_remote_dir 方法在构造远程删除路径时, 错误地依赖了 WebDAV 返回的 href.text 绝对路径。当服务器返回 /Data/Backup/xxx/ 这样的路径时,拼接逻辑出错导致 DELETE 请求失败(No scheme supplied)。 修复:改用 os.path.basename() 从 href.text 提取目录名,然后与 remote_base 拼接成完整 URL,确保无论 href.text 是什么格式都能正确处理。 diff --git a/backup.py b/backup.py new file mode 100644 index 0000000..d945d0f --- /dev/null +++ b/backup.py @@ -0,0 +1,517 @@ +#!/usr/bin/env python3 +""" +WebDAV备份工具 - 全量备份指定目录到WebDAV服务器 +""" + +import os +import sys +import yaml +import time +import logging +import argparse +from pathlib import Path +from datetime import datetime, timedelta +import re +import requests +from requests.auth import HTTPBasicAuth +from urllib.parse import urljoin + + +class WebDAVBackup: + """WebDAV备份类""" + + def __init__(self, config_path: str): + """初始化备份工具""" + self.config = self.load_config(config_path) + self.setup_logging() + self.session = requests.Session() + self.session.auth = HTTPBasicAuth( + self.config['webdav']['username'], + self.config['webdav']['password'] + ) + self.uploaded_count = 0 + self.failed_count = 0 + self.total_size = 0 + self.deleted_local_count = 0 + self.deleted_local_size = 0 + + def load_config(self, config_path: str) -> dict: + """加载配置文件""" + if not os.path.exists(config_path): + print(f"错误: 配置文件 {config_path} 不存在") + sys.exit(1) + + with open(config_path, 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) + + # 验证必需配置 + required_keys = ['webdav', 'backup'] + for key in required_keys: + if key not in config: + print(f"错误: 配置文件中缺少必需的 '{key}' 部分") + sys.exit(1) + + webdav_keys = ['url', 'username', 'password'] + for key in webdav_keys: + if key not in config['webdav']: + print(f"错误: 配置文件中webdav部分缺少 '{key}'") + sys.exit(1) + + if 'directories' not in config['backup']: + print("错误: 配置文件中backup部分缺少 'directories'") + sys.exit(1) + + return config + + def setup_logging(self): + """设置日志""" + log_level = self.config.get('logging', {}).get('level', 'INFO') + log_file = self.config.get('logging', {}).get('file', 'backup.log') + + logging.basicConfig( + level=getattr(logging, log_level), + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(log_file, encoding='utf-8'), + logging.StreamHandler() + ] + ) + self.logger = logging.getLogger(__name__) + + def test_connection(self) -> bool: + """测试WebDAV连接""" + self.logger.info("测试WebDAV连接...") + try: + response = self.session.request( + 'PROPFIND', + self.config['webdav']['url'], + headers={'Depth': '0'} + ) + if response.status_code in [200, 207]: + self.logger.info("WebDAV连接成功") + return True + else: + self.logger.error(f"WebDAV连接失败: HTTP {response.status_code}") + return False + except Exception as e: + self.logger.error(f"WebDAV连接异常: {e}") + return False + + def get_latest_date_dir(self, base_dir: str) -> str: + """获取最新的日期文件夹""" + if not os.path.exists(base_dir): + self.logger.error(f"数据目录不存在: {base_dir}") + return None + + date_pattern = re.compile(r'^\d{8}$') + date_dirs = [] + + for item in os.listdir(base_dir): + item_path = os.path.join(base_dir, item) + if os.path.isdir(item_path) and date_pattern.match(item): + date_dirs.append(item) + + if not date_dirs: + self.logger.error(f"未找到日期文件夹: {base_dir}") + return None + + # 按日期排序,获取最新的 + latest = sorted(date_dirs, reverse=True)[0] + self.logger.info(f"找到最新日期文件夹: {latest}") + return latest + + def get_exclude_patterns(self) -> list: + """获取排除模式""" + return self.config['backup'].get('exclude', []) + + def should_exclude(self, path: str) -> bool: + """检查路径是否应该排除""" + exclude_patterns = self.get_exclude_patterns() + path_name = os.path.basename(path) + + for pattern in exclude_patterns: + if pattern.startswith('*.'): + # 扩展名匹配 + ext = pattern[1:] + if path_name.endswith(ext): + return True + elif pattern in path_name: + # 名称包含匹配 + return True + + return False + + def create_remote_dir(self, remote_path: str): + """创建远程目录""" + try: + response = self.session.request( + 'MKCOL', + remote_path + ) + if response.status_code in [200, 201, 405]: + # 405表示已存在,也是成功的 + return True + return False + except Exception as e: + self.logger.warning(f"创建目录失败 {remote_path}: {e}") + return False + + def upload_file(self, local_path: str, remote_path: str) -> bool: + """上传单个文件""" + try: + with open(local_path, 'rb') as f: + content = f.read() + + self.logger.debug(f"上传到: {remote_path}") + response = self.session.put( + remote_path, + data=content, + headers={'Content-Type': 'application/octet-stream'} + ) + + if response.status_code in [200, 201, 204]: + file_size = len(content) + self.uploaded_count += 1 + self.total_size += file_size + return True + else: + self.logger.warning(f"上传失败 {local_path}: HTTP {response.status_code}") + self.logger.warning(f"响应内容: {response.text[:500] if response.text else '无'}") + return False + + except PermissionError: + self.logger.warning(f"权限不足 {local_path}") + self.failed_count += 1 + return False + except Exception as e: + self.logger.warning(f"上传失败 {local_path}: {e}") + self.failed_count += 1 + return False + + def delete_remote_path(self, remote_path: str) -> bool: + """删除远程文件或目录""" + try: + # 先尝试删除文件 + response = self.session.delete(remote_path) + if response.status_code in [200, 204, 404]: + return True + # 如果是目录,尝试用 PROPFIND 检查 + propfind_response = self.session.request( + 'PROPFIND', + remote_path, + headers={'Depth': '0'} + ) + if propfind_response.status_code in [200, 207]: + # 是目录,需要递归删除 + return self._delete_remote_dir(remote_path) + return False + except Exception as e: + self.logger.warning(f"删除失败 {remote_path}: {e}") + return False + + def _delete_remote_dir(self, remote_path: str) -> bool: + """递归删除远程目录""" + try: + # 先获取目录内容 + response = self.session.request( + 'PROPFIND', + remote_path, + headers={'Depth': '1'} + ) + if response.status_code not in [200, 207]: + return False + + # 解析XML获取子项 + from xml.etree import ElementTree + ns = {'d': 'DAV:'} + try: + root = ElementTree.fromstring(response.text) + # 删除所有子文件 + for response_elem in root.findall('.//d:response', ns): + href = response_elem.find('d:href', ns) + if href is not None: + href_text = href.text.rstrip('/') + if href_text != remote_path.rstrip('/'): + # 从 href.text 提取子项名称,构造完整路径 + child_name = os.path.basename(href_text) + child_path = remote_path.rstrip('/') + '/' + child_name + self.delete_remote_path(child_path) + except Exception: + pass + + # 最后删除目录本身 + delete_response = self.session.delete(remote_path + '/') + return delete_response.status_code in [200, 204, 404] + + except Exception as e: + self.logger.warning(f"删除目录失败 {remote_path}: {e}") + return False + + def cleanup_old_backups(self, remote_base: str, days: int = 3): + """清理超过指定天数的备份""" + self.logger.info(f"开始清理 {days} 天前的备份...") + + # 确保 remote_base 以 / 结尾 + if not remote_base.endswith('/'): + remote_base += '/' + + try: + # 获取远程目录列表 + response = self.session.request( + 'PROPFIND', + remote_base, + headers={'Depth': '1'} + ) + + if response.status_code not in [200, 207]: + self.logger.warning(f"无法获取远程目录列表: HTTP {response.status_code}") + return + + # 解析XML + from xml.etree import ElementTree + ns = {'d': 'DAV:'} + root = ElementTree.fromstring(response.text) + + # 计算过期日期 + expire_date = datetime.now() - timedelta(days=days) + deleted_count = 0 + + for response_elem in root.findall('.//d:response', ns): + href = response_elem.find('d:href', ns) + if href is None: + continue + + # 先从 href.text 提取日期目录名称 + dir_name = os.path.basename(href.text.rstrip('/')) + + # 使用 remote_base + dir_name 构建完整路径 + # 这样无论 href.text 是什么格式(完整URL、绝对路径、相对路径) + # 都能正确构造出远程路径 + remote_path = remote_base.rstrip('/') + '/' + dir_name + + # 检查是否是日期格式的目录 + if not re.match(r'^\d{8}$', dir_name): + continue + + # 解析日期 + try: + dir_date = datetime.strptime(dir_name, '%Y%m%d') + except ValueError: + continue + + # 检查是否过期 + if dir_date < expire_date: + self.logger.info(f"删除过期备份: {dir_name}") + if self.delete_remote_path(remote_path): + deleted_count += 1 + self.logger.info(f"已删除: {dir_name}") + + self.logger.info(f"清理完成,共删除 {deleted_count} 个过期备份") + + except Exception as e: + self.logger.error(f"清理备份时发生错误: {e}") + + def backup_directory(self, local_dir: str, remote_base: str): + """备份目录""" + local_dir = os.path.abspath(local_dir) + dir_name = os.path.basename(local_dir) + remote_dir = urljoin(remote_base, dir_name + '/') + + self.logger.info(f"开始备份目录: {local_dir}") + + # 创建远程根目录 + self.create_remote_dir(remote_dir) + + # 遍历所有文件 + for root, dirs, files in os.walk(local_dir): + # 计算相对路径 + rel_dir = os.path.relpath(root, local_dir) + + # 处理排除的目录 + if rel_dir != '.' and self.should_exclude(rel_dir): + dirs.clear() + continue + + # 构建远程目录路径 + if rel_dir == '.': + current_remote_dir = remote_dir + else: + current_remote_dir = urljoin(remote_dir, rel_dir + '/') + + # 创建子目录 + for d in dirs: + if self.should_exclude(d): + continue + self.create_remote_dir(urljoin(current_remote_dir, d + '/')) + + # 上传文件 + for filename in files: + if self.should_exclude(filename): + continue + + local_path = os.path.join(root, filename) + remote_path = urljoin(current_remote_dir, filename) + + if self.upload_file(local_path, remote_path): + self.logger.debug(f"已上传: {local_path}") + else: + self.logger.warning(f"跳过: {local_path}") + + def run(self): + """执行备份""" + self.logger.info("=" * 50) + self.logger.info("WebDAV备份开始") + self.logger.info("=" * 50) + + # 测试连接 + if not self.test_connection(): + self.logger.error("WebDAV连接失败,退出") + sys.exit(1) + + # 获取远程路径基础URL + remote_base = self.config['webdav']['url'] + if not remote_base.endswith('/'): + remote_base += '/' + + # 获取保留天数配置 + retain_days = self.config['backup'].get('retain_days', 3) + + # 获取上传后是否删除本地文件的配置 + delete_after_upload = self.config['backup'].get('delete_after_upload', False) + + # 用于记录已上传的日期文件夹路径 + uploaded_dirs = [] + + # 备份每个目录(只上传最新日期文件夹) + directories = self.config['backup']['directories'] + for local_dir in directories: + if not os.path.exists(local_dir): + self.logger.warning(f"目录不存在,跳过: {local_dir}") + continue + + if not os.path.isdir(local_dir): + self.logger.warning(f"不是有效目录,跳过: {local_dir}") + continue + + # 获取最新日期文件夹 + latest_date = self.get_latest_date_dir(local_dir) + if latest_date: + date_dir = os.path.join(local_dir, latest_date) + self.backup_directory(date_dir, remote_base) + # 记录已上传的目录,用于后续删除 + uploaded_dirs.append((date_dir, latest_date)) + + # 上传完成后删除本地文件 + if delete_after_upload and uploaded_dirs: + self.logger.info("=" * 50) + self.logger.info("开始删除已上传的本地文件...") + for date_dir, latest_date in uploaded_dirs: + try: + # 计算目录大小 + dir_size = self.get_dir_size(date_dir) + import shutil + shutil.rmtree(date_dir) + self.deleted_local_count += 1 + self.deleted_local_size += dir_size + self.logger.info(f"已删除本地目录: {date_dir}") + except Exception as e: + self.logger.warning(f"删除本地目录失败 {date_dir}: {e}") + + # 输出统计 + self.logger.info("=" * 50) + self.logger.info("备份完成") + self.logger.info(f"成功上传: {self.uploaded_count} 个文件") + self.logger.info(f"总大小: {self.format_size(self.total_size)}") + self.logger.info(f"失败数量: {self.failed_count}") + + # 输出删除本地文件的统计 + if delete_after_upload and self.deleted_local_count > 0: + self.logger.info(f"已删除本地目录: {self.deleted_local_count} 个") + self.logger.info(f"释放空间: {self.format_size(self.deleted_local_size)}") + + # 清理过期备份 + self.cleanup_old_backups(remote_base, retain_days) + + self.logger.info("=" * 50) + + def get_dir_size(self, dir_path: str) -> int: + """计算目录大小""" + total_size = 0 + try: + for root, dirs, files in os.walk(dir_path): + for file in files: + file_path = os.path.join(root, file) + try: + total_size += os.path.getsize(file_path) + except (OSError, IOError): + pass + except Exception: + pass + return total_size + + @staticmethod + def format_size(size: int) -> str: + """格式化文件大小""" + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024: + return f"{size:.2f} {unit}" + size /= 1024 + return f"{size:.2f} PB" + + +def create_default_config(config_path: str): + """创建默认配置文件""" + default_config = { + 'webdav': { + 'url': 'https://dav.jianguoyun.com/dav/', + 'username': 'your_username', + 'password': 'your_password' + }, + 'backup': { + 'directories': [ + '/path/to/backup/dir1', + '/path/to/backup/dir2' + ], + 'exclude': [ + '*.tmp', + '*.log', + '__pycache__', + '.git', + 'node_modules' + ] + }, + 'logging': { + 'level': 'INFO', + 'file': 'backup.log' + } + } + + with open(config_path, 'w', encoding='utf-8') as f: + yaml.dump(default_config, f, default_flow_style=False, allow_unicode=True) + + print(f"默认配置文件已创建: {config_path}") + print("请编辑配置文件后再运行程序") + + +def main(): + """主函数""" + parser = argparse.ArgumentParser(description='WebDAV备份工具') + parser.add_argument('-c', '--config', default='config.yaml', + help='配置文件路径 (默认: config.yaml)') + parser.add_argument('--init', action='store_true', + help='创建默认配置文件') + + args = parser.parse_args() + + # 创建默认配置 + if args.init: + create_default_config(args.config) + sys.exit(0) + + # 运行备份 + backup = WebDAVBackup(args.config) + backup.run() + + +if __name__ == '__main__': + main()