import os import gzip import logging import pandas as pd import numpy as np from typing import Optional, Dict, Any, Tuple logger = logging.getLogger(__name__) class FileHandler: """文件处理器""" def __init__(self, file_root: str): """初始化文件处理器 Args: file_root: 文件根目录 """ self.file_root = file_root logger.info(f"FileHandler initialized with root: {file_root}") def set_file_root(self, file_root: str): """设置文件根目录 Args: file_root: 新的文件根目录 """ self.file_root = file_root logger.info(f"File root changed to: {file_root}") def get_file_path(self, filename: str) -> str: """获取文件的完整路径 Args: filename: 文件名 Returns: str: 文件完整路径 """ # 解析文件名,获取日期和小时 dir_parts = filename.split('_') if len(dir_parts) >= 2: date = dir_parts[0] hour = dir_parts[1][0:2] dir_path = os.path.join(self.file_root, date, hour) else: dir_path = self.file_root return os.path.join(dir_path, filename) def ensure_directory(self, filename: str, dev_id: str = None) -> str: """确保文件目录存在,如果不存在则创建 Args: filename: 文件名 dev_id: 设备ID Returns: str: 目录路径 """ root_dir = self.file_root if dev_id is None else os.path.join(self.file_root, dev_id) # 解析文件名,获取日期和小时 dir_parts = filename.split('_') if len(dir_parts) >= 2: date = dir_parts[0] hour = dir_parts[1][0:2] dir_path = os.path.join(root_dir, date, hour) else: dir_path = root_dir # 创建目录 if not os.path.exists(dir_path): os.makedirs(dir_path) logger.info(f"Created directory: {dir_path}") return dir_path def file_exists(self, filename: str, dev_id: str = None) -> bool: """检查文件是否存在 Args: filename: 文件名 dev_id: 设备ID Returns: bool: 文件是否存在 """ file_path = self.get_file_path(filename, dev_id) return os.path.exists(file_path) def save_raw_data(self, filename: str, file_content: bytes, scale: float = 1.0, offset: float = 0.0, row_count: int = 0, dev_id: str = None) -> Optional[str]: """保存原始数据为CSV文件 Args: filename: 文件名 file_content: 文件内容(字节数据) scale: 缩放因子 offset: 偏移量 row_count: 行数限制(0表示不限制) Returns: str: 保存的文件名,失败返回None """ try: # 确保目录存在 dir_path = self.ensure_directory(filename, dev_id) file_path = os.path.join(dir_path, filename) # 处理数据 data = np.frombuffer(file_content, dtype=np.float32) # 应用缩放和偏移 if scale != 1.0: data = data * scale if offset != 0.0: data = data + offset # 限制行数 if row_count > 0 and len(data) > row_count: data = data[:row_count] # 保存为CSV文件 df = pd.DataFrame({ 'CH1': data[::2], # 偶数索引 'CH2': data[1::2] # 奇数索引 }) df.to_csv(file_path, index=False) logger.info(f"Saved raw data to {file_path}") return file_path except Exception as e: logger.error(f"Failed to save raw data: {e}") return None def save_gz_data(self, filename: str, file_content: bytes, dev_id: str = None) -> Optional[str]: """解压并保存GZ压缩数据 Args: filename: 文件名 file_content: GZ压缩的文件内容 dev_id: 设备ID Returns: str: 保存的文件名,失败返回None """ try: # 解压数据 decompressed = gzip.decompress(file_content) # 保存解压后的数据 filename = filename.replace('.gz', '') return self.save_raw_data(filename, decompressed, scale=1.0, offset=0.0, row_count=0, dev_id=dev_id) except Exception as e: logger.error(f"Failed to save GZ data: {e}") return None def delete_file(self, filename: str, dev_id: str = None) -> bool: """删除文件 Args: filename: 文件名 dev_id: 设备ID Returns: bool: 删除是否成功 """ try: file_path = self.get_file_path(filename, dev_id) if os.path.exists(file_path): os.remove(file_path) logger.info(f"Deleted file: {file_path}") return True else: logger.warning(f"File not found: {file_path}") return False except Exception as e: logger.error(f"Failed to delete file {filename}: {e}") return False def get_file_info(self, filename: str, dev_id: str = None) -> Optional[Dict[str, Any]]: """获取文件信息 Args: filename: 文件名 dev_id: 设备ID Returns: dict: 文件信息字典,失败返回None """ try: file_path = self.get_file_path(filename, dev_id) if not os.path.exists(file_path): return None stat = os.stat(file_path) return { 'filename': filename, 'path': file_path, 'size': stat.st_size, 'created': stat.st_ctime, 'modified': stat.st_mtime, 'exists': True } except Exception as e: logger.error(f"Failed to get file info for {filename}: {e}") return None def list_files(self, date: Optional[str] = None, hour: Optional[str] = None, dev_id: str = None) -> list: """列出文件 Args: date: 日期(可选) hour: 小时(可选) dev_id: 设备ID Returns: list: 文件列表 """ try: # 构建目录路径 if dev_id: dir_path = os.path.join(self.file_root, dev_id) else: dir_path = self.file_root if date and hour: dir_path = os.path.join(dir_path, date, hour) elif date: dir_path = os.path.join(dir_path, date) if not os.path.exists(dir_path): return [] # 列出文件 files = [] for item in os.listdir(dir_path): item_path = os.path.join(dir_path, item) if os.path.isfile(item_path): files.append(item) logger.info(f"Found {len(files)} files in {dir_path}") return files except Exception as e: logger.error(f"Failed to list files: {e}") return [] def process_file_upload(self, filename: str, file_content: bytes, metadata: Dict[str, Any]) -> Optional[str]: """处理文件上传 Args: filename: 文件名 file_content: 文件内容 metadata: 元数据字典 Returns: str: 处理后的文件名,失败返回None """ try: # 获取参数 file_type = metadata.get('file_type', 'raw') compress = metadata.get('compress', 'none') scale = float(metadata.get('scale', 1.0)) offset = float(metadata.get('offset', 0.0)) row_count = int(metadata.get('row_count', 0)) dev_id = metadata.get('dev_id', None) # 根据文件类型处理 if file_type == 'raw': if compress == 'none': return self.save_raw_data(filename, file_content, scale, offset, row_count, dev_id) elif compress == 'gz': return self.save_gz_data(filename, file_content, dev_id) else: logger.error(f"Unsupported compression type: {compress}") return None else: logger.error(f"Unsupported file type: {file_type}") return None except Exception as e: logger.error(f"Failed to process file upload: {e}") return None def read_csv(self, filename: str, dev_id: str = None, nrows: Optional[int] = None) -> Optional[pd.DataFrame]: """读取CSV文件 Args: filename: 文件名 dev_id: 设备ID nrows: 读取的行数(None表示全部) Returns: DataFrame: 数据框,失败返回None """ try: file_path = self.get_file_path(filename, dev_id) if not os.path.exists(file_path): logger.error(f"File not found: {file_path}") return None if nrows: df = pd.read_csv(file_path, nrows=nrows) else: df = pd.read_csv(file_path) logger.info(f"Read CSV file: {file_path}, shape: {df.shape}") return df except Exception as e: logger.error(f"Failed to read CSV file {filename}: {e}") return None def clean_old_files(self, days: int = 7) -> int: """清理旧文件 Args: days: 保留天数 Returns: int: 删除的文件数量 """ try: import time current_time = time.time() cutoff_time = current_time - (days * 24 * 60 * 60) deleted_count = 0 # 遍历所有文件 for root, dirs, files in os.walk(self.file_root): for file in files: file_path = os.path.join(root, file) file_mtime = os.path.getmtime(file_path) if file_mtime < cutoff_time: try: os.remove(file_path) deleted_count += 1 logger.info(f"Deleted old file: {file_path}") except Exception as e: logger.error(f"Failed to delete {file_path}: {e}") logger.info(f"Cleaned {deleted_count} old files") return deleted_count except Exception as e: logger.error(f"Failed to clean old files: {e}") return 0