first commit

This commit is contained in:
WIN-20230613SRA\Administrator
2026-07-02 16:07:26 +08:00
commit c4a347b287
17 changed files with 2924 additions and 0 deletions
+364
View File
@@ -0,0 +1,364 @@
import os
import gzip
import logging
import pandas as pd
import numpy as np
from typing import Optional, Dict, Any, Tuple
logger = logging.getLogger(__name__)
class FileHandler:
"""文件处理器"""
def __init__(self, file_root: str):
"""初始化文件处理器
Args:
file_root: 文件根目录
"""
self.file_root = file_root
logger.info(f"FileHandler initialized with root: {file_root}")
def set_file_root(self, file_root: str):
"""设置文件根目录
Args:
file_root: 新的文件根目录
"""
self.file_root = file_root
logger.info(f"File root changed to: {file_root}")
def get_file_path(self, filename: str) -> str:
"""获取文件的完整路径
Args:
filename: 文件名
Returns:
str: 文件完整路径
"""
# 解析文件名,获取日期和小时
dir_parts = filename.split('_')
if len(dir_parts) >= 2:
date = dir_parts[0]
hour = dir_parts[1][0:2]
dir_path = os.path.join(self.file_root, date, hour)
else:
dir_path = self.file_root
return os.path.join(dir_path, filename)
def ensure_directory(self, filename: str, dev_id: str = None) -> str:
"""确保文件目录存在,如果不存在则创建
Args:
filename: 文件名
dev_id: 设备ID
Returns:
str: 目录路径
"""
root_dir = self.file_root if dev_id is None else os.path.join(self.file_root, dev_id)
# 解析文件名,获取日期和小时
dir_parts = filename.split('_')
if len(dir_parts) >= 2:
date = dir_parts[0]
hour = dir_parts[1][0:2]
dir_path = os.path.join(root_dir, date, hour)
else:
dir_path = root_dir
# 创建目录
if not os.path.exists(dir_path):
os.makedirs(dir_path)
logger.info(f"Created directory: {dir_path}")
return dir_path
def file_exists(self, filename: str, dev_id: str = None) -> bool:
"""检查文件是否存在
Args:
filename: 文件名
dev_id: 设备ID
Returns:
bool: 文件是否存在
"""
file_path = self.get_file_path(filename, dev_id)
return os.path.exists(file_path)
def save_raw_data(self, filename: str, file_content: bytes,
scale: float = 1.0, offset: float = 0.0,
row_count: int = 0, dev_id: str = None) -> Optional[str]:
"""保存原始数据为CSV文件
Args:
filename: 文件名
file_content: 文件内容(字节数据)
scale: 缩放因子
offset: 偏移量
row_count: 行数限制(0表示不限制)
Returns:
str: 保存的文件名,失败返回None
"""
try:
# 确保目录存在
dir_path = self.ensure_directory(filename, dev_id)
file_path = os.path.join(dir_path, filename)
# 处理数据
data = np.frombuffer(file_content, dtype=np.float32)
# 应用缩放和偏移
if scale != 1.0:
data = data * scale
if offset != 0.0:
data = data + offset
# 限制行数
if row_count > 0 and len(data) > row_count:
data = data[:row_count]
# 保存为CSV文件
df = pd.DataFrame({
'CH1': data[::2], # 偶数索引
'CH2': data[1::2] # 奇数索引
})
df.to_csv(file_path, index=False)
logger.info(f"Saved raw data to {file_path}")
return file_path
except Exception as e:
logger.error(f"Failed to save raw data: {e}")
return None
def save_gz_data(self, filename: str, file_content: bytes, dev_id: str = None) -> Optional[str]:
"""解压并保存GZ压缩数据
Args:
filename: 文件名
file_content: GZ压缩的文件内容
dev_id: 设备ID
Returns:
str: 保存的文件名,失败返回None
"""
try:
# 解压数据
decompressed = gzip.decompress(file_content)
# 保存解压后的数据
filename = filename.replace('.gz', '')
return self.save_raw_data(filename, decompressed, scale=1.0, offset=0.0, row_count=0, dev_id=dev_id)
except Exception as e:
logger.error(f"Failed to save GZ data: {e}")
return None
def delete_file(self, filename: str, dev_id: str = None) -> bool:
"""删除文件
Args:
filename: 文件名
dev_id: 设备ID
Returns:
bool: 删除是否成功
"""
try:
file_path = self.get_file_path(filename, dev_id)
if os.path.exists(file_path):
os.remove(file_path)
logger.info(f"Deleted file: {file_path}")
return True
else:
logger.warning(f"File not found: {file_path}")
return False
except Exception as e:
logger.error(f"Failed to delete file {filename}: {e}")
return False
def get_file_info(self, filename: str, dev_id: str = None) -> Optional[Dict[str, Any]]:
"""获取文件信息
Args:
filename: 文件名
dev_id: 设备ID
Returns:
dict: 文件信息字典,失败返回None
"""
try:
file_path = self.get_file_path(filename, dev_id)
if not os.path.exists(file_path):
return None
stat = os.stat(file_path)
return {
'filename': filename,
'path': file_path,
'size': stat.st_size,
'created': stat.st_ctime,
'modified': stat.st_mtime,
'exists': True
}
except Exception as e:
logger.error(f"Failed to get file info for {filename}: {e}")
return None
def list_files(self, date: Optional[str] = None,
hour: Optional[str] = None, dev_id: str = None) -> list:
"""列出文件
Args:
date: 日期(可选)
hour: 小时(可选)
dev_id: 设备ID
Returns:
list: 文件列表
"""
try:
# 构建目录路径
if dev_id:
dir_path = os.path.join(self.file_root, dev_id)
else:
dir_path = self.file_root
if date and hour:
dir_path = os.path.join(dir_path, date, hour)
elif date:
dir_path = os.path.join(dir_path, date)
if not os.path.exists(dir_path):
return []
# 列出文件
files = []
for item in os.listdir(dir_path):
item_path = os.path.join(dir_path, item)
if os.path.isfile(item_path):
files.append(item)
logger.info(f"Found {len(files)} files in {dir_path}")
return files
except Exception as e:
logger.error(f"Failed to list files: {e}")
return []
def process_file_upload(self, filename: str, file_content: bytes,
metadata: Dict[str, Any]) -> Optional[str]:
"""处理文件上传
Args:
filename: 文件名
file_content: 文件内容
metadata: 元数据字典
Returns:
str: 处理后的文件名,失败返回None
"""
try:
# 获取参数
file_type = metadata.get('file_type', 'raw')
compress = metadata.get('compress', 'none')
scale = float(metadata.get('scale', 1.0))
offset = float(metadata.get('offset', 0.0))
row_count = int(metadata.get('row_count', 0))
dev_id = metadata.get('dev_id', None)
# 根据文件类型处理
if file_type == 'raw':
if compress == 'none':
return self.save_raw_data(filename, file_content, scale, offset, row_count, dev_id)
elif compress == 'gz':
return self.save_gz_data(filename, file_content, dev_id)
else:
logger.error(f"Unsupported compression type: {compress}")
return None
else:
logger.error(f"Unsupported file type: {file_type}")
return None
except Exception as e:
logger.error(f"Failed to process file upload: {e}")
return None
def read_csv(self, filename: str, dev_id: str = None, nrows: Optional[int] = None) -> Optional[pd.DataFrame]:
"""读取CSV文件
Args:
filename: 文件名
dev_id: 设备ID
nrows: 读取的行数(None表示全部)
Returns:
DataFrame: 数据框,失败返回None
"""
try:
file_path = self.get_file_path(filename, dev_id)
if not os.path.exists(file_path):
logger.error(f"File not found: {file_path}")
return None
if nrows:
df = pd.read_csv(file_path, nrows=nrows)
else:
df = pd.read_csv(file_path)
logger.info(f"Read CSV file: {file_path}, shape: {df.shape}")
return df
except Exception as e:
logger.error(f"Failed to read CSV file {filename}: {e}")
return None
def clean_old_files(self, days: int = 7) -> int:
"""清理旧文件
Args:
days: 保留天数
Returns:
int: 删除的文件数量
"""
try:
import time
current_time = time.time()
cutoff_time = current_time - (days * 24 * 60 * 60)
deleted_count = 0
# 遍历所有文件
for root, dirs, files in os.walk(self.file_root):
for file in files:
file_path = os.path.join(root, file)
file_mtime = os.path.getmtime(file_path)
if file_mtime < cutoff_time:
try:
os.remove(file_path)
deleted_count += 1
logger.info(f"Deleted old file: {file_path}")
except Exception as e:
logger.error(f"Failed to delete {file_path}: {e}")
logger.info(f"Cleaned {deleted_count} old files")
return deleted_count
except Exception as e:
logger.error(f"Failed to clean old files: {e}")
return 0