365 lines
12 KiB
Python
365 lines
12 KiB
Python
import os
|
|
import gzip
|
|
import logging
|
|
import pandas as pd
|
|
import numpy as np
|
|
from typing import Optional, Dict, Any, Tuple
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class FileHandler:
|
|
"""文件处理器"""
|
|
|
|
def __init__(self, file_root: str):
|
|
"""初始化文件处理器
|
|
|
|
Args:
|
|
file_root: 文件根目录
|
|
"""
|
|
self.file_root = file_root
|
|
logger.info(f"FileHandler initialized with root: {file_root}")
|
|
|
|
def set_file_root(self, file_root: str):
|
|
"""设置文件根目录
|
|
|
|
Args:
|
|
file_root: 新的文件根目录
|
|
"""
|
|
self.file_root = file_root
|
|
logger.info(f"File root changed to: {file_root}")
|
|
|
|
def get_file_path(self, filename: str) -> str:
|
|
"""获取文件的完整路径
|
|
|
|
Args:
|
|
filename: 文件名
|
|
|
|
Returns:
|
|
str: 文件完整路径
|
|
"""
|
|
# 解析文件名,获取日期和小时
|
|
dir_parts = filename.split('_')
|
|
if len(dir_parts) >= 2:
|
|
date = dir_parts[0]
|
|
hour = dir_parts[1][0:2]
|
|
dir_path = os.path.join(self.file_root, date, hour)
|
|
else:
|
|
dir_path = self.file_root
|
|
|
|
return os.path.join(dir_path, filename)
|
|
|
|
def ensure_directory(self, filename: str, dev_id: str = None) -> str:
|
|
"""确保文件目录存在,如果不存在则创建
|
|
|
|
Args:
|
|
filename: 文件名
|
|
dev_id: 设备ID
|
|
|
|
Returns:
|
|
str: 目录路径
|
|
"""
|
|
|
|
root_dir = self.file_root if dev_id is None else os.path.join(self.file_root, dev_id)
|
|
# 解析文件名,获取日期和小时
|
|
dir_parts = filename.split('_')
|
|
if len(dir_parts) >= 2:
|
|
date = dir_parts[0]
|
|
hour = dir_parts[1][0:2]
|
|
dir_path = os.path.join(root_dir, date, hour)
|
|
else:
|
|
dir_path = root_dir
|
|
|
|
# 创建目录
|
|
if not os.path.exists(dir_path):
|
|
os.makedirs(dir_path)
|
|
logger.info(f"Created directory: {dir_path}")
|
|
|
|
return dir_path
|
|
|
|
def file_exists(self, filename: str, dev_id: str = None) -> bool:
|
|
"""检查文件是否存在
|
|
|
|
Args:
|
|
filename: 文件名
|
|
dev_id: 设备ID
|
|
|
|
Returns:
|
|
bool: 文件是否存在
|
|
"""
|
|
file_path = self.get_file_path(filename, dev_id)
|
|
return os.path.exists(file_path)
|
|
|
|
def save_raw_data(self, filename: str, file_content: bytes,
|
|
scale: float = 1.0, offset: float = 0.0,
|
|
row_count: int = 0, dev_id: str = None) -> Optional[str]:
|
|
"""保存原始数据为CSV文件
|
|
|
|
Args:
|
|
filename: 文件名
|
|
file_content: 文件内容(字节数据)
|
|
scale: 缩放因子
|
|
offset: 偏移量
|
|
row_count: 行数限制(0表示不限制)
|
|
|
|
Returns:
|
|
str: 保存的文件名,失败返回None
|
|
"""
|
|
try:
|
|
# 确保目录存在
|
|
dir_path = self.ensure_directory(filename, dev_id)
|
|
file_path = os.path.join(dir_path, filename)
|
|
|
|
# 处理数据
|
|
data = np.frombuffer(file_content, dtype=np.float32)
|
|
|
|
# 应用缩放和偏移
|
|
if scale != 1.0:
|
|
data = data * scale
|
|
if offset != 0.0:
|
|
data = data + offset
|
|
|
|
# 限制行数
|
|
if row_count > 0 and len(data) > row_count:
|
|
data = data[:row_count]
|
|
|
|
# 保存为CSV文件
|
|
df = pd.DataFrame({
|
|
'CH1': data[::2], # 偶数索引
|
|
'CH2': data[1::2] # 奇数索引
|
|
})
|
|
|
|
df.to_csv(file_path, index=False)
|
|
logger.info(f"Saved raw data to {file_path}")
|
|
|
|
return file_path
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to save raw data: {e}")
|
|
return None
|
|
|
|
def save_gz_data(self, filename: str, file_content: bytes, dev_id: str = None) -> Optional[str]:
|
|
"""解压并保存GZ压缩数据
|
|
|
|
Args:
|
|
filename: 文件名
|
|
file_content: GZ压缩的文件内容
|
|
dev_id: 设备ID
|
|
|
|
Returns:
|
|
str: 保存的文件名,失败返回None
|
|
"""
|
|
try:
|
|
# 解压数据
|
|
decompressed = gzip.decompress(file_content)
|
|
|
|
# 保存解压后的数据
|
|
filename = filename.replace('.gz', '')
|
|
return self.save_raw_data(filename, decompressed, scale=1.0, offset=0.0, row_count=0, dev_id=dev_id)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to save GZ data: {e}")
|
|
return None
|
|
|
|
def delete_file(self, filename: str, dev_id: str = None) -> bool:
|
|
"""删除文件
|
|
|
|
Args:
|
|
filename: 文件名
|
|
dev_id: 设备ID
|
|
|
|
Returns:
|
|
bool: 删除是否成功
|
|
"""
|
|
try:
|
|
file_path = self.get_file_path(filename, dev_id)
|
|
|
|
if os.path.exists(file_path):
|
|
os.remove(file_path)
|
|
logger.info(f"Deleted file: {file_path}")
|
|
return True
|
|
else:
|
|
logger.warning(f"File not found: {file_path}")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete file {filename}: {e}")
|
|
return False
|
|
|
|
def get_file_info(self, filename: str, dev_id: str = None) -> Optional[Dict[str, Any]]:
|
|
"""获取文件信息
|
|
|
|
Args:
|
|
filename: 文件名
|
|
dev_id: 设备ID
|
|
|
|
Returns:
|
|
dict: 文件信息字典,失败返回None
|
|
"""
|
|
try:
|
|
file_path = self.get_file_path(filename, dev_id)
|
|
|
|
if not os.path.exists(file_path):
|
|
return None
|
|
|
|
stat = os.stat(file_path)
|
|
|
|
return {
|
|
'filename': filename,
|
|
'path': file_path,
|
|
'size': stat.st_size,
|
|
'created': stat.st_ctime,
|
|
'modified': stat.st_mtime,
|
|
'exists': True
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get file info for {filename}: {e}")
|
|
return None
|
|
|
|
def list_files(self, date: Optional[str] = None,
|
|
hour: Optional[str] = None, dev_id: str = None) -> list:
|
|
"""列出文件
|
|
|
|
Args:
|
|
date: 日期(可选)
|
|
hour: 小时(可选)
|
|
dev_id: 设备ID
|
|
|
|
Returns:
|
|
list: 文件列表
|
|
"""
|
|
try:
|
|
# 构建目录路径
|
|
if dev_id:
|
|
dir_path = os.path.join(self.file_root, dev_id)
|
|
else:
|
|
dir_path = self.file_root
|
|
|
|
if date and hour:
|
|
dir_path = os.path.join(dir_path, date, hour)
|
|
elif date:
|
|
dir_path = os.path.join(dir_path, date)
|
|
|
|
if not os.path.exists(dir_path):
|
|
return []
|
|
|
|
# 列出文件
|
|
files = []
|
|
for item in os.listdir(dir_path):
|
|
item_path = os.path.join(dir_path, item)
|
|
if os.path.isfile(item_path):
|
|
files.append(item)
|
|
|
|
logger.info(f"Found {len(files)} files in {dir_path}")
|
|
return files
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to list files: {e}")
|
|
return []
|
|
|
|
def process_file_upload(self, filename: str, file_content: bytes,
|
|
metadata: Dict[str, Any]) -> Optional[str]:
|
|
"""处理文件上传
|
|
|
|
Args:
|
|
filename: 文件名
|
|
file_content: 文件内容
|
|
metadata: 元数据字典
|
|
|
|
Returns:
|
|
str: 处理后的文件名,失败返回None
|
|
"""
|
|
try:
|
|
# 获取参数
|
|
file_type = metadata.get('file_type', 'raw')
|
|
compress = metadata.get('compress', 'none')
|
|
scale = float(metadata.get('scale', 1.0))
|
|
offset = float(metadata.get('offset', 0.0))
|
|
row_count = int(metadata.get('row_count', 0))
|
|
dev_id = metadata.get('dev_id', None)
|
|
|
|
# 根据文件类型处理
|
|
if file_type == 'raw':
|
|
if compress == 'none':
|
|
return self.save_raw_data(filename, file_content, scale, offset, row_count, dev_id)
|
|
elif compress == 'gz':
|
|
return self.save_gz_data(filename, file_content, dev_id)
|
|
else:
|
|
logger.error(f"Unsupported compression type: {compress}")
|
|
return None
|
|
else:
|
|
logger.error(f"Unsupported file type: {file_type}")
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to process file upload: {e}")
|
|
return None
|
|
|
|
def read_csv(self, filename: str, dev_id: str = None, nrows: Optional[int] = None) -> Optional[pd.DataFrame]:
|
|
"""读取CSV文件
|
|
|
|
Args:
|
|
filename: 文件名
|
|
dev_id: 设备ID
|
|
nrows: 读取的行数(None表示全部)
|
|
|
|
Returns:
|
|
DataFrame: 数据框,失败返回None
|
|
"""
|
|
try:
|
|
file_path = self.get_file_path(filename, dev_id)
|
|
|
|
if not os.path.exists(file_path):
|
|
logger.error(f"File not found: {file_path}")
|
|
return None
|
|
|
|
if nrows:
|
|
df = pd.read_csv(file_path, nrows=nrows)
|
|
else:
|
|
df = pd.read_csv(file_path)
|
|
|
|
logger.info(f"Read CSV file: {file_path}, shape: {df.shape}")
|
|
return df
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to read CSV file {filename}: {e}")
|
|
return None
|
|
|
|
def clean_old_files(self, days: int = 7) -> int:
|
|
"""清理旧文件
|
|
|
|
Args:
|
|
days: 保留天数
|
|
|
|
Returns:
|
|
int: 删除的文件数量
|
|
"""
|
|
try:
|
|
import time
|
|
current_time = time.time()
|
|
cutoff_time = current_time - (days * 24 * 60 * 60)
|
|
|
|
deleted_count = 0
|
|
|
|
# 遍历所有文件
|
|
for root, dirs, files in os.walk(self.file_root):
|
|
for file in files:
|
|
file_path = os.path.join(root, file)
|
|
file_mtime = os.path.getmtime(file_path)
|
|
|
|
if file_mtime < cutoff_time:
|
|
try:
|
|
os.remove(file_path)
|
|
deleted_count += 1
|
|
logger.info(f"Deleted old file: {file_path}")
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete {file_path}: {e}")
|
|
|
|
logger.info(f"Cleaned {deleted_count} old files")
|
|
return deleted_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to clean old files: {e}")
|
|
return 0
|