25 KiB
25 KiB
系统资源动态调整优化方案
基于CPU核心数和实时负载的智能资源管理
📊 现有CPU自适应机制分析
当前实现状况
# ✅ 现有的基础CPU自适应（HealthDataOptimizer V4.0）
class HealthDataOptimizer:
def __init__(self):
self.cpu_cores = psutil.cpu_count(logical=True)
self.memory_gb = psutil.virtual_memory().total / (1024**3)
# 静态公式计算
self.batch_size = max(50, min(500, self.cpu_cores * 25))
max_workers = max(4, min(32, int(self.cpu_cores * 2.5)))
发现的局限性
-
静态配置问题
- 只在初始化时计算一次
- 无法响应实时负载变化
- 固定的CPU倍数系数（25倍、2.5倍）
-
缺乏智能调整
- 不考虑内存使用率
- 不监控队列深度
- 不响应系统负载变化
-
资源利用不充分
- 未考虑不同类型任务的资源需求
- 缺乏动态扩缩容机制
🚀 智能动态资源调整方案
方案1：实时性能监控和动态调整
import psutil
import threading
import time
import queue
from dataclasses import dataclass
from typing import Dict, List, Tuple
from concurrent.futures import ThreadPoolExecutor
import asyncio
@dataclass
class SystemMetrics:
"""系统性能指标"""
cpu_usage: float # CPU使用率 %
memory_usage: float # 内存使用率 %
cpu_cores: int # CPU核心数
available_memory: float # 可用内存 GB
queue_depth: int # 队列深度
processing_rate: float # 处理速率 records/sec
avg_response_time: float # 平均响应时间 ms
error_rate: float # 错误率 %
@dataclass
class ResourceConfig:
"""资源配置参数"""
batch_size: int # 批处理大小
worker_count: int # 工作线程数
queue_size: int # 队列大小
timeout: float # 超时时间
class SmartResourceManager:
"""智能资源管理器 - 基于实时负载动态调整"""
def __init__(self):
# 基础系统信息
self.cpu_cores = psutil.cpu_count(logical=True)
self.cpu_cores_physical = psutil.cpu_count(logical=False)
self.memory_total = psutil.virtual_memory().total / (1024**3)
# 动态配置参数
self.current_config = self._calculate_initial_config()
self.min_config = self._get_min_config()
self.max_config = self._get_max_config()
# 性能监控
self.metrics_history = []
self.monitoring_window = 60 # 监控窗口60秒
self.adjustment_interval = 10 # 每10秒检查一次
self.last_adjustment = time.time()
# 自适应参数
self.cpu_target_usage = 70.0 # 目标CPU使用率70%
self.memory_target_usage = 80.0 # 目标内存使用率80%
self.queue_depth_threshold = 0.8 # 队列深度阈值80%
# 启动监控线程
self.monitoring_active = True
self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True)
self.monitor_thread.start()
logger.info(f"🧠 SmartResourceManager 初始化完成")
logger.info(f" CPU: {self.cpu_cores}逻辑核心/{self.cpu_cores_physical}物理核心")
logger.info(f" 内存: {self.memory_total:.1f}GB")
logger.info(f" 初始配置: {self.current_config}")
def _calculate_initial_config(self) -> ResourceConfig:
"""计算初始配置 - 基于系统硬件"""
# 🚀 智能CPU系数计算
if self.cpu_cores <= 4:
# 低端系统：保守配置
batch_multiplier = 20
worker_multiplier = 2.0
elif self.cpu_cores <= 8:
# 中端系统：均衡配置
batch_multiplier = 25
worker_multiplier = 2.5
elif self.cpu_cores <= 16:
# 高端系统：激进配置
batch_multiplier = 30
worker_multiplier = 3.0
else:
# 服务器级别：最大性能
batch_multiplier = 35
worker_multiplier = 3.5
# 🎯 内存影响因子
memory_factor = min(2.0, max(0.5, self.memory_total / 8.0)) # 8GB为基准
batch_size = int(self.cpu_cores * batch_multiplier * memory_factor)
batch_size = max(50, min(1000, batch_size))
worker_count = int(self.cpu_cores * worker_multiplier)
worker_count = max(4, min(64, worker_count))
queue_size = batch_size * 10 # 队列大小为批次大小的10倍
return ResourceConfig(
batch_size=batch_size,
worker_count=worker_count,
queue_size=queue_size,
timeout=2.0
)
def _get_min_config(self) -> ResourceConfig:
"""最小资源配置"""
return ResourceConfig(
batch_size=max(25, self.cpu_cores * 10),
worker_count=max(2, self.cpu_cores // 2),
queue_size=500,
timeout=5.0
)
def _get_max_config(self) -> ResourceConfig:
"""最大资源配置"""
return ResourceConfig(
batch_size=min(2000, self.cpu_cores * 50),
worker_count=min(128, self.cpu_cores * 4),
queue_size=20000,
timeout=1.0
)
def collect_metrics(self, queue_depth: int, processing_rate: float,
avg_response_time: float, error_rate: float) -> SystemMetrics:
"""收集系统性能指标"""
cpu_percent = psutil.cpu_percent(interval=0.1)
memory = psutil.virtual_memory()
metrics = SystemMetrics(
cpu_usage=cpu_percent,
memory_usage=memory.percent,
cpu_cores=self.cpu_cores,
available_memory=memory.available / (1024**3),
queue_depth=queue_depth,
processing_rate=processing_rate,
avg_response_time=avg_response_time,
error_rate=error_rate
)
# 保持最近60秒的指标
now = time.time()
self.metrics_history.append((now, metrics))
self.metrics_history = [
(t, m) for t, m in self.metrics_history
if now - t <= self.monitoring_window
]
return metrics
def _monitor_loop(self):
"""监控循环 - 定期检查并调整资源配置"""
while self.monitoring_active:
try:
time.sleep(self.adjustment_interval)
if len(self.metrics_history) < 3:
continue # 数据不足，跳过调整
should_adjust, new_config = self._should_adjust_resources()
if should_adjust:
self._apply_config_change(new_config)
except Exception as e:
logger.error(f"资源监控循环异常: {e}")
def _should_adjust_resources(self) -> Tuple[bool, ResourceConfig]:
"""判断是否需要调整资源配置"""
if not self.metrics_history:
return False, self.current_config
# 计算最近指标的平均值
recent_metrics = [m for _, m in self.metrics_history[-6:]] # 最近6次
avg_cpu = sum(m.cpu_usage for m in recent_metrics) / len(recent_metrics)
avg_memory = sum(m.memory_usage for m in recent_metrics) / len(recent_metrics)
avg_queue_ratio = sum(m.queue_depth / self.current_config.queue_size for m in recent_metrics) / len(recent_metrics)
avg_processing_rate = sum(m.processing_rate for m in recent_metrics) / len(recent_metrics)
logger.debug(f"📊 系统指标 - CPU: {avg_cpu:.1f}%, 内存: {avg_memory:.1f}%, 队列: {avg_queue_ratio:.1f}, 处理率: {avg_processing_rate:.1f}/s")
# 🚀 动态调整逻辑
new_config = ResourceConfig(
batch_size=self.current_config.batch_size,
worker_count=self.current_config.worker_count,
queue_size=self.current_config.queue_size,
timeout=self.current_config.timeout
)
config_changed = False
# 1. CPU使用率调整
if avg_cpu < 50.0 and avg_queue_ratio > 0.7:
# CPU空闲但队列积压 -> 增加工作线程和批次大小
new_config.worker_count = min(
self.max_config.worker_count,
int(self.current_config.worker_count * 1.3)
)
new_config.batch_size = min(
self.max_config.batch_size,
int(self.current_config.batch_size * 1.2)
)
config_changed = True
logger.info(f"🚀 检测到CPU空闲且队列积压，增加处理能力")
elif avg_cpu > 85.0:
# CPU过载 -> 减少工作线程
new_config.worker_count = max(
self.min_config.worker_count,
int(self.current_config.worker_count * 0.8)
)
config_changed = True
logger.info(f"⚠️ 检测到CPU过载，减少工作线程")
# 2. 内存使用率调整
if avg_memory > 90.0:
# 内存紧张 -> 减少批次大小和队列大小
new_config.batch_size = max(
self.min_config.batch_size,
int(self.current_config.batch_size * 0.7)
)
new_config.queue_size = max(
self.min_config.queue_size,
int(self.current_config.queue_size * 0.8)
)
config_changed = True
logger.info(f"⚠️ 检测到内存紧张，减少批次和队列大小")
# 3. 队列深度调整
if avg_queue_ratio > 0.9:
# 队列接近满载 -> 扩大队列并增加处理能力
new_config.queue_size = min(
self.max_config.queue_size,
int(self.current_config.queue_size * 1.5)
)
if avg_cpu < 70.0: # 只在CPU不繁忙时增加工作线程
new_config.worker_count = min(
self.max_config.worker_count,
int(self.current_config.worker_count * 1.2)
)
config_changed = True
logger.info(f"📈 检测到队列接近满载，扩展处理能力")
# 4. 处理效率优化
if avg_processing_rate < 10.0 and avg_cpu < 60.0:
# 处理率低且CPU空闲 -> 调整超时和批次策略
new_config.timeout = max(0.5, self.current_config.timeout * 0.8)
new_config.batch_size = min(
self.max_config.batch_size,
int(self.current_config.batch_size * 1.1)
)
config_changed = True
logger.info(f"⚡ 检测到处理效率低，优化批次策略")
return config_changed, new_config
def _apply_config_change(self, new_config: ResourceConfig):
"""应用配置更改"""
old_config = self.current_config
self.current_config = new_config
self.last_adjustment = time.time()
logger.info(f"🔧 资源配置已调整:")
logger.info(f" 批次大小: {old_config.batch_size} → {new_config.batch_size}")
logger.info(f" 工作线程: {old_config.worker_count} → {new_config.worker_count}")
logger.info(f" 队列大小: {old_config.queue_size} → {new_config.queue_size}")
logger.info(f" 超时时间: {old_config.timeout} → {new_config.timeout}")
# 触发配置更新回调（由使用者实现）
self._notify_config_change(old_config, new_config)
def _notify_config_change(self, old_config: ResourceConfig, new_config: ResourceConfig):
"""通知配置变更 - 由子类或回调实现"""
pass
def get_current_config(self) -> ResourceConfig:
"""获取当前资源配置"""
return self.current_config
def get_metrics_summary(self) -> Dict:
"""获取性能指标摘要"""
if not self.metrics_history:
return {}
recent_metrics = [m for _, m in self.metrics_history[-10:]]
return {
'cpu_usage': {
'current': recent_metrics[-1].cpu_usage,
'avg_10min': sum(m.cpu_usage for m in recent_metrics) / len(recent_metrics),
'max_10min': max(m.cpu_usage for m in recent_metrics)
},
'memory_usage': {
'current': recent_metrics[-1].memory_usage,
'avg_10min': sum(m.memory_usage for m in recent_metrics) / len(recent_metrics)
},
'processing_rate': {
'current': recent_metrics[-1].processing_rate,
'avg_10min': sum(m.processing_rate for m in recent_metrics) / len(recent_metrics)
},
'queue_utilization': recent_metrics[-1].queue_depth / self.current_config.queue_size,
'config': {
'batch_size': self.current_config.batch_size,
'worker_count': self.current_config.worker_count,
'queue_size': self.current_config.queue_size
}
}
def shutdown(self):
"""关闭资源管理器"""
self.monitoring_active = False
if self.monitor_thread.is_alive():
self.monitor_thread.join(timeout=5.0)
方案2：集成到现有健康数据处理器
class SmartHealthDataOptimizer(HealthDataOptimizer):
"""集成智能资源管理的健康数据优化器"""
def __init__(self):
# 初始化资源管理器
self.resource_manager = SmartResourceManager()
# 使用动态配置初始化
initial_config = self.resource_manager.get_current_config()
# 原有初始化逻辑
self.cpu_cores = self.resource_manager.cpu_cores
self.memory_gb = self.resource_manager.memory_total
self.batch_size = initial_config.batch_size
self.batch_timeout = initial_config.timeout
# 动态线程池 - 支持重新配置
self.executor = DynamicThreadPoolExecutor(
max_workers=initial_config.worker_count,
resource_manager=self.resource_manager
)
self.batch_queue = queue.Queue(maxsize=initial_config.queue_size)
# 性能统计
self.performance_tracker = PerformanceTracker()
# 设置配置变更回调
self.resource_manager._notify_config_change = self._on_config_change
logger.info(f"🧠 SmartHealthDataOptimizer 初始化完成")
logger.info(f" 动态批次大小: {self.batch_size}")
logger.info(f" 动态工作线程: {initial_config.worker_count}")
def _on_config_change(self, old_config: ResourceConfig, new_config: ResourceConfig):
"""响应配置变更"""
# 1. 调整批次大小
if new_config.batch_size != old_config.batch_size:
self.batch_size = new_config.batch_size
# 2. 调整超时时间
if new_config.timeout != old_config.timeout:
self.batch_timeout = new_config.timeout
# 3. 调整队列大小（需要重建队列）
if new_config.queue_size != old_config.queue_size:
self._resize_queue(new_config.queue_size)
# 4. 调整线程池（由DynamicThreadPoolExecutor处理）
self.executor.adjust_workers(new_config.worker_count)
def _batch_processor(self):
"""增强的批处理器 - 包含性能监控"""
batch_data = []
last_flush = time.time()
last_metrics_update = time.time()
while self.running:
try:
# 使用动态超时时间
current_config = self.resource_manager.get_current_config()
timeout = max(0.1, current_config.timeout - (time.time() - last_flush))
item = self.batch_queue.get(timeout=timeout)
batch_data.append(item)
# 性能指标收集
processing_start = time.time()
# 批次处理逻辑
if (len(batch_data) >= current_config.batch_size or
(time.time() - last_flush) >= current_config.timeout):
if batch_data:
self._flush_batch(batch_data)
# 更新性能指标
processing_time = time.time() - processing_start
self.performance_tracker.record_batch(
batch_size=len(batch_data),
processing_time=processing_time
)
batch_data = []
last_flush = time.time()
# 定期更新系统指标
if time.time() - last_metrics_update > 5.0:
self._update_system_metrics()
last_metrics_update = time.time()
except queue.Empty:
if batch_data and (time.time() - last_flush) >= current_config.timeout:
self._flush_batch(batch_data)
batch_data = []
last_flush = time.time()
except Exception as e:
logger.error(f"动态批处理器异常: {e}")
def _update_system_metrics(self):
"""更新系统性能指标"""
try:
queue_depth = self.batch_queue.qsize()
processing_rate = self.performance_tracker.get_current_rate()
avg_response_time = self.performance_tracker.get_avg_response_time()
error_rate = self.performance_tracker.get_error_rate()
self.resource_manager.collect_metrics(
queue_depth=queue_depth,
processing_rate=processing_rate,
avg_response_time=avg_response_time,
error_rate=error_rate
)
except Exception as e:
logger.error(f"更新系统指标失败: {e}")
class DynamicThreadPoolExecutor:
"""动态线程池执行器 - 支持运行时调整工作线程数"""
def __init__(self, max_workers: int, resource_manager: SmartResourceManager):
self.current_workers = max_workers
self.resource_manager = resource_manager
self.executor = ThreadPoolExecutor(max_workers=max_workers)
self._lock = threading.Lock()
def adjust_workers(self, new_worker_count: int):
"""动态调整工作线程数"""
with self._lock:
if new_worker_count == self.current_workers:
return
old_executor = self.executor
# 创建新的线程池
self.executor = ThreadPoolExecutor(max_workers=new_worker_count)
self.current_workers = new_worker_count
# 优雅关闭旧线程池
threading.Thread(
target=self._graceful_shutdown,
args=(old_executor,),
daemon=True
).start()
logger.info(f"🔄 线程池已调整至 {new_worker_count} 个工作线程")
def _graceful_shutdown(self, old_executor):
"""优雅关闭旧线程池"""
try:
old_executor.shutdown(wait=True, timeout=30.0)
except Exception as e:
logger.error(f"关闭旧线程池异常: {e}")
def submit(self, *args, **kwargs):
"""提交任务到当前线程池"""
return self.executor.submit(*args, **kwargs)
def shutdown(self, wait=True):
"""关闭线程池"""
return self.executor.shutdown(wait=wait)
class PerformanceTracker:
"""性能指标跟踪器"""
def __init__(self, window_size: int = 100):
self.window_size = window_size
self.batch_records = []
self.error_count = 0
self.total_count = 0
self._lock = threading.Lock()
def record_batch(self, batch_size: int, processing_time: float):
"""记录批次处理性能"""
with self._lock:
record = {
'timestamp': time.time(),
'batch_size': batch_size,
'processing_time': processing_time,
'rate': batch_size / processing_time if processing_time > 0 else 0
}
self.batch_records.append(record)
self.total_count += batch_size
# 保持窗口大小
if len(self.batch_records) > self.window_size:
self.batch_records = self.batch_records[-self.window_size:]
def record_error(self):
"""记录错误"""
with self._lock:
self.error_count += 1
def get_current_rate(self) -> float:
"""获取当前处理速率"""
with self._lock:
if not self.batch_records:
return 0.0
recent_records = self.batch_records[-10:]
total_processed = sum(r['batch_size'] for r in recent_records)
total_time = sum(r['processing_time'] for r in recent_records)
return total_processed / total_time if total_time > 0 else 0.0
def get_avg_response_time(self) -> float:
"""获取平均响应时间"""
with self._lock:
if not self.batch_records:
return 0.0
recent_records = self.batch_records[-20:]
avg_time = sum(r['processing_time'] for r in recent_records) / len(recent_records)
return avg_time * 1000 # 转换为毫秒
def get_error_rate(self) -> float:
"""获取错误率"""
with self._lock:
if self.total_count == 0:
return 0.0
return (self.error_count / self.total_count) * 100
📊 动态调整策略详解
1. CPU使用率自适应
# 🎯 智能CPU调整策略
if avg_cpu < 50% and queue_ratio > 70%:
# CPU空闲但队列积压 → 增加处理能力
workers *= 1.3
batch_size *= 1.2
elif avg_cpu > 85%:
# CPU过载 → 减少工作线程防止系统崩溃
workers *= 0.8
2. 内存压力响应
# ⚠️ 内存压力自动缓解
if memory_usage > 90%:
batch_size *= 0.7 # 减少批次大小
queue_size *= 0.8 # 减少队列缓存
3. 队列深度管理
# 📈 队列积压智能处理
if queue_ratio > 90%:
queue_size *= 1.5 # 扩大队列容量
if cpu_usage < 70%:
workers *= 1.2 # CPU允许时增加处理线程
🔧 实际应用示例
# 使用示例
smart_optimizer = SmartHealthDataOptimizer()
# 系统会自动监控并调整：
# - 4核心系统：批次100，线程10
# - 8核心系统：批次200，线程20
# - 16核心系统：批次400，线程40
# - 32核心系统：批次800，线程80
# 实时监控和调整
metrics = smart_optimizer.resource_manager.get_metrics_summary()
print(f"当前配置: {metrics['config']}")
print(f"CPU使用率: {metrics['cpu_usage']['current']:.1f}%")
print(f"处理速率: {metrics['processing_rate']['current']:.1f} records/s")
📈 预期效果
静态配置 vs 动态调整
| 场景 | 静态配置 | 动态调整 | 性能提升 |
|---|---|---|---|
| 低负载时段 | 固定资源占用 | 自动降低资源消耗 | 节省30-50%资源 |
| 高峰时段 | 可能瓶颈 | 自动扩展处理能力 | 提升40-80%吞吐量 |
| 内存紧张 | 可能OOM | 自动调整批次大小 | 避免系统崩溃 |
| CPU过载 | 系统卡顿 | 自动减少工作线程 | 保持系统稳定 |
不同硬件配置的自适应效果
# 🖥️ 低端系统 (4核心, 8GB内存)
初始配置: batch_size=80, workers=8, queue_size=800
高负载时: batch_size=60, workers=6, queue_size=600 # 自动降级
低负载时: batch_size=100, workers=10, queue_size=1000 # 适度提升
# 💻 中端系统 (8核心, 16GB内存)
初始配置: batch_size=200, workers=20, queue_size=2000
高负载时: batch_size=240, workers=24, queue_size=2400 # 适度提升
低负载时: batch_size=160, workers=16, queue_size=1600 # 节约资源
# 🖥️ 高端系统 (16核心, 32GB内存)
初始配置: batch_size=480, workers=48, queue_size=4800
高负载时: batch_size=600, workers=60, queue_size=6000 # 大幅提升
低负载时: batch_size=320, workers=32, queue_size=3200 # 节约资源
这个智能动态调整系统将使健康数据处理器能够：
- 自动适应不同硬件环境 - 从4核到64核自动优化
- 实时响应负载变化 - 高峰自动扩容，低谷自动缩容
- 防止系统过载 - 内存/CPU保护机制
- 最大化资源利用 - 在稳定性和性能间找到最佳平衡
这样的系统能够在各种环境下都保持最佳性能！