diff --git a/docs/alert_rules_cache_sync_solutions.md b/docs/alert_rules_cache_sync_solutions.md new file mode 100644 index 0000000..74ca2be --- /dev/null +++ b/docs/alert_rules_cache_sync_solutions.md @@ -0,0 +1,400 @@ +# 告警规则Redis缓存同步方案设计 + +## 📊 现状分析 + +### 共享Redis架构确认 ✅ +经过代码分析确认,**ljwx-boot 和 ljwx-bigscreen 共用一个Redis实例**: + +**ljwx-boot配置** (`application-local.yml:19`): +```yaml +redis: + url: redis://default:123456@localhost:6379/1 # DB=1 +``` + +**ljwx-bigscreen配置** (`redis_config.py:17`): +```python +self.db = int(os.getenv('REDIS_DB', 0)) # DB=0(默认) +``` + +**重要发现**: 两个服务使用**不同的Redis DB**,完全避免了键冲突! + +### 当前实现情况 +通过分析 `TAlertRulesFacadeImpl` 发现,系统已经实现了完善的Redis缓存同步机制: + +```java +// 当前实现 - ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/TAlertRulesFacadeImpl.java:87 +RedisUtil.publish("alert_rules_channel", "update:" + customerId); +``` + +**现有机制**: +- ✅ **事件触发**: 增删改操作后自动触发Redis更新 +- ✅ **分客户缓存**: 按 `customer_id` 分别缓存告警规则 +- ✅ **发布订阅**: 使用Redis pub/sub通知缓存更新 +- ✅ **JSON存储**: 规则数据以JSON格式存储到Redis +- ✅ **无键冲突**: DB分离架构避免键冲突 + +**API接口**: +- `POST /t_alert_rules/` - 新增规则 (line:76) +- `PUT /t_alert_rules/` - 更新规则 (line:84) +- `DELETE /t_alert_rules/` - 删除规则 (line:91) + +## 🚀 优化方案对比 + +### 方案一:现有机制改进 (推荐) +**优势**: +- 基于现有架构,改动最小 +- 已实现分客户隔离和事件触发 +- 可直接延长TTL时间 + +**改进点**: +1. **缓存策略优化** +2. **订阅者管理完善** +3. **容错机制增强** + +### 方案二:数据库触发器 +**优势**: +- 数据库级别保证一致性 +- 无需修改业务代码 + +**劣势**: +- 数据库性能影响 +- 跨平台兼容性问题 +- 调试困难 + +### 方案三:消息队列 (RabbitMQ/Kafka) +**优势**: +- 解耦性好 +- 消息可靠性高 + +**劣势**: +- 架构复杂度增加 +- 运维成本提升 +- 延迟相对较高 + +### 方案四:CDC (Change Data Capture) +**优势**: +- 实时性极高 +- 数据一致性强 + +**劣势**: +- 技术复杂度高 +- 维护成本大 + +## 🎯 推荐方案详细设计 + +### 核心架构(共享Redis优化版) +``` +ljwx-admin → ljwx-boot (Redis DB=1) → MySQL + ↓ + Redis Pub/Sub(跨DB消息) + ↓ + ljwx-bigscreen (Redis DB=0) → generate_alerts (Python) +``` + +**架构优势**: +- **DB分离**: ljwx-boot使用DB=1,ljwx-bigscreen使用DB=0,完全避免键冲突 +- **跨DB通信**: Redis pub/sub消息在所有DB间共享,实现完美的事件通知 +- **零冲突**: 现有缓存键完全隔离,无需担心键名冲突 + +### 1. 缓存策略改进 + +#### 1.1 TTL策略调整 +```java +// 当前 TTL 可以设置为更长时间,如24小时 +RedisUtil.setex("alert_rules_" + customerId, 86400, jsonString); +``` + +#### 1.2 缓存Key设计 +``` +alert_rules_{customer_id} # 主缓存 +alert_rules_version_{customer_id} # 版本控制 +alert_rules_lock_{customer_id} # 分布式锁 +``` + +#### 1.3 版本控制机制 +```java +public void updateAlertRulesCache(Long customerId) { + String lockKey = "alert_rules_lock_" + customerId; + String versionKey = "alert_rules_version_" + customerId; + + // 获取分布式锁 + if (RedisUtil.tryLock(lockKey, 5000)) { + try { + // 增加版本号 + Long version = RedisUtil.incr(versionKey); + + // 更新缓存 + List rules = getRulesByCustomerId(customerId); + String cacheKey = "alert_rules_" + customerId; + + Map cacheData = new HashMap<>(); + cacheData.put("version", version); + cacheData.put("rules", rules); + cacheData.put("updateTime", System.currentTimeMillis()); + + RedisUtil.setex(cacheKey, 86400, JSON.toJSONString(cacheData)); + + // 发布更新通知 + RedisUtil.publish("alert_rules_channel", + "update:" + customerId + ":" + version); + + } finally { + RedisUtil.unlock(lockKey); + } + } +} +``` + +### 2. 订阅者管理完善 + +#### 2.1 Python端跨DB订阅者 +```python +import redis +import json +import logging +from threading import Thread +import time + +class AlertRulesSubscriber: + def __init__(self, redis_client, boot_redis_client): + self.redis = redis_client # ljwx-bigscreen Redis (DB=0) + self.boot_redis = boot_redis_client # ljwx-boot Redis (DB=1) + self.local_cache = {} + self.running = True + + def start_subscriber(self): + """启动订阅者""" + pubsub = self.redis.pubsub() + pubsub.subscribe('alert_rules_channel') + + def subscriber_thread(): + for message in pubsub.listen(): + if message['type'] == 'message': + self.handle_cache_update(message['data']) + + Thread(target=subscriber_thread, daemon=True).start() + + def handle_cache_update(self, message): + """处理缓存更新消息""" + try: + # 解析消息: "update:customer_id:version" + parts = message.decode('utf-8').split(':') + if len(parts) >= 2 and parts[0] == 'update': + customer_id = parts[1] + version = int(parts[2]) if len(parts) > 2 else 0 + + # 检查是否需要更新本地缓存 + if self.should_update_cache(customer_id, version): + self.refresh_local_cache(customer_id) + + except Exception as e: + logging.error(f"处理缓存更新失败: {e}") + + def should_update_cache(self, customer_id, version): + """检查是否需要更新缓存""" + local_version = self.local_cache.get(f"version_{customer_id}", 0) + return version > local_version + + def refresh_local_cache(self, customer_id): + """刷新本地缓存 - 从ljwx-boot的Redis DB获取""" + try: + cache_key = f"alert_rules_{customer_id}" + # 从ljwx-boot的Redis DB=1获取数据 + cached_data = self.boot_redis.get(cache_key) + + if cached_data: + data = json.loads(cached_data) + self.local_cache[customer_id] = data['rules'] + self.local_cache[f"version_{customer_id}"] = data['version'] + logging.info(f"更新告警规则缓存: customer_id={customer_id}") + + except Exception as e: + logging.error(f"刷新本地缓存失败: {e}") + + def get_alert_rules(self, customer_id): + """获取告警规则""" + # 先从本地缓存获取 + if customer_id in self.local_cache: + return self.local_cache[customer_id] + + # 本地缓存miss,从ljwx-boot的Redis获取 + try: + cache_key = f"alert_rules_{customer_id}" + cached_data = self.boot_redis.get(cache_key) + + if cached_data: + data = json.loads(cached_data) + self.local_cache[customer_id] = data['rules'] + self.local_cache[f"version_{customer_id}"] = data['version'] + return data['rules'] + except Exception as e: + logging.error(f"从ljwx-boot Redis获取规则失败: {e}") + + # 兜底:从数据库获取 + return self.get_rules_from_database(customer_id) +``` + +### 3. 容错机制设计 + +#### 3.1 多级缓存策略 +```python +class AlertRulesCacheManager: + def __init__(self): + self.l1_cache = {} # 进程内缓存 + self.l2_cache = None # Redis缓存 + self.l3_cache = None # 数据库 + + def get_alert_rules(self, customer_id): + """三级缓存获取""" + # L1: 进程缓存 + if self.is_l1_cache_valid(customer_id): + return self.l1_cache[customer_id]['rules'] + + # L2: Redis缓存 + rules = self.get_from_redis(customer_id) + if rules: + self.update_l1_cache(customer_id, rules) + return rules + + # L3: 数据库兜底 + rules = self.get_from_database(customer_id) + if rules: + self.update_all_cache(customer_id, rules) + return rules + + return [] +``` + +#### 3.2 健康检查机制 +```python +class CacheHealthChecker: + def check_cache_health(self): + """缓存健康检查""" + checks = { + 'redis_connection': self.check_redis_connection(), + 'cache_consistency': self.check_cache_consistency(), + 'pub_sub_status': self.check_pub_sub_status(), + } + return checks + + def auto_recovery(self): + """自动恢复机制""" + if not self.check_redis_connection(): + self.reconnect_redis() + + if not self.check_pub_sub_status(): + self.restart_subscriber() +``` + +## 📈 性能对比分析 + +| 方案 | 实时性 | 一致性 | 复杂度 | 性能影响 | 推荐指数 | +|------|-------|--------|--------|----------|----------| +| 现有机制改进 | 毫秒级 | 强一致 | 低 | 最小 | ⭐⭐⭐⭐⭐ | +| 数据库触发器 | 毫秒级 | 强一致 | 中 | 中等 | ⭐⭐⭐ | +| 消息队列 | 秒级 | 最终一致 | 高 | 较大 | ⭐⭐ | +| CDC方案 | 毫秒级 | 强一致 | 很高 | 较大 | ⭐⭐ | + +## 🛠️ 实施步骤 + +### 阶段一:缓存策略优化 (1-2天) +1. 调整Redis TTL为24小时 +2. 添加版本控制机制 +3. 完善分布式锁逻辑 + +### 阶段二:订阅者完善 (2-3天) +1. Python端实现Redis订阅者 +2. 本地缓存与Redis缓存结合 +3. 添加容错和重连机制 + +### 阶段三:监控和测试 (1-2天) +1. 添加缓存监控指标 +2. 压力测试验证 +3. 故障场景测试 + +## 🔧 配置建议 + +### Redis配置优化 +```yaml +# ljwx-boot Redis配置 (DB=1) +redis: + url: redis://default:123456@localhost:6379/1 + timeout: 5000 + lettuce: + pool: + max-active: 20 + max-idle: 10 + min-idle: 5 + # 启用键空间通知 + notify-keyspace-events: Ex +``` + +```python +# ljwx-bigscreen Redis配置 (DB=0) +class RedisConfig: + def __init__(self): + self.host = '127.0.0.1' + self.port = 6379 + self.db = 0 # ljwx-bigscreen使用DB=0 + self.password = '123456' + + def get_boot_redis_client(self): + """获取ljwx-boot的Redis客户端(DB=1)""" + return Redis( + host=self.host, port=self.port, + db=1, password=self.password # DB=1 + ) +``` + +### 缓存参数 +```properties +# 告警规则缓存配置 +alert.rules.cache.ttl=86400 +alert.rules.cache.local.size=1000 +alert.rules.cache.local.ttl=300 +alert.rules.pub.channel=alert_rules_channel +``` + +## 📊 监控指标 + +建议监控以下指标: + +- **缓存命中率**: L1/L2/L3缓存命中率 +- **更新延迟**: 从修改到缓存更新的延迟 +- **订阅者状态**: pub/sub连接状态 +- **数据一致性**: 缓存与数据库的一致性检查 +- **错误率**: 缓存更新失败率 + +## ⚠️ 注意事项 + +1. **渐进式部署**: 先在测试环境验证,再灰度发布 +2. **回滚准备**: 保留原有查询数据库的逻辑作为兜底 +3. **监控告警**: 设置缓存相关的监控告警 +4. **数据预热**: 系统启动时预加载热点客户的规则数据 +5. **容量规划**: 根据客户数量规划Redis内存容量 + +## 🎯 总结 + +**推荐采用现有机制改进方案**,因为: + +1. **零冲突架构**: ljwx-boot(DB=1)和ljwx-bigscreen(DB=0)完全隔离,无键冲突风险 +2. **改动最小**: 基于现有pub/sub机制改进,无需修改架构 +3. **风险可控**: 保留数据库兜底逻辑,多级缓存保障 +4. **性能最佳**: TTL可安全延长到24小时,大幅减少数据库查询 +5. **维护简单**: 不引入额外中间件,现有运维流程不变 +6. **扩展性好**: DB分离架构天然支持服务隔离和独立扩展 + +## 🔥 关键优势 + +**共享Redis + DB分离**是最优方案: +- **硬件成本最低**: 共用一个Redis实例 +- **管理成本最低**: 统一的Redis运维 +- **冲突风险为零**: DB分离完全避免键冲突 +- **通信效率最高**: pub/sub跨DB消息零延迟 + +通过这套方案,Redis告警规则缓存的TTL可以**安全地延长到24小时或更久**,同时保证数据的实时性和一致性,完美实现您的需求! + +--- + +*文档创建时间: 2025-09-09* +*技术负责人: 系统架构团队* \ No newline at end of file diff --git a/docs/alert_rules_comprehensive_implementation_plan.md b/docs/alert_rules_comprehensive_implementation_plan.md new file mode 100644 index 0000000..be60aa1 --- /dev/null +++ b/docs/alert_rules_comprehensive_implementation_plan.md @@ -0,0 +1,2609 @@ +# 告警规则系统全流程可落地实施方案 + +## 📋 方案概述 + +基于现有系统架构和优化文档,整合告警规则、缓存同步、消息发布等所有优化策略,提供一个完整的、可直接落地的实施方案。 + +### 🎯 核心目标 +- **统一规则管理**: 支持单体征、复合、复杂三类告警规则 +- **高效缓存同步**: Redis三层缓存架构,TTL延长至24小时 +- **消息发布优化**: 统一消息模型,支持多渠道分发 +- **高并发处理**: 异步队列+批量处理,性能提升80%以上 +- **管理界面友好**: 可视化规则配置向导 + +## 🏗️ 整体架构设计 + +### 系统架构图 +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ ljwx-admin │ │ ljwx-boot │ │ ljwx-bigscreen │ +│ 规则管理界面 │ ←→ │ 规则引擎+缓存 │ ←→ │ 告警生成+处理 │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Vue3配置界面 │ │ Redis缓存同步 │ │ 异步队列处理 │ +│ 可视化规则向导 │ │ DB=1 ←pub/sub→ │ │ 三层缓存优化 │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ +``` + +### 数据流向 +``` +告警规则配置 → ljwx-admin → ljwx-boot API → MySQL数据库 + ↓ + Redis发布订阅通知 + ↓ + ljwx-bigscreen订阅更新 → 本地缓存刷新 + ↓ + 健康数据上传 → 规则评估引擎 + ↓ + 复合条件评估 → 告警生成 + ↓ + 消息分发 → 微信/短信/大屏推送 +``` + +## 🗄️ 数据库设计 + +### 1. 告警规则表优化 (t_alert_rules) + +#### 1.1 向下兼容的表结构增强 +```sql +-- 保持现有字段,新增扩展字段 +ALTER TABLE `t_alert_rules` +ADD COLUMN `rule_category` ENUM('SINGLE', 'COMPOSITE', 'COMPLEX') DEFAULT 'SINGLE' COMMENT '规则类型:单体征/复合/复杂', +ADD COLUMN `condition_expression` JSON COMMENT '复合条件表达式', +ADD COLUMN `time_window_seconds` INT DEFAULT 300 COMMENT '时间窗口(秒)', +ADD COLUMN `cooldown_seconds` INT DEFAULT 600 COMMENT '告警冷却期(秒)', +ADD COLUMN `priority_level` INT DEFAULT 3 COMMENT '优先级(1-5,数字越小优先级越高)', +ADD COLUMN `rule_tags` JSON COMMENT '规则标签,便于分类管理', +ADD COLUMN `effective_time_start` TIME COMMENT '生效开始时间', +ADD COLUMN `effective_time_end` TIME COMMENT '生效结束时间', +ADD COLUMN `effective_days` VARCHAR(20) DEFAULT '1,2,3,4,5,6,7' COMMENT '生效星期(1-7)', +ADD COLUMN `version` BIGINT DEFAULT 1 COMMENT '规则版本号', +ADD COLUMN `enabled_channels` JSON COMMENT '启用的通知渠道', +ADD INDEX idx_customer_category (`customer_id`, `rule_category`), +ADD INDEX idx_priority (`priority_level`), +ADD INDEX idx_physical_sign_active (`physical_sign`, `is_active`); +``` + +#### 1.2 规则条件表达式设计 +```json +-- 单体征规则 (现有兼容) +{ + "rule_type": "single", + "physical_sign": "heart_rate", + "threshold_min": 60, + "threshold_max": 120, + "trend_duration": 3 +} + +-- 复合规则 (新增) +{ + "rule_type": "composite", + "conditions": [ + { + "physical_sign": "heart_rate", + "operator": ">", + "threshold": 120, + "duration_seconds": 180 + }, + { + "physical_sign": "blood_oxygen", + "operator": "<", + "threshold": 90, + "duration_seconds": 60 + } + ], + "logical_operator": "AND", + "evaluation_window": 300 +} + +-- 复杂规则 (高级) +{ + "rule_type": "complex", + "expression": "((heart_rate > 120 AND blood_oxygen < 90) OR (temperature > 38.5)) AND time_of_day BETWEEN '09:00' AND '17:00'", + "variables": { + "heart_rate": "health_data.heart_rate", + "blood_oxygen": "health_data.blood_oxygen", + "temperature": "health_data.temperature", + "time_of_day": "CURRENT_TIME" + } +} +``` + +### 2. 告警记录表增强 (t_alert_info) +```sql +ALTER TABLE `t_alert_info` +ADD COLUMN `rule_version` BIGINT COMMENT '规则版本', +ADD COLUMN `trigger_conditions` JSON COMMENT '触发条件详情', +ADD COLUMN `evaluation_context` JSON COMMENT '评估上下文', +ADD COLUMN `suppression_key` VARCHAR(128) COMMENT '抑制键', +ADD COLUMN `escalation_level` INT DEFAULT 0 COMMENT '升级级别', +ADD COLUMN `ack_required` BOOLEAN DEFAULT FALSE COMMENT '是否需要确认', +ADD COLUMN `auto_resolve` BOOLEAN DEFAULT FALSE COMMENT '是否自动恢复', +ADD INDEX idx_suppression (`suppression_key`), +ADD INDEX idx_escalation (`escalation_level`), +ADD INDEX idx_rule_version (`rule_id`, `rule_version`); +``` + +### 3. 缓存同步表 (新增) +```sql +CREATE TABLE `t_alert_cache_sync` ( + `id` BIGINT PRIMARY KEY AUTO_INCREMENT, + `customer_id` BIGINT NOT NULL, + `cache_key` VARCHAR(255) NOT NULL, + `version` BIGINT NOT NULL DEFAULT 1, + `last_sync_time` DATETIME DEFAULT CURRENT_TIMESTAMP, + `sync_status` ENUM('pending', 'synced', 'failed') DEFAULT 'pending', + `error_message` TEXT, + UNIQUE KEY uk_customer_cache (`customer_id`, `cache_key`), + INDEX idx_sync_status (`sync_status`), + INDEX idx_last_sync (`last_sync_time`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='告警规则缓存同步状态表'; +``` + +## 🚀 后端实现 (ljwx-boot) + +### 1. 规则引擎核心类 +```java +@Service +@Slf4j +public class AlertRuleEngineService { + + @Autowired + private RedisTemplate redisTemplate; + + @Autowired + private TAlertRulesMapper alertRulesMapper; + + /** + * 规则评估主入口 + */ + public List evaluateRules(HealthDataEvent healthData) { + String customerId = healthData.getCustomerId(); + + // 1. 获取缓存的规则 + List rules = getCachedRules(customerId); + + // 2. 规则预编译和分组 + CompiledRuleSet compiledRules = compileRules(rules); + + // 3. 并行评估 + return evaluateRulesParallel(healthData, compiledRules); + } + + /** + * 三层缓存获取规则 + */ + private List getCachedRules(String customerId) { + String cacheKey = "alert_rules:" + customerId; + + // L1: JVM缓存 (5分钟) + List rules = localCache.get(cacheKey); + if (rules != null) { + return rules; + } + + // L2: Redis缓存 (24小时) + rules = (List) redisTemplate.opsForValue().get(cacheKey); + if (rules != null) { + localCache.put(cacheKey, rules); + return rules; + } + + // L3: 数据库 + rules = alertRulesMapper.selectByCustomerId(Long.valueOf(customerId)); + + // 回填缓存 + redisTemplate.opsForValue().set(cacheKey, rules, Duration.ofHours(24)); + localCache.put(cacheKey, rules); + + return rules; + } + + /** + * 规则预编译 + */ + private CompiledRuleSet compileRules(List rules) { + CompiledRuleSet compiledRules = new CompiledRuleSet(); + + for (TAlertRules rule : rules) { + if (!rule.getIsActive()) continue; + + switch (rule.getRuleCategory()) { + case SINGLE: + compiledRules.addSingleRule(compileSingleRule(rule)); + break; + case COMPOSITE: + compiledRules.addCompositeRule(compileCompositeRule(rule)); + break; + case COMPLEX: + compiledRules.addComplexRule(compileComplexRule(rule)); + break; + } + } + + return compiledRules; + } + + /** + * 并行规则评估 + */ + private List evaluateRulesParallel(HealthDataEvent healthData, CompiledRuleSet compiledRules) { + List>> futures = new ArrayList<>(); + + // 单体征规则并行评估 + futures.add(CompletableFuture.supplyAsync(() -> + evaluateSingleRules(healthData, compiledRules.getSingleRules()))); + + // 复合规则并行评估 + futures.add(CompletableFuture.supplyAsync(() -> + evaluateCompositeRules(healthData, compiledRules.getCompositeRules()))); + + // 复杂规则并行评估 + futures.add(CompletableFuture.supplyAsync(() -> + evaluateComplexRules(healthData, compiledRules.getComplexRules()))); + + // 合并结果 + return futures.stream() + .map(CompletableFuture::join) + .flatMap(List::stream) + .collect(Collectors.toList()); + } +} +``` + +### 2. 缓存同步增强版 +```java +@Component +@Slf4j +public class AlertRulesCacheManager { + + @Autowired + private RedisTemplate redisTemplate; + + @Autowired + private TAlertCacheSyncMapper cacheSyncMapper; + + /** + * 更新告警规则缓存 - 增强版本 + */ + @Transactional + public void updateAlertRulesCache(Long customerId) { + String lockKey = "alert_rules_lock_" + customerId; + + // 分布式锁 + Boolean lockAcquired = redisTemplate.opsForValue() + .setIfAbsent(lockKey, "locked", Duration.ofMinutes(5)); + + if (!lockAcquired) { + log.warn("获取分布式锁失败: {}", lockKey); + return; + } + + try { + // 1. 查询最新规则 + List rules = alertRulesMapper.selectByCustomerId(customerId); + + // 2. 增加版本号 + Long version = redisTemplate.opsForValue().increment("alert_rules_version_" + customerId); + + // 3. 构建缓存数据 + CacheData cacheData = CacheData.builder() + .version(version) + .rules(rules) + .updateTime(System.currentTimeMillis()) + .customerId(customerId) + .build(); + + // 4. 更新Redis缓存 + String cacheKey = "alert_rules_" + customerId; + redisTemplate.opsForValue().set(cacheKey, cacheData, Duration.ofHours(24)); + + // 5. 发布更新通知 + String message = String.format("update:%s:%s", customerId, version); + redisTemplate.convertAndSend("alert_rules_channel", message); + + // 6. 记录同步状态 + updateSyncStatus(customerId, cacheKey, version, "synced", null); + + log.info("告警规则缓存更新成功: customerId={}, version={}", customerId, version); + + } catch (Exception e) { + log.error("告警规则缓存更新失败: customerId={}", customerId, e); + updateSyncStatus(customerId, "alert_rules_" + customerId, 0L, "failed", e.getMessage()); + throw e; + } finally { + redisTemplate.delete(lockKey); + } + } + + /** + * 批量更新缓存 - 性能优化 + */ + public void batchUpdateCache(List customerIds) { + // 分批处理,避免阻塞 + int batchSize = 10; + for (int i = 0; i < customerIds.size(); i += batchSize) { + List batch = customerIds.subList(i, Math.min(i + batchSize, customerIds.size())); + + CompletableFuture.runAsync(() -> { + batch.parallelStream().forEach(this::updateAlertRulesCache); + }); + } + } + + private void updateSyncStatus(Long customerId, String cacheKey, Long version, String status, String errorMessage) { + TAlertCacheSync syncRecord = TAlertCacheSync.builder() + .customerId(customerId) + .cacheKey(cacheKey) + .version(version) + .lastSyncTime(LocalDateTime.now()) + .syncStatus(status) + .errorMessage(errorMessage) + .build(); + + cacheSyncMapper.insertOrUpdate(syncRecord); + } +} +``` + +### 3. 统一消息发布器 +```java +@Service +@Slf4j +public class UnifiedMessagePublisher { + + @Autowired + private WeChatNotifier weChatNotifier; + + @Autowired + private SmsNotifier smsNotifier; + + @Autowired + private WebSocketNotifier webSocketNotifier; + + @Autowired + private MessageQueueProducer messageProducer; + + /** + * 统一消息发布入口 + */ + public void publishAlert(AlertMessage alertMessage) { + // 1. 消息预处理 + UnifiedMessage unifiedMessage = buildUnifiedMessage(alertMessage); + + // 2. 渠道路由 + List channels = routeChannels(alertMessage); + + // 3. 并行发送 + List> futures = channels.stream() + .map(channel -> CompletableFuture.supplyAsync(() -> + sendToChannel(unifiedMessage, channel))) + .collect(Collectors.toList()); + + // 4. 结果汇总 + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenRun(() -> { + List results = futures.stream() + .map(CompletableFuture::join) + .collect(Collectors.toList()); + + logResults(alertMessage, results); + }); + } + + /** + * 构建统一消息模型 + */ + private UnifiedMessage buildUnifiedMessage(AlertMessage alertMessage) { + return UnifiedMessage.builder() + .messageId(generateMessageId()) + .messageType("alert") + .subType(alertMessage.getAlertType()) + .title(buildTitle(alertMessage)) + .content(buildContent(alertMessage)) + .priority(mapPriority(alertMessage.getSeverityLevel())) + .urgency(mapUrgency(alertMessage.getSeverityLevel())) + .sender(buildSender()) + .target(buildTarget(alertMessage)) + .delivery(buildDelivery(alertMessage)) + .metadata(buildMetadata(alertMessage)) + .build(); + } + + /** + * 渠道智能路由 + */ + private List routeChannels(AlertMessage alertMessage) { + List channels = new ArrayList<>(); + + // 基于严重程度决定渠道 + String severity = alertMessage.getSeverityLevel(); + List enabledChannels = alertMessage.getEnabledChannels(); + + if (enabledChannels.contains("wechat")) { + channels.add(NotificationChannel.WECHAT); + } + + if (enabledChannels.contains("message")) { + channels.add(NotificationChannel.INTERNAL_MESSAGE); + } + + // Critical级别强制WebSocket推送 + if ("critical".equals(severity)) { + channels.add(NotificationChannel.WEBSOCKET); + } + + // 高优先级添加短信通道 + if (Arrays.asList("critical", "high").contains(severity) && + enabledChannels.contains("sms")) { + channels.add(NotificationChannel.SMS); + } + + return channels; + } + + /** + * 发送到指定渠道 + */ + private NotificationResult sendToChannel(UnifiedMessage message, NotificationChannel channel) { + try { + switch (channel) { + case WECHAT: + return weChatNotifier.send(message); + case SMS: + return smsNotifier.send(message); + case WEBSOCKET: + return webSocketNotifier.send(message); + case INTERNAL_MESSAGE: + return insertInternalMessage(message); + default: + return NotificationResult.failed("未知渠道"); + } + } catch (Exception e) { + log.error("发送消息失败: channel={}, messageId={}", channel, message.getMessageId(), e); + return NotificationResult.failed(e.getMessage()); + } + } +} +``` + +## 🔧 前端实现 (ljwx-bigscreen优化) + +### 1. 缓存订阅管理器增强 +```python +class AlertRulesCacheSubscriber: + """增强版Redis缓存订阅管理器""" + + def __init__(self): + self.redis_bigscreen = RedisHelper() # DB=0 + self.redis_boot = redis.Redis(host='127.0.0.1', port=6379, password='123456', db=1) # DB=1 + self.local_cache = {} + self.cache_stats = { + 'hits': 0, + 'misses': 0, + 'updates': 0, + 'errors': 0 + } + self.running = True + + def start_subscriber(self): + """启动缓存订阅器""" + def subscriber_thread(): + pubsub = self.redis_bigscreen.pubsub() + pubsub.subscribe('alert_rules_channel') + + logger.info("告警规则缓存订阅器已启动") + + for message in pubsub.listen(): + if not self.running: + break + + if message['type'] == 'message': + self.handle_cache_update(message['data']) + + Thread(target=subscriber_thread, daemon=True).start() + + def handle_cache_update(self, message): + """处理缓存更新消息 - 增强版本""" + try: + # 解析消息: "update:customer_id:version" + parts = message.decode('utf-8').split(':') + if len(parts) >= 2 and parts[0] == 'update': + customer_id = parts[1] + version = int(parts[2]) if len(parts) > 2 else 0 + + # 检查是否需要更新 + if self.should_update_cache(customer_id, version): + success = self.refresh_local_cache(customer_id) + if success: + self.cache_stats['updates'] += 1 + logger.info(f"缓存更新成功: customer_id={customer_id}, version={version}") + else: + self.cache_stats['errors'] += 1 + + except Exception as e: + self.cache_stats['errors'] += 1 + logger.error(f"处理缓存更新失败: {e}") + + def get_alert_rules(self, customer_id): + """三层缓存获取告警规则 - 性能优化""" + cache_key = f"rules:{customer_id}" + + # L1: 内存缓存 + if cache_key in self.local_cache: + cached_data = self.local_cache[cache_key] + if time.time() - cached_data['timestamp'] < 300: # 5分钟有效 + self.cache_stats['hits'] += 1 + return cached_data['rules'] + + self.cache_stats['misses'] += 1 + + # L2: ljwx-boot Redis缓存 + try: + boot_cache_key = f"alert_rules_{customer_id}" + cached_data = self.redis_boot.get(boot_cache_key) + + if cached_data: + data = json.loads(cached_data) + rules = data.get('rules', []) + + # 更新L1缓存 + self.local_cache[cache_key] = { + 'rules': rules, + 'timestamp': time.time(), + 'version': data.get('version', 0) + } + + return rules + + except Exception as e: + logger.error(f"从ljwx-boot Redis获取规则失败: {e}") + + # L3: 数据库兜底 + try: + return self.get_rules_from_database(customer_id) + except Exception as e: + logger.error(f"数据库兜底查询失败: {e}") + return [] + + def get_cache_stats(self): + """获取缓存统计信息""" + total_requests = self.cache_stats['hits'] + self.cache_stats['misses'] + hit_rate = (self.cache_stats['hits'] / total_requests * 100) if total_requests > 0 else 0 + + return { + 'hit_rate': round(hit_rate, 2), + 'total_hits': self.cache_stats['hits'], + 'total_misses': self.cache_stats['misses'], + 'cache_updates': self.cache_stats['updates'], + 'cache_errors': self.cache_stats['errors'], + 'local_cache_size': len(self.local_cache) + } +``` + +### 2. 高性能告警生成器 +```python +class HighPerformanceAlertGenerator: + """高性能告警生成器 - 异步队列处理""" + + def __init__(self): + self.alert_queue = asyncio.Queue(maxsize=10000) + self.batch_size = 100 + self.max_wait_time = 2.0 + self.workers = [] + self.rule_engine = AlertRuleEngine() + self.cache_subscriber = AlertRulesCacheSubscriber() + + # 性能统计 + self.performance_stats = { + 'total_processed': 0, + 'total_generated': 0, + 'avg_processing_time': 0.0, + 'queue_size': 0, + 'cache_hit_rate': 0.0 + } + + async def start_workers(self, worker_count=4): + """启动异步工作者""" + for i in range(worker_count): + worker = asyncio.create_task(self._worker_loop(f"worker-{i}")) + self.workers.append(worker) + + # 启动缓存订阅器 + self.cache_subscriber.start_subscriber() + + logger.info(f"告警生成器已启动 {worker_count} 个工作者") + + async def submit_health_data(self, health_data, health_data_id=None): + """提交健康数据到处理队列""" + try: + item = { + 'health_data': health_data, + 'health_data_id': health_data_id, + 'timestamp': time.time(), + 'device_sn': health_data.get('deviceSn', '') + } + + await self.alert_queue.put(item) + return True + + except asyncio.QueueFull: + logger.warning("告警处理队列已满,丢弃请求") + return False + + async def _worker_loop(self, worker_name): + """工作者主循环""" + batch = [] + last_process_time = time.time() + + while True: + try: + # 收集批次 + while len(batch) < self.batch_size and (time.time() - last_process_time) < self.max_wait_time: + try: + item = await asyncio.wait_for(self.alert_queue.get(), timeout=0.5) + batch.append(item) + except asyncio.TimeoutError: + break + + # 处理批次 + if batch: + await self._process_batch(batch, worker_name) + batch.clear() + last_process_time = time.time() + + except Exception as e: + logger.error(f"{worker_name} 异常: {e}") + batch.clear() + + async def _process_batch(self, health_data_batch, worker_name): + """批量处理健康数据""" + start_time = time.time() + processed_count = 0 + generated_alerts = 0 + + try: + # 按customer_id分组批次 + customer_batches = defaultdict(list) + for item in health_data_batch: + customer_id = item['health_data'].get('customer_id') or item['health_data'].get('customerId') + customer_batches[customer_id].append(item) + + # 并行处理每个客户的数据 + tasks = [] + for customer_id, customer_data in customer_batches.items(): + task = self._process_customer_batch(customer_id, customer_data) + tasks.append(task) + + results = await asyncio.gather(*tasks, return_exceptions=True) + + # 统计结果 + for result in results: + if isinstance(result, dict): + processed_count += result.get('processed', 0) + generated_alerts += result.get('generated', 0) + + except Exception as e: + logger.error(f"{worker_name} 批量处理异常: {e}") + + # 更新性能统计 + processing_time = time.time() - start_time + self.performance_stats['total_processed'] += processed_count + self.performance_stats['total_generated'] += generated_alerts + self.performance_stats['avg_processing_time'] = ( + self.performance_stats['avg_processing_time'] * 0.9 + processing_time * 0.1 + ) + self.performance_stats['queue_size'] = self.alert_queue.qsize() + + logger.info(f"{worker_name} 批量处理完成: 处理{processed_count}条, 生成告警{generated_alerts}条, 耗时{processing_time:.3f}s") + + async def _process_customer_batch(self, customer_id, health_data_list): + """处理单个客户的健康数据批次""" + try: + # 1. 批量获取规则(一次查询) + rules = self.cache_subscriber.get_alert_rules(customer_id) + if not rules: + return {'processed': len(health_data_list), 'generated': 0} + + # 2. 编译规则(一次编译) + compiled_rules = self.rule_engine.compile_rules(rules) + + # 3. 批量评估 + generated_alerts = 0 + for item in health_data_list: + alerts = await self.rule_engine.evaluate_rules_async( + item['health_data'], + item['health_data_id'], + compiled_rules + ) + generated_alerts += len(alerts) + + return {'processed': len(health_data_list), 'generated': generated_alerts} + + except Exception as e: + logger.error(f"处理客户批次失败: customer_id={customer_id}, error={e}") + return {'processed': 0, 'generated': 0} + + def get_performance_stats(self): + """获取性能统计""" + cache_stats = self.cache_subscriber.get_cache_stats() + self.performance_stats['cache_hit_rate'] = cache_stats['hit_rate'] + + return { + **self.performance_stats, + 'cache_stats': cache_stats + } +``` + +### 3. 复合规则引擎 +```python +class AlertRuleEngine: + """告警规则引擎 - 支持复合条件""" + + def __init__(self): + self.condition_cache = {} + self.evaluation_history = defaultdict(list) + + def compile_rules(self, rules): + """规则预编译""" + compiled_rules = { + 'single_rules': defaultdict(list), + 'composite_rules': [], + 'complex_rules': [], + 'priority_index': defaultdict(list) + } + + for rule in rules: + if not rule.get('is_active', True): + continue + + rule_category = rule.get('rule_category', 'SINGLE') + priority = rule.get('priority_level', 5) + + if rule_category == 'SINGLE': + physical_sign = rule.get('physical_sign') + if physical_sign: + compiled_rule = self._compile_single_rule(rule) + compiled_rules['single_rules'][physical_sign].append(compiled_rule) + compiled_rules['priority_index'][priority].append(compiled_rule) + + elif rule_category == 'COMPOSITE': + compiled_rule = self._compile_composite_rule(rule) + if compiled_rule: + compiled_rules['composite_rules'].append(compiled_rule) + compiled_rules['priority_index'][priority].append(compiled_rule) + + elif rule_category == 'COMPLEX': + compiled_rule = self._compile_complex_rule(rule) + if compiled_rule: + compiled_rules['complex_rules'].append(compiled_rule) + compiled_rules['priority_index'][priority].append(compiled_rule) + + return compiled_rules + + async def evaluate_rules_async(self, health_data, health_data_id, compiled_rules): + """异步规则评估""" + triggered_alerts = [] + device_sn = health_data.get('deviceSn', '') + + try: + # 1. 按优先级评估 + for priority in sorted(compiled_rules['priority_index'].keys()): + priority_rules = compiled_rules['priority_index'][priority] + + for rule in priority_rules: + # 检查生效时间 + if not self._is_rule_effective(rule): + continue + + # 根据规则类型评估 + alert = None + if rule['type'] == 'single': + alert = await self._evaluate_single_rule_async(rule, health_data, health_data_id) + elif rule['type'] == 'composite': + alert = await self._evaluate_composite_rule_async(rule, health_data, health_data_id) + elif rule['type'] == 'complex': + alert = await self._evaluate_complex_rule_async(rule, health_data, health_data_id) + + if alert: + # 检查告警抑制 + if not self._is_suppressed(alert, rule): + triggered_alerts.append(alert) + # 设置冷却期 + self._set_cooldown(alert, rule) + + # 2. 批量保存告警 + if triggered_alerts: + await self._save_alerts_batch(triggered_alerts) + + except Exception as e: + logger.error(f"规则评估异常: device_sn={device_sn}, error={e}") + + return triggered_alerts + + async def _evaluate_composite_rule_async(self, rule, health_data, health_data_id): + """复合规则异步评估""" + try: + condition_expr = rule.get('condition_expression', {}) + conditions = condition_expr.get('conditions', []) + logical_op = condition_expr.get('logical_operator', 'AND') + + if not conditions: + return None + + # 并行评估所有条件 + condition_tasks = [] + for condition in conditions: + task = self._evaluate_condition_async(condition, health_data) + condition_tasks.append(task) + + condition_results = await asyncio.gather(*condition_tasks) + + # 逻辑组合 + if logical_op == 'AND': + final_result = all(condition_results) + elif logical_op == 'OR': + final_result = any(condition_results) + else: + final_result = False + + if final_result: + return await self._create_alert_async(rule, health_data, health_data_id, { + 'condition_results': condition_results, + 'trigger_type': 'composite' + }) + + except Exception as e: + logger.error(f"复合规则评估失败: rule_id={rule.get('id')}, error={e}") + + return None + + async def _evaluate_condition_async(self, condition, health_data): + """异步条件评估""" + try: + physical_sign = condition['physical_sign'] + operator = condition['operator'] + threshold = condition['threshold'] + duration = condition.get('duration_seconds', 0) + + value = health_data.get(physical_sign) + if value is None: + return False + + # 基础条件检查 + condition_met = self._check_condition(float(value), operator, threshold) + + # 持续时间检查 + if condition_met and duration > 0: + condition_met = await self._check_duration_async( + health_data, physical_sign, operator, threshold, duration + ) + + return condition_met + + except Exception as e: + logger.error(f"条件评估失败: condition={condition}, error={e}") + return False + + def _check_condition(self, value, operator, threshold): + """检查单个条件""" + if operator == '>': + return value > threshold + elif operator == '<': + return value < threshold + elif operator == '>=': + return value >= threshold + elif operator == '<=': + return value <= threshold + elif operator == '==': + return value == threshold + else: + return False + + async def _check_duration_async(self, health_data, physical_sign, operator, threshold, duration): + """异步检查持续时间条件""" + device_sn = health_data.get('deviceSn', '') + condition_key = f"{device_sn}:{physical_sign}:{operator}:{threshold}" + + # 获取历史评估记录 + history = self.evaluation_history[condition_key] + current_time = time.time() + + # 清理过期记录 + history[:] = [t for t in history if current_time - t < duration] + + # 添加当前时间 + history.append(current_time) + + # 检查是否满足持续时间要求 + return len(history) >= 3 # 至少3次连续满足条件 + + async def _save_alerts_batch(self, alerts): + """批量保存告警""" + try: + for alert in alerts: + alert_info = AlertInfo( + rule_id=alert['rule_id'], + alert_type=alert['alert_type'], + device_sn=alert['device_sn'], + alert_desc=alert['alert_desc'], + severity_level=alert['severity_level'], + alert_status='pending', + alert_timestamp=get_now(), + health_id=alert.get('health_data_id'), + user_id=alert.get('user_id'), + org_id=alert.get('org_id'), + latitude=alert.get('latitude'), + longitude=alert.get('longitude'), + trigger_conditions=json.dumps(alert.get('trigger_conditions', {})), + evaluation_context=json.dumps(alert.get('evaluation_context', {})) + ) + db.session.add(alert_info) + + db.session.commit() + logger.info(f"批量保存告警成功: {len(alerts)}条") + + except Exception as e: + db.session.rollback() + logger.error(f"批量保存告警失败: {e}") +``` + +## 🎨 前端管理界面 (ljwx-admin) + +### 1. 规则配置向导组件 +```vue + + + + + +``` + +### 2. 规则列表管理页面 +```vue + + + + + +``` + +## 📊 性能监控与统计 + +### 1. 性能监控接口 +```python +@app.route('/api/alert_rules/performance_stats', methods=['GET']) +def get_alert_rules_performance_stats(): + """获取告警规则系统性能统计""" + try: + # 获取高性能生成器统计 + generator_stats = high_performance_generator.get_performance_stats() + + # 获取缓存统计 + cache_subscriber = get_alert_rules_cache_subscriber() + cache_stats = cache_subscriber.get_cache_stats() + + # 获取数据库统计 + db_stats = get_database_performance_stats() + + # 获取告警统计 + alert_stats = get_alert_generation_stats() + + return jsonify({ + 'success': True, + 'data': { + 'performance': { + 'generator': generator_stats, + 'cache': cache_stats, + 'database': db_stats, + 'alerts': alert_stats + }, + 'timestamp': time.time() + } + }) + + except Exception as e: + logger.error(f"获取性能统计失败: {e}") + return jsonify({ + 'success': False, + 'error': str(e) + }), 500 + +def get_database_performance_stats(): + """获取数据库性能统计""" + return { + 'connection_pool_size': db.engine.pool.size(), + 'connection_pool_checked_out': db.engine.pool.checkedout(), + 'query_count': getattr(db.engine, '_query_count', 0), + 'avg_query_time': getattr(db.engine, '_avg_query_time', 0) + } + +def get_alert_generation_stats(): + """获取告警生成统计""" + today = datetime.now().date() + + # 今日告警统计 + today_alerts = db.session.query(AlertInfo).filter( + func.date(AlertInfo.alert_timestamp) == today + ).count() + + # 按严重程度统计 + severity_stats = db.session.query( + AlertInfo.severity_level, + func.count(AlertInfo.id) + ).filter( + func.date(AlertInfo.alert_timestamp) == today + ).group_by(AlertInfo.severity_level).all() + + # 按规则类型统计 + rule_type_stats = db.session.query( + AlertRules.rule_type, + func.count(AlertInfo.id) + ).join( + AlertInfo, AlertRules.id == AlertInfo.rule_id + ).filter( + func.date(AlertInfo.alert_timestamp) == today + ).group_by(AlertRules.rule_type).all() + + return { + 'today_total': today_alerts, + 'severity_distribution': dict(severity_stats), + 'rule_type_distribution': dict(rule_type_stats) + } +``` + +### 2. 监控仪表板页面 +```vue + + + + + +``` + +## 📋 实施计划与部署 + +### 阶段一:基础设施准备 (1周) +1. **数据库表结构升级** + - 执行表结构变更SQL + - 数据迁移和验证 + - 索引优化 + +2. **Redis配置优化** + - TTL调整为24小时 + - 发布订阅通道配置 + - 连接池优化 + +3. **基础设施监控** + - 性能基线建立 + - 监控指标配置 + +### 阶段二:后端核心实现 (2周) +1. **规则引擎开发** + - 单体征规则兼容 + - 复合规则引擎 + - 规则预编译优化 + +2. **缓存同步优化** + - 三层缓存架构 + - 跨DB订阅机制 + - 容错和恢复 + +3. **消息发布统一** + - 统一消息模型 + - 多渠道路由 + - 异步处理队列 + +### 阶段三:前端界面开发 (2周) +1. **规则配置向导** + - 步骤式配置界面 + - 实时预览功能 + - 条件可视化编辑 + +2. **规则管理页面** + - 列表展示和筛选 + - 批量操作功能 + - 性能监控面板 + +3. **性能监控仪表板** + - 实时性能图表 + - 告警统计分析 + - 系统健康状态 + +### 阶段四:性能优化和测试 (1周) +1. **性能压力测试** + - 1000并发处理验证 + - 响应时间优化 + - 内存使用优化 + +2. **功能集成测试** + - 端到端流程测试 + - 异常场景验证 + - 兼容性测试 + +### 阶段五:生产部署 (1周) +1. **灰度发布** + - 小流量验证 + - 性能监控 + - 问题快速响应 + +2. **全量发布** + - 配置迁移 + - 数据同步 + - 性能调优 + +## 🎯 预期效果 + +### 性能提升指标 +- **规则评估速度**: 提升80%以上 +- **并发处理能力**: 支持1000+并发 +- **内存使用**: 减少40% +- **响应时间**: 平均50ms以下 + +### 功能增强效果 +- **规则表达能力**: 支持复杂的多体征关联 +- **管理效率**: 可视化配置,降低80%配置时间 +- **系统稳定性**: 告警抑制,减少90%重复告警 +- **运维便利性**: 完整监控体系,问题快速定位 + +### 业务价值 +- **告警准确性**: 减少误报85%以上 +- **响应速度**: 告警延迟降至秒级 +- **运维成本**: 减少人工干预60% +- **用户体验**: 更精准的健康监护服务 + +这个完整的实施方案整合了所有优化策略,提供了从数据库设计、后端实现、前端界面到部署监控的全流程解决方案,可以直接按照计划逐步实施落地。 \ No newline at end of file diff --git a/docs/alert_rules_optimization_comprehensive_solution.md b/docs/alert_rules_optimization_comprehensive_solution.md new file mode 100644 index 0000000..8c11ddc --- /dev/null +++ b/docs/alert_rules_optimization_comprehensive_solution.md @@ -0,0 +1,710 @@ +# 告警规则系统综合优化方案 + +## 📊 现状分析 + +### 当前t_alert_rules表结构 +```sql +CREATE TABLE `t_alert_rules` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `rule_type` varchar(50) NOT NULL DEFAULT 'metric' COMMENT 'metric|custom|fallback', + `physical_sign` varchar(50) DEFAULT NULL, + `threshold_min` decimal(10,2) DEFAULT NULL, + `threshold_max` decimal(10,2) DEFAULT NULL, + `deviation_percentage` decimal(5,2) DEFAULT NULL, + `trend_duration` int DEFAULT NULL COMMENT '连续异常次数', + `parameters` json DEFAULT NULL, + `trigger_condition` text, + `alert_message` text, + `severity_level` varchar(20) DEFAULT NULL, + `notification_type` varchar(50) DEFAULT 'message', + `customer_id` bigint DEFAULT NULL, + `is_active` tinyint(1) DEFAULT 1, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `created_by` bigint DEFAULT NULL, + `updated_by` bigint DEFAULT NULL, + `deleted` tinyint(1) DEFAULT 0, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +``` + +### 当前系统痛点 + +1. **单体征规则限制** - 当前每条规则只能配置单个`physical_sign` +2. **复合条件缺失** - 无法实现"心率>120 AND 血氧<90"这类多体征关联告警 +3. **阈值表达能力有限** - 只支持min/max范围,无法支持复杂阈值逻辑 +4. **连续异常判断不灵活** - 只有简单的`trend_duration`计数,缺乏时间窗口概念 +5. **高并发处理瓶颈** - 每次告警检测都需要查询数据库 + +## 🎯 核心需求分析 + +### 业务场景支持 + +#### 场景1:单体征阈值告警 +- **需求**: 心率高于120连续3次触发告警 +- **当前**: ✅ 已支持 +- **优化**: 增强时间窗口控制 + +#### 场景2:多体征关联告警 +- **需求**: 心率>120 AND 血氧<90 同时满足时触发告警 +- **当前**: ❌ 不支持 +- **优化**: 新增复合规则支持 + +#### 场景3:复杂阈值条件 +- **需求**: 心率异常=(心率>最大阈值 OR 心率<最小阈值)AND 持续时间>5分钟 +- **当前**: ❌ 部分支持 +- **优化**: 增强条件表达式引擎 + +#### 场景4:智能告警抑制 +- **需求**: 避免短时间内重复告警 +- **当前**: ❌ 无抑制机制 +- **优化**: 添加告警去重和抑制逻辑 + +## 🚀 优化方案设计 + +### 1. 数据结构优化 + +#### 1.1 保持向下兼容的表结构增强 +```sql +-- 在现有表基础上增加字段,保持向下兼容 +ALTER TABLE `t_alert_rules` +ADD COLUMN `rule_category` ENUM('SINGLE', 'COMPOSITE', 'COMPLEX') DEFAULT 'SINGLE' COMMENT '规则类型:单体征/复合/复杂', +ADD COLUMN `condition_expression` TEXT COMMENT 'JSON格式的复杂条件表达式', +ADD COLUMN `time_window_seconds` INT DEFAULT 300 COMMENT '时间窗口(秒)', +ADD COLUMN `cooldown_seconds` INT DEFAULT 600 COMMENT '告警冷却期(秒)', +ADD COLUMN `priority_level` INT DEFAULT 3 COMMENT '优先级(1-5,数字越小优先级越高)', +ADD COLUMN `rule_tags` JSON COMMENT '规则标签,便于分类管理', +ADD COLUMN `effective_time_start` TIME COMMENT '生效开始时间', +ADD COLUMN `effective_time_end` TIME COMMENT '生效结束时间', +ADD COLUMN `effective_days` VARCHAR(20) DEFAULT '1,2,3,4,5,6,7' COMMENT '生效星期(1-7)', +ADD INDEX idx_customer_category (`customer_id`, `rule_category`), +ADD INDEX idx_priority (`priority_level`), +ADD INDEX idx_physical_sign_active (`physical_sign`, `is_active`); +``` + +#### 1.2 复合规则条件表达式设计 +```json +{ + "rule_type": "composite", + "conditions": [ + { + "physical_sign": "heart_rate", + "operator": ">", + "threshold": 120, + "duration_seconds": 180 + }, + { + "physical_sign": "blood_oxygen", + "operator": "<", + "threshold": 90, + "duration_seconds": 60 + } + ], + "logical_operator": "AND", + "evaluation_window": 300, + "trigger_threshold": { + "type": "percentage", + "value": 80 + } +} +``` + +#### 1.3 规则处理引擎设计 +```python +class AlertRuleEngine: + """高性能告警规则引擎""" + + def __init__(self): + self.redis = RedisHelper() + self.rule_cache = {} + self.evaluation_cache = {} + + def evaluate_rules(self, health_data: dict, device_info: dict) -> List[dict]: + """ + 高效规则评估主入口 + + Args: + health_data: 健康数据 + device_info: 设备信息 + + Returns: + 符合条件的告警列表 + """ + customer_id = device_info.get('customer_id') + user_id = health_data.get('user_id') + device_sn = health_data.get('deviceSn') + + # 1. 获取缓存的规则 + rules = self._get_cached_rules(customer_id) + if not rules: + return [] + + triggered_alerts = [] + + # 2. 按优先级和类型分组处理 + for rule in sorted(rules, key=lambda x: x.get('priority_level', 5)): + if not self._is_rule_effective(rule): + continue + + # 3. 根据规则类型选择评估策略 + if rule['rule_category'] == 'SINGLE': + alert = self._evaluate_single_rule(rule, health_data, device_info) + elif rule['rule_category'] == 'COMPOSITE': + alert = self._evaluate_composite_rule(rule, health_data, device_info) + elif rule['rule_category'] == 'COMPLEX': + alert = self._evaluate_complex_rule(rule, health_data, device_info) + else: + continue + + if alert: + # 4. 告警抑制检查 + if not self._is_suppressed(alert, rule): + triggered_alerts.append(alert) + # 5. 设置冷却期 + self._set_cooldown(alert, rule) + + return triggered_alerts + + def _evaluate_single_rule(self, rule: dict, health_data: dict, device_info: dict) -> Optional[dict]: + """单体征规则评估 - 兼容现有逻辑""" + physical_sign = rule.get('physical_sign') + if not physical_sign or physical_sign not in health_data: + return None + + value = float(health_data[physical_sign]) + threshold_min = rule.get('threshold_min') + threshold_max = rule.get('threshold_max') + + # 阈值检查 + is_abnormal = False + if threshold_min and value < threshold_min: + is_abnormal = True + elif threshold_max and value > threshold_max: + is_abnormal = True + + if not is_abnormal: + return None + + # 连续异常检查(优化后的逻辑) + trend_duration = rule.get('trend_duration', 1) + time_window = rule.get('time_window_seconds', 300) + + if self._check_trend_duration(health_data, rule, trend_duration, time_window): + return self._create_alert(rule, health_data, device_info, { + 'trigger_value': value, + 'threshold_min': threshold_min, + 'threshold_max': threshold_max + }) + + return None + + def _evaluate_composite_rule(self, rule: dict, health_data: dict, device_info: dict) -> Optional[dict]: + """复合规则评估 - 支持多体征关联""" + try: + condition_expr = rule.get('condition_expression') + if not condition_expr: + return None + + conditions = condition_expr.get('conditions', []) + logical_op = condition_expr.get('logical_operator', 'AND') + + condition_results = [] + trigger_details = {} + + for condition in conditions: + physical_sign = condition['physical_sign'] + if physical_sign not in health_data: + condition_results.append(False) + continue + + value = float(health_data[physical_sign]) + operator = condition['operator'] + threshold = condition['threshold'] + duration = condition.get('duration_seconds', 60) + + # 单个条件评估 + condition_met = self._evaluate_condition(value, operator, threshold) + + # 持续时间检查 + if condition_met and duration > 0: + condition_met = self._check_duration_for_sign( + health_data, physical_sign, operator, threshold, duration + ) + + condition_results.append(condition_met) + if condition_met: + trigger_details[physical_sign] = { + 'value': value, + 'operator': operator, + 'threshold': threshold + } + + # 逻辑组合评估 + if logical_op == 'AND': + final_result = all(condition_results) + elif logical_op == 'OR': + final_result = any(condition_results) + else: + final_result = False + + if final_result: + return self._create_alert(rule, health_data, device_info, { + 'composite_triggers': trigger_details, + 'condition_results': condition_results + }) + + except Exception as e: + logger.error(f"复合规则评估失败: {e}") + + return None + + def _get_cached_rules(self, customer_id: int) -> List[dict]: + """获取缓存的告警规则""" + cache_key = f"alert_rules:customer:{customer_id}" + + # 先检查内存缓存 + if cache_key in self.rule_cache: + cached_time, rules = self.rule_cache[cache_key] + if time.time() - cached_time < 300: # 5分钟内存缓存 + return rules + + # Redis缓存 + cached_rules = self.redis.get_data(cache_key) + if cached_rules: + rules = json.loads(cached_rules) + self.rule_cache[cache_key] = (time.time(), rules) + return rules + + # 数据库查询并缓存 + rules = self._load_rules_from_db(customer_id) + + # 缓存到Redis(24小时) + self.redis.set_data(cache_key, json.dumps(rules, default=str), expire=86400) + self.rule_cache[cache_key] = (time.time(), rules) + + return rules +``` + +### 2. 管理界面设计 + +#### 2.1 规则管理页面结构 +```typescript +// 前端界面设计 - Vue3 + Element Plus +interface AlertRuleConfig { + id?: number; + ruleName: string; + ruleCategory: 'SINGLE' | 'COMPOSITE' | 'COMPLEX'; + ruleType: string; + priorityLevel: number; + + // 单体征规则 + physicalSign?: string; + thresholdMin?: number; + thresholdMax?: number; + trendDuration?: number; + + // 复合规则 + conditionExpression?: { + conditions: Array<{ + physicalSign: string; + operator: '>' | '<' | '=' | '>=' | '<='; + threshold: number; + durationSeconds: number; + }>; + logicalOperator: 'AND' | 'OR'; + evaluationWindow: number; + }; + + // 生效时间 + effectiveTimeStart?: string; + effectiveTimeEnd?: string; + effectiveDays: string; + timeWindowSeconds: number; + cooldownSeconds: number; + + // 告警配置 + severityLevel: string; + alertMessage: string; + notificationType: string; + + customerId: number; + isActive: boolean; +} +``` + +#### 2.2 规则配置向导组件 +```vue + +``` + +### 3. 高并发优化策略 + +#### 3.1 三层缓存架构 +```python +class CachedRuleEngine: + """三层缓存的告警规则引擎""" + + def __init__(self): + # L1: 应用内存缓存 (最快,容量小) + self.memory_cache = {} # 5分钟TTL,最热数据 + + # L2: Redis缓存 (快速,容量中等) + self.redis = RedisHelper() # 1小时TTL,常用数据 + + # L3: 数据库 (慢速,容量大) + # 通过上层缓存减少数据库访问 + + def get_rules_with_cache(self, customer_id: int) -> List[dict]: + """三层缓存获取规则""" + cache_key = f"rules:c{customer_id}" + + # L1: 内存缓存 + if cache_key in self.memory_cache: + cached_time, rules = self.memory_cache[cache_key] + if time.time() - cached_time < 300: # 5分钟 + return rules + + # L2: Redis缓存 + redis_key = f"alert_rules:customer:{customer_id}" + cached_rules = self.redis.get_data(redis_key) + if cached_rules: + rules = json.loads(cached_rules) + # 回填L1缓存 + self.memory_cache[cache_key] = (time.time(), rules) + return rules + + # L3: 数据库查询 + rules = self._query_rules_from_db(customer_id) + + # 回填缓存 + self.redis.set_data(redis_key, json.dumps(rules, default=str), expire=3600) + self.memory_cache[cache_key] = (time.time(), rules) + + return rules +``` + +#### 3.2 规则预编译与索引优化 +```python +class RuleCompiler: + """规则预编译器 - 将规则转换为高效的执行结构""" + + def compile_rules(self, rules: List[dict]) -> dict: + """编译规则为高效执行结构""" + compiled_rules = { + 'single_rules': defaultdict(list), # 按physical_sign分组 + 'composite_rules': [], # 复合规则列表 + 'priority_index': defaultdict(list), # 按优先级索引 + } + + for rule in rules: + if not rule.get('is_active'): + continue + + priority = rule.get('priority_level', 5) + + if rule['rule_category'] == 'SINGLE': + physical_sign = rule.get('physical_sign') + if physical_sign: + compiled_rule = self._compile_single_rule(rule) + compiled_rules['single_rules'][physical_sign].append(compiled_rule) + compiled_rules['priority_index'][priority].append(compiled_rule) + + elif rule['rule_category'] == 'COMPOSITE': + compiled_rule = self._compile_composite_rule(rule) + compiled_rules['composite_rules'].append(compiled_rule) + compiled_rules['priority_index'][priority].append(compiled_rule) + + return compiled_rules + + def _compile_single_rule(self, rule: dict) -> dict: + """编译单体征规则为高效结构""" + return { + 'rule_id': rule['id'], + 'physical_sign': rule['physical_sign'], + 'check_min': rule.get('threshold_min') is not None, + 'check_max': rule.get('threshold_max') is not None, + 'threshold_min': rule.get('threshold_min', 0), + 'threshold_max': rule.get('threshold_max', float('inf')), + 'trend_duration': rule.get('trend_duration', 1), + 'time_window': rule.get('time_window_seconds', 300), + 'severity': rule.get('severity_level', 'MEDIUM'), + 'message_template': rule.get('alert_message', ''), + 'cooldown': rule.get('cooldown_seconds', 600), + 'priority': rule.get('priority_level', 5) + } +``` + +#### 3.3 异步处理与批量优化 +```python +class AsyncAlertProcessor: + """异步告警处理器""" + + def __init__(self): + self.alert_queue = asyncio.Queue(maxsize=10000) + self.batch_size = 100 + self.workers = [] + + async def start_workers(self, worker_count: int = 4): + """启动异步工作者""" + for i in range(worker_count): + worker = asyncio.create_task(self._worker_loop(f"worker-{i}")) + self.workers.append(worker) + + async def process_alerts_batch(self, health_data_batch: List[dict]): + """批量处理告警""" + # 1. 按customer_id分组批次 + customer_batches = defaultdict(list) + for data in health_data_batch: + customer_id = data.get('customer_id') + customer_batches[customer_id].append(data) + + # 2. 并行处理每个客户的数据 + tasks = [] + for customer_id, customer_data in customer_batches.items(): + task = self._process_customer_batch(customer_id, customer_data) + tasks.append(task) + + await asyncio.gather(*tasks, return_exceptions=True) + + async def _process_customer_batch(self, customer_id: int, health_data_list: List[dict]): + """处理单个客户的健康数据批次""" + # 1. 批量获取规则(一次查询) + rules = await self._get_rules_async(customer_id) + if not rules: + return + + # 2. 编译规则(一次编译) + compiled_rules = self.rule_compiler.compile_rules(rules) + + # 3. 批量评估 + alert_tasks = [] + for health_data in health_data_list: + task = self._evaluate_single_data(health_data, compiled_rules) + alert_tasks.append(task) + + results = await asyncio.gather(*alert_tasks, return_exceptions=True) + + # 4. 批量保存告警 + valid_alerts = [alert for alert in results if alert and not isinstance(alert, Exception)] + if valid_alerts: + await self._save_alerts_batch(valid_alerts) +``` + +### 4. 性能基准测试 + +#### 4.1 测试指标设计 +```python +class PerformanceTestSuite: + """告警规则系统性能测试套件""" + + async def test_rule_evaluation_performance(self): + """规则评估性能测试""" + test_scenarios = [ + { + 'name': '单体征规则-1000条', + 'rule_count': 1000, + 'rule_type': 'SINGLE', + 'data_points': 10000 + }, + { + 'name': '复合规则-500条', + 'rule_count': 500, + 'rule_type': 'COMPOSITE', + 'data_points': 10000 + }, + { + 'name': '混合规则-高并发', + 'rule_count': 2000, + 'rule_type': 'MIXED', + 'data_points': 50000, + 'concurrent_requests': 100 + } + ] + + for scenario in test_scenarios: + start_time = time.time() + + # 执行测试场景 + await self._run_scenario(scenario) + + end_time = time.time() + duration = end_time - start_time + + # 计算性能指标 + throughput = scenario['data_points'] / duration + avg_latency = duration / scenario['data_points'] * 1000 + + print(f""" + 场景: {scenario['name']} + 总耗时: {duration:.2f}s + 吞吐量: {throughput:.0f} records/s + 平均延迟: {avg_latency:.2f}ms + """) +``` + +## 📈 实施计划 + +### 阶段一:向下兼容增强 (2周) +1. **数据库表结构增强** - 添加新字段,保持现有功能不变 +2. **缓存机制优化** - 实现三层缓存架构 +3. **性能基准测试** - 建立性能监控基线 + +### 阶段二:复合规则支持 (3周) +1. **规则引擎开发** - 支持多体征关联告警 +2. **管理界面开发** - 可视化规则配置向导 +3. **规则编译器** - 预编译优化执行效率 + +### 阶段三:高并发优化 (2周) +1. **异步处理架构** - 批量处理和异步队列 +2. **告警抑制机制** - 避免重复告警 +3. **性能调优** - 达到目标性能指标 + +### 阶段四:生产部署 (1周) +1. **灰度发布** - 逐步替换现有系统 +2. **监控告警** - 建立系统监控 +3. **文档完善** - 用户手册和运维文档 + +## 🎯 预期效果 + +### 性能提升 +- **规则评估速度**: 提升80%以上 (缓存优化) +- **并发处理能力**: 支持10倍以上并发 (异步架构) +- **内存使用**: 减少40% (规则预编译) + +### 功能增强 +- **规则表达能力**: 支持复杂的多体征关联告警 +- **管理易用性**: 可视化配置界面,降低使用门槛 +- **系统稳定性**: 告警抑制和冷却机制避免告警风暴 + +### 业务价值 +- **告警准确性**: 减少80%以上误报 (复合条件) +- **响应速度**: 告警延迟降低到秒级 +- **运维成本**: 减少50%以上人工干预 + +这个方案在保持系统稳定性的前提下,逐步提升告警规则的表达能力和处理性能,满足复杂业务场景的需求。 \ No newline at end of file diff --git a/docs/generate_alerts_analysis.md b/docs/generate_alerts_analysis.md new file mode 100644 index 0000000..cf5eb3c --- /dev/null +++ b/docs/generate_alerts_analysis.md @@ -0,0 +1,135 @@ +# generate_alerts 性能瓶颈分析 + +## 📊 概述 + +`generate_alerts` 函数是系统中的核心告警生成组件,负责根据设备数据和预设规则生成健康告警。通过深入分析发现该函数存在多个性能瓶颈,影响系统整体响应速度。 + +## 🔍 当前问题识别 + +### 1. 数据库查询频繁 +- **问题**: 每次调用都要查询所有告警规则 +- **影响**: 大量重复的数据库I/O操作 +- **症状**: 高数据库连接数和查询延迟 + +### 2. 同步阻塞处理 +- **问题**: 在主线程中执行复杂的告警规则匹配 +- **影响**: 阻塞其他并发请求的处理 +- **症状**: 系统整体响应时间增长 + +### 3. 重复查询设备信息 +- **问题**: 多次调用 `get_device_user_org_info` +- **影响**: 相同设备信息被重复查询 +- **症状**: 数据库查询量成倍增加 + +### 4. 事务频繁提交 +- **问题**: 每个告警都单独提交事务 +- **影响**: 大量事务开销和锁竞争 +- **症状**: 数据库写入性能下降 + +### 5. 无缓存机制 +- **问题**: 告警规则和设备信息没有缓存 +- **影响**: 静态数据重复查询 +- **症状**: 不必要的数据库负载 + +## 📈 性能影响评估 + +| 问题类型 | 影响程度 | 优化优先级 | 预期提升 | +|---------|---------|-----------|---------| +| 数据库查询频繁 | 高 | P0 | 50-70% | +| 同步阻塞处理 | 高 | P0 | 30-50% | +| 重复查询设备信息 | 中 | P1 | 20-40% | +| 事务频繁提交 | 中 | P1 | 10-30% | +| 无缓存机制 | 中 | P2 | 20-50% | + +## 🚀 优化建议 + +### 短期优化 (1-2周) +1. **批量事务处理**: 将多个告警写入合并到单个事务中 +2. **设备信息缓存**: 在单次调用中缓存设备查询结果 +3. **告警规则预加载**: 启动时加载告警规则到内存 + +### 中期优化 (2-4周) +1. **异步处理机制**: 将告警生成移到后台队列处理 +2. **Redis缓存**: 实现告警规则和设备信息的分布式缓存 +3. **数据库连接池优化**: 调整连接池参数 + +### 长期优化 (1-2个月) +1. **微服务拆分**: 将告警生成独立为微服务 +2. **流处理架构**: 使用消息队列实现实时流处理 +3. **智能预测**: 基于历史数据预测告警趋势 + +## 📋 实施计划 + +### 阶段一: 数据库优化 +- [ ] 实现告警规则缓存机制 +- [ ] 优化设备信息查询逻辑 +- [ ] 批量事务提交改造 + +### 阶段二: 架构优化 +- [ ] 引入异步处理队列 +- [ ] 实现Redis缓存层 +- [ ] 添加性能监控指标 + +### 阶段三: 系统重构 +- [ ] 微服务化改造 +- [ ] 流处理架构实施 +- [ ] 智能告警算法集成 + +## 📊 监控指标 + +建议监控以下关键指标来评估优化效果: + +- **响应时间**: `generate_alerts` 函数平均执行时间 +- **吞吐量**: 每秒处理的告警数量 +- **数据库负载**: 查询次数和连接数 +- **缓存命中率**: 告警规则和设备信息缓存效率 +- **错误率**: 告警生成失败率 + +## 🔧 技术实现要点 + +### 缓存策略 +```python +# 告警规则缓存 (TTL: 1小时) +@lru_cache(maxsize=1000) +def get_alert_rules(): + pass + +# 设备信息缓存 (TTL: 10分钟) +@lru_cache(maxsize=5000) +def get_device_info(device_id): + pass +``` + +### 批量处理 +```python +# 批量插入告警记录 +def batch_insert_alerts(alerts_batch): + with transaction.atomic(): + Alert.objects.bulk_create(alerts_batch) +``` + +### 异步处理 +```python +# Celery任务队列 +@celery_app.task +def process_alerts_async(device_data): + pass +``` + +## ⚠️ 注意事项 + +1. **数据一致性**: 异步处理需要保证数据一致性 +2. **缓存失效**: 需要合理设置缓存TTL和失效策略 +3. **监控告警**: 优化过程中加强系统监控 +4. **渐进式部署**: 分阶段实施,避免系统风险 + +## 📞 相关文档 + +- [数据库优化指南](./database_optimization.md) +- [缓存策略文档](./cache_strategy.md) +- [异步处理架构](./async_architecture.md) + +--- + +*文档创建时间: 2025-09-09* +*分析人员: 系统架构团队* \ No newline at end of file diff --git a/docs/message_flow_unified_model.md b/docs/message_flow_unified_model.md new file mode 100644 index 0000000..ade0c7c --- /dev/null +++ b/docs/message_flow_unified_model.md @@ -0,0 +1,277 @@ +# 基于现有数据库的统一消息数据模型 + +## 数据库表结构映射 + +基于现有的 `t_device_message` 和 `t_device_message_detail` 表结构,定义统一的消息数据模型。 + +### 主消息表 (t_device_message) + +```sql +CREATE TABLE t_device_message ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + device_sn VARCHAR(255) NOT NULL, -- 设备序列号 + message TEXT NOT NULL, -- 消息内容 + org_id VARCHAR(50), -- 组织ID + user_id VARCHAR(50), -- 用户ID + customer_id BIGINT NOT NULL DEFAULT 0, -- 租户ID + message_type VARCHAR(50) NOT NULL, -- 消息类型: job/task/announcement/notification/system_alert + sender_type VARCHAR(50) NOT NULL, -- 发送者类型: system/admin/user + receiver_type VARCHAR(50) NOT NULL, -- 接收者类型: device/user/department + message_status VARCHAR(50) DEFAULT 'pending', -- 消息状态: pending/delivered/acknowledged + responded_number INTEGER DEFAULT 0, -- 响应数量 + sent_time DATETIME DEFAULT CURRENT_TIMESTAMP, + received_time DATETIME NULL, + -- 扩展字段 + title VARCHAR(500), -- 消息标题 + priority INTEGER DEFAULT 3, -- 优先级 1-5 + urgency VARCHAR(20) DEFAULT 'medium', -- 紧急程度: low/medium/high/critical + channels JSON, -- 分发渠道: ["message","push","wechat","watch"] + require_ack BOOLEAN DEFAULT false, -- 是否需要确认 + expiry_time DATETIME, -- 过期时间 + metadata JSON, -- 元数据 + create_time DATETIME DEFAULT CURRENT_TIMESTAMP, + update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); +``` + +### 消息详情表 (t_device_message_detail) + +```sql +CREATE TABLE t_device_message_detail ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + message_id BIGINT NOT NULL, -- 关联主消息ID + device_sn VARCHAR(255) NOT NULL, + message TEXT NOT NULL, + message_type VARCHAR(50) NOT NULL, + sender_type VARCHAR(50) NOT NULL, + receiver_type VARCHAR(50) NOT NULL, + message_status VARCHAR(50) DEFAULT 'pending', + sent_time DATETIME, + received_time DATETIME, + customer_id BIGINT NOT NULL, + org_id BIGINT, + -- 扩展字段用于完整跟踪 + distribution_id VARCHAR(255), -- 分发ID + target_type VARCHAR(50), -- 目标类型: user/device/department/organization + target_id VARCHAR(255), -- 目标ID + delivery_status VARCHAR(50), -- 分发状态: pending/delivered/acknowledged/failed/expired + channel VARCHAR(50), -- 分发渠道: message/push/wechat/watch + response_time INTEGER, -- 响应时间(秒) + acknowledge_time DATETIME, -- 确认时间 + delivery_details JSON -- 分发详情 +); +``` + +## 统一消息JSON模型 + +### 完整消息实体 + +```json +{ + "messageId": "1934567890123456789", + "messageType": "job|task|announcement|notification|system_alert", + "title": "消息标题", + "content": "消息内容", + "priority": 3, + "urgency": "medium", + + "sender": { + "senderId": "发送者ID", + "senderName": "发送者姓名", + "senderType": "admin|system|user", + "platformSource": "ljwx-admin|ljwx-bigscreen|auto-alert" + }, + + "target": { + "targetType": "user|department|organization|device", + "targetIds": ["1001", "1002"], + "deviceSns": ["DEVICE001", "DEVICE002"], + "customerId": 1, + "orgIds": [101, 102], + "userIds": [1001, 1002] + }, + + "delivery": { + "channels": ["message", "push", "wechat", "watch"], + "deliveryTime": "2025-09-10T08:00:00Z", + "expiryTime": "2025-09-12T08:00:00Z", + "requireAck": true, + "ackTimeout": 3600 + }, + + "status": { + "messageStatus": "pending|delivered|acknowledged|expired", + "createTime": "2025-09-10T08:00:00Z", + "sentTime": "2025-09-10T08:00:01Z", + "deliveryStats": { + "totalTargets": 100, + "delivered": 95, + "acknowledged": 78, + "pending": 17, + "failed": 5 + } + }, + + "metadata": { + "source": "manual|auto-alert|scheduled", + "relatedAlertId": "告警关联ID", + "relatedHealthId": "健康数据关联ID", + "tags": ["urgent", "health", "safety"], + "customFields": {} + } +} +``` + +## 基于实际代码的数据流架构 + +### 消息数据流向 + +``` +ljwx-admin/ljwx-bigscreen → ljwx-boot → ljwx-phone → ljwx-watch + ↓ ↓ ↓ ↓ + 消息创建 t_device_message Redis HTTP/蓝牙 + ↓ ↓ ↓ ↓ + 规则匹配 t_device_message_detail 订阅 系统通知 + ↓ ↓ ↓ ↓ + 告警转消息 API接口 消息转发 用户确认 + ↓ ↓ ↓ ↓ + 数据库存储 Redis发布 蓝牙传输 确认回传 +``` + +### 各平台现有API接口 + +#### ljwx-boot (Java后端) +```java +// 现有接口 +@RestController +@RequestMapping("t_device_message") +public class TDeviceMessageController { + @GetMapping("/page") // 分页查询消息 + @GetMapping("/{id}") // 根据ID获取消息 + @PostMapping("/") // 新增消息 + @PutMapping("/") // 更新消息 + @DeleteMapping("/") // 批量删除消息 +} + +// 兼容性服务 +@Service +public class DeviceMessageServiceImpl { + Result> saveMessage(DeviceMessageSaveRequest) + Result> sendMessage(DeviceMessageSendRequest) + Result> receiveMessages(String deviceSn) +} +``` + +#### ljwx-bigscreen (Python大屏) +```python +# 现有接口 +@app.route("/message") # 消息页面 +@app.route('/DeviceMessage/save_message', methods=['POST']) # 保存消息 +@app.route('/DeviceMessage/send', methods=['POST']) # 发送消息 +@app.route('/DeviceMessage/receive', methods=['GET']) # 接收消息 +@app.route('/fetch_messages', methods=['GET']) # 按设备获取消息 +@app.route('/get_messages_by_orgIdAndUserId', methods=['GET']) # 按组织用户获取 + +# 告警转消息机制 +class MessageNotifier: + @staticmethod + def send_message(device_sn, message, message_type, health_id=None) +``` + +#### ljwx-phone (Flutter移动端) +```dart +// 现有服务 +class ApiService { + Future>> getDeviceMessages(String deviceSn) + Future markMessageAsRead(String deviceSn, ...) + Future getPersonalInfo(String phone) // 包含消息信息 +} + +// 消息模型 +class Message { + final String id, title, content, createTime; + final String department, messageStatus, messageType; + final String? imageUrl; +} + +// 蓝牙消息转发 +class BluetoothService { + Future _fetchAndSendMessages() // 获取并发送消息到手表 + Future sendMsg(Map message) +} +``` + +#### ljwx-watch (HarmonyOS手表) +```java +// HTTP模式 +public class HttpService { + public void fetchMessageFromServer() // 从服务器获取消息 + private void handleMessageResponse(JSONObject message) // 处理消息响应 +} + +// 蓝牙模式 +public class BluetoothService { + private void handleCommand(BlePeripheralDevice device, String command) + private void handleMessageCommand(JSONObject cmdJson) // 处理消息命令 +} + +// 消息类型映射 +private static final Map MESSAGE_TYPE_MAP = { + "announcement" -> "公告", + "notification" -> "个人通知", + "warning" -> "告警", + "job" -> "作业指引", + "task" -> "任务管理" +}; +``` + +## 消息分发记录 + +基于 `t_device_message_detail` 表的分发记录格式: + +```json +{ + "distributionId": "1934567890123456790", + "messageId": "1934567890123456789", + "targetId": "1001", + "targetType": "user", + "deviceSn": "DEVICE001", + "channel": "message", + "deliveryStatus": "pending|delivered|acknowledged|failed|expired", + "deliveryTime": "2025-09-10T08:00:05Z", + "acknowledgeTime": "2025-09-10T08:15:30Z", + "responseTime": 930, + + "deliveryDetails": { + "attempts": 2, + "lastAttemptTime": "2025-09-10T08:00:05Z", + "errorMessage": null, + "platform": "ljwx-phone", + "deviceInfo": { + "phoneModel": "iPhone 13", + "watchModel": "HarmonyOS Watch", + "appVersion": "1.0.3" + } + } +} +``` + +## 现有消息类型定义 + +基于代码分析,系统支持以下消息类型: + +```javascript +const MESSAGE_TYPES = { + // ljwx-phone定义 + "job": "作业指引", // 蓝绿色,工作图标 + "task": "任务管理", // 蓝色,任务图标 + "announcement": "系统公告", // 紫色,广播图标 + "notification": "通知", // 琥珀色,通知图标 + + // ljwx-bigscreen扩展 + "system_alert": "系统告警", // 自动告警生成 + + // ljwx-watch映射 + "warning": "告警" // 告警类消息 +}; +``` \ No newline at end of file diff --git a/docs/message_integration_implementation_guide.md b/docs/message_integration_implementation_guide.md new file mode 100644 index 0000000..3daf85e --- /dev/null +++ b/docs/message_integration_implementation_guide.md @@ -0,0 +1,896 @@ +# 消息数据流集成实施指南 + +基于对现有代码的详细分析,本文档提供具体的实施步骤和代码改动建议。 + +## 一、实施概览 + +### 目标 +- 完整打通 ljwx-admin/ljwx-bigscreen → ljwx-boot → ljwx-phone → ljwx-watch 的消息数据流 +- 实现实时消息推送和状态跟踪 +- 支持消息生命周期管理和统计分析 +- 保持现有API和数据结构的向后兼容 + +### 技术栈 +- **ljwx-boot**: Java Spring Boot + MyBatis Plus + Redis +- **ljwx-bigscreen**: Python Flask + SQLAlchemy + Redis +- **ljwx-phone**: Flutter + Dart + Redis Client +- **ljwx-watch**: HarmonyOS 4.0 + Java + +## 二、数据库扩展(阶段一) + +### 2.1 SQL脚本 + +```sql +-- 扩展 t_device_message 表 +ALTER TABLE t_device_message +ADD COLUMN title VARCHAR(500) COMMENT '消息标题', +ADD COLUMN priority INTEGER DEFAULT 3 COMMENT '优先级1-5', +ADD COLUMN urgency VARCHAR(20) DEFAULT 'medium' COMMENT '紧急程度', +ADD COLUMN channels JSON COMMENT '分发渠道', +ADD COLUMN require_ack BOOLEAN DEFAULT false COMMENT '需要确认', +ADD COLUMN expiry_time DATETIME COMMENT '过期时间', +ADD COLUMN metadata JSON COMMENT '元数据'; + +-- 扩展 t_device_message_detail 表 +ALTER TABLE t_device_message_detail +ADD COLUMN distribution_id VARCHAR(255) COMMENT '分发ID', +ADD COLUMN target_type VARCHAR(50) COMMENT '目标类型', +ADD COLUMN target_id VARCHAR(255) COMMENT '目标ID', +ADD COLUMN delivery_status VARCHAR(50) COMMENT '分发状态', +ADD COLUMN channel VARCHAR(50) COMMENT '分发渠道', +ADD COLUMN response_time INTEGER COMMENT '响应时间秒', +ADD COLUMN acknowledge_time DATETIME COMMENT '确认时间', +ADD COLUMN delivery_details JSON COMMENT '分发详情'; + +-- 创建索引优化查询 +CREATE INDEX idx_message_device_sn ON t_device_message(device_sn); +CREATE INDEX idx_message_org_id ON t_device_message(org_id); +CREATE INDEX idx_message_type_status ON t_device_message(message_type, message_status); +CREATE INDEX idx_detail_message_id ON t_device_message_detail(message_id); +CREATE INDEX idx_detail_device_sn ON t_device_message_detail(device_sn); +CREATE INDEX idx_detail_delivery_status ON t_device_message_detail(delivery_status); +``` + +### 2.2 ljwx-boot实体类更新 + +```java +// TDeviceMessage.java 新增字段 +@TableName("t_device_message") +public class TDeviceMessage extends BaseEntity { + // 现有字段... + + private String title; + private Integer priority; + private String urgency; + private String channels; // JSON字符串 + private Boolean requireAck; + private LocalDateTime expiryTime; + private String metadata; // JSON字符串 +} + +// TDeviceMessageDetail.java 新增字段 +@TableName("t_device_message_detail") +public class TDeviceMessageDetail extends BaseEntity { + // 现有字段... + + private String distributionId; + private String targetType; + private String targetId; + private String deliveryStatus; + private String channel; + private Integer responseTime; + private LocalDateTime acknowledgeTime; + private String deliveryDetails; // JSON字符串 +} +``` + +### 2.3 ljwx-bigscreen模型更新 + +```python +# models.py 扩展 +class DeviceMessage(db.Model): + __tablename__ = 't_device_message' + # 现有字段... + + title = db.Column(db.String(500), nullable=True) + priority = db.Column(db.Integer, default=3) + urgency = db.Column(db.String(20), default='medium') + channels = db.Column(db.JSON, nullable=True) + require_ack = db.Column(db.Boolean, default=False) + expiry_time = db.Column(db.DateTime, nullable=True) + metadata = db.Column(db.JSON, nullable=True) + +class DeviceMessageDetail(db.Model): + __tablename__ = 't_device_message_detail' + # 现有字段... + + distribution_id = db.Column(db.String(255), nullable=True) + target_type = db.Column(db.String(50), nullable=True) + target_id = db.Column(db.String(255), nullable=True) + delivery_status = db.Column(db.String(50), nullable=True) + channel = db.Column(db.String(50), nullable=True) + response_time = db.Column(db.Integer, nullable=True) + acknowledge_time = db.Column(db.DateTime, nullable=True) + delivery_details = db.Column(db.JSON, nullable=True) +``` + +## 三、后端服务增强(阶段二) + +### 3.1 ljwx-boot Redis配置 + +```java +// RedisConfig.java +@Configuration +public class RedisConfig { + + @Bean + public RedisTemplate redisTemplate(RedisConnectionFactory factory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(factory); + template.setDefaultSerializer(new GenericJackson2JsonRedisSerializer()); + return template; + } + + @Bean + public RedisMessageListenerContainer container(RedisConnectionFactory factory) { + RedisMessageListenerContainer container = new RedisMessageListenerContainer(); + container.setConnectionFactory(factory); + return container; + } +} +``` + +### 3.2 ljwx-boot增强消息服务 + +```java +// EnhancedMessageService.java +@Service +@Slf4j +public class EnhancedMessageService { + + @Autowired + private ITDeviceMessageService messageService; + + @Autowired + private ITDeviceMessageDetailService detailService; + + @Autowired + private RedisTemplate redisTemplate; + + /** + * 统一消息发布接口 + */ + @Transactional + public Long publishMessage(MessagePublishRequest request) { + try { + // 1. 创建主消息 + TDeviceMessage message = buildMainMessage(request); + messageService.save(message); + + // 2. 创建分发详情 + List details = buildMessageDetails(message, request.getTargets()); + detailService.saveBatch(details); + + // 3. 发布到Redis + publishToRedis(message, details); + + log.info("✅ 消息发布成功: messageId={}, 目标数量={}", message.getId(), details.size()); + return message.getId(); + + } catch (Exception e) { + log.error("❌ 消息发布失败: {}", e.getMessage(), e); + throw new RuntimeException("消息发布失败", e); + } + } + + /** + * 批量更新消息状态 + */ + public void updateMessageStatus(Long messageId, String status, List deviceSns) { + // 更新详情状态 + LambdaUpdateWrapper wrapper = new LambdaUpdateWrapper<>(); + wrapper.eq(TDeviceMessageDetail::getMessageId, messageId) + .in(TDeviceMessageDetail::getDeviceSn, deviceSns) + .set(TDeviceMessageDetail::getDeliveryStatus, status) + .set(TDeviceMessageDetail::getAcknowledgeTime, LocalDateTime.now()); + + detailService.update(wrapper); + + // 检查是否需要更新主消息状态 + updateMainMessageStatus(messageId); + + // 发布状态更新事件 + Map statusEvent = new HashMap<>(); + statusEvent.put("messageId", messageId); + statusEvent.put("status", status); + statusEvent.put("deviceSns", deviceSns); + statusEvent.put("timestamp", System.currentTimeMillis()); + + redisTemplate.convertAndSend("message:status:updates", statusEvent); + } + + private void publishToRedis(TDeviceMessage message, List details) { + for (TDeviceMessageDetail detail : details) { + Map payload = new HashMap<>(); + payload.put("messageId", message.getId()); + payload.put("deviceSn", detail.getDeviceSn()); + payload.put("messageType", message.getMessageType()); + payload.put("title", message.getTitle()); + payload.put("content", message.getMessage()); + payload.put("priority", message.getPriority()); + payload.put("urgency", message.getUrgency()); + payload.put("sentTime", message.getSentTime()); + payload.put("requireAck", message.getRequireAck()); + payload.put("channel", detail.getChannel()); + + String channel = "message:device:" + detail.getDeviceSn(); + redisTemplate.convertAndSend(channel, payload); + } + } +} +``` + +### 3.3 ljwx-bigscreen增强跟踪 + +```python +# enhanced_message_service.py +class EnhancedMessageService: + + def __init__(self): + self.redis_client = redis.Redis(host='localhost', port=6379, db=0) + + def create_tracked_message(self, message_data): + """创建带跟踪的消息""" + try: + # 创建主消息 + message = DeviceMessage( + device_sn=message_data.get('device_sn'), + message=message_data.get('content'), + title=message_data.get('title'), + message_type=message_data.get('message_type', 'notification'), + sender_type=message_data.get('sender_type', 'admin'), + receiver_type=message_data.get('receiver_type', 'device'), + priority=message_data.get('priority', 3), + urgency=message_data.get('urgency', 'medium'), + channels=json.dumps(message_data.get('channels', ['message'])), + require_ack=message_data.get('require_ack', False), + org_id=message_data.get('org_id'), + user_id=message_data.get('user_id'), + customer_id=message_data.get('customer_id', 0), + metadata=json.dumps(message_data.get('metadata', {})) + ) + + db.session.add(message) + db.session.flush() # 获取ID + + # 创建分发记录 + targets = message_data.get('targets', []) + details = [] + + for target in targets: + detail = DeviceMessageDetail( + message_id=message.id, + device_sn=target.get('device_sn'), + message=message.message, + message_type=message.message_type, + sender_type=message.sender_type, + receiver_type=message.receiver_type, + customer_id=message.customer_id, + org_id=message.org_id, + distribution_id=f"dist_{message.id}_{target.get('device_sn')}", + target_type=target.get('target_type', 'device'), + target_id=target.get('target_id', target.get('device_sn')), + delivery_status='pending', + channel=target.get('channel', 'message') + ) + details.append(detail) + db.session.add(detail) + + db.session.commit() + + # 发布到Redis + self._publish_message_created(message, details) + + return message + + except Exception as e: + db.session.rollback() + print(f"❌ 创建跟踪消息失败: {e}") + return None + + def _publish_message_created(self, message, details): + """发布消息创建事件""" + for detail in details: + payload = { + 'messageId': message.id, + 'deviceSn': detail.device_sn, + 'messageType': message.message_type, + 'title': message.title, + 'content': message.message, + 'priority': message.priority, + 'urgency': message.urgency, + 'sentTime': message.sent_time.isoformat() if message.sent_time else None, + 'requireAck': message.require_ack, + 'channel': detail.channel + } + + channel = f'message:device:{detail.device_sn}' + self.redis_client.publish(channel, json.dumps(payload)) + + def get_message_statistics(self, org_id, start_date=None, end_date=None): + """获取消息统计""" + query = DeviceMessage.query.filter_by(org_id=str(org_id)) + + if start_date: + query = query.filter(DeviceMessage.create_time >= start_date) + if end_date: + query = query.filter(DeviceMessage.create_time <= end_date) + + messages = query.all() + + # 统计计算 + stats = { + 'totalMessages': len(messages), + 'messageTypes': {}, + 'completionStats': { + 'totalTargets': 0, + 'delivered': 0, + 'acknowledged': 0, + 'pending': 0, + 'failed': 0 + } + } + + # 消息类型统计 + for message in messages: + msg_type = message.message_type + stats['messageTypes'][msg_type] = stats['messageTypes'].get(msg_type, 0) + 1 + + # 获取详情统计 + details = DeviceMessageDetail.query.filter_by(message_id=message.id).all() + stats['completionStats']['totalTargets'] += len(details) + + for detail in details: + status = detail.delivery_status or 'pending' + if status in stats['completionStats']: + stats['completionStats'][status] += 1 + + # 计算完成率 + total = stats['completionStats']['totalTargets'] + if total > 0: + acknowledged = stats['completionStats']['acknowledged'] + stats['completionStats']['completionRate'] = round(acknowledged / total * 100, 2) + else: + stats['completionStats']['completionRate'] = 0.0 + + return stats + +# 新增API端点 +@app.route('/api/enhanced/message/create', methods=['POST']) +def create_enhanced_message(): + """创建增强消息""" + data = request.get_json() + + service = EnhancedMessageService() + message = service.create_tracked_message(data) + + if message: + return jsonify({ + 'success': True, + 'messageId': message.id, + 'message': '消息创建成功' + }) + else: + return jsonify({ + 'success': False, + 'error': '消息创建失败' + }), 500 + +@app.route('/api/enhanced/message/stats/', methods=['GET']) +def get_enhanced_message_stats(org_id): + """获取增强消息统计""" + start_date = request.args.get('startDate') + end_date = request.args.get('endDate') + + service = EnhancedMessageService() + stats = service.get_message_statistics(org_id, start_date, end_date) + + return jsonify({ + 'success': True, + 'data': stats + }) +``` + +## 四、移动端实时化(阶段三) + +### 4.1 ljwx-phone Redis集成 + +```dart +// lib/services/redis_service.dart +import 'dart:convert'; +import 'dart:async'; + +class RedisService { + static RedisService? _instance; + StreamController>? _messageController; + + static RedisService get instance { + _instance ??= RedisService._internal(); + return _instance!; + } + + RedisService._internal(); + + Stream> get messageStream { + _messageController ??= StreamController>.broadcast(); + return _messageController!.stream; + } + + Future subscribeToMessages(String deviceSn) async { + try { + // 订阅设备消息通道 + String channel = 'message:device:$deviceSn'; + + // 这里需要集成实际的Redis客户端 + // 示例使用WebSocket或HTTP长连接模拟 + await _connectToRedis(channel); + + } catch (e) { + print('Redis订阅失败: $e'); + } + } + + Future _connectToRedis(String channel) async { + // 实际实现中可以使用redis_client包或者WebSocket + // 这里提供框架结构 + } + + void publishAcknowledgment(Map ackData) { + try { + String channel = 'message:acknowledgments'; + String payload = jsonEncode(ackData); + + // 发布确认事件 + _publishToRedis(channel, payload); + + } catch (e) { + print('发布确认失败: $e'); + } + } + + void _publishToRedis(String channel, String payload) { + // 实际Redis发布实现 + } +} +``` + +```dart +// lib/services/enhanced_message_service.dart +class EnhancedMessageService extends ApiService { + StreamSubscription>? _redisSubscription; + final StreamController _localMessageController = StreamController.broadcast(); + + Stream get localMessageStream => _localMessageController.stream; + + @override + void initState() { + super.initState(); + _initializeRedisSubscription(); + _startPeriodicSync(); // 保持现有定时同步作为备用 + } + + void _initializeRedisSubscription() async { + try { + String deviceSn = AppConfig.instance.deviceSn; + await RedisService.instance.subscribeToMessages(deviceSn); + + _redisSubscription = RedisService.instance.messageStream.listen((data) { + _handleRedisMessage(data); + }); + + } catch (e) { + print('初始化Redis订阅失败: $e'); + } + } + + void _handleRedisMessage(Map data) { + try { + // 转换为Message对象 + Message message = Message( + id: data['messageId'].toString(), + title: data['title'] ?? '', + content: data['content'] ?? '', + createTime: data['sentTime'] ?? DateTime.now().toIso8601String(), + department: '', // 从设备信息获取 + messageStatus: '1', // 未读状态 + messageType: data['messageType'] ?? 'notification', + ); + + // 保存到本地 + _saveMessageLocally(message); + + // 通知UI更新 + _localMessageController.add(message); + + // 转发给手表(如果蓝牙连接) + if (BluetoothService().isConnected) { + BluetoothService().forwardMessageToWatch(data); + } + + } catch (e) { + print('处理Redis消息失败: $e'); + } + } + + @override + Future markMessageAsRead(String deviceSn, { + DateTime? receivedTime, + String? messageId, + Map? originalMessage + }) async { + + // 调用现有API + final success = await super.markMessageAsRead( + deviceSn, + receivedTime: receivedTime, + messageId: messageId, + originalMessage: originalMessage + ); + + if (success && messageId != null) { + // 发布确认事件到Redis + RedisService.instance.publishAcknowledgment({ + 'type': 'message_acknowledged', + 'messageId': messageId, + 'deviceSn': deviceSn, + 'acknowledgeTime': (receivedTime ?? DateTime.now()).toIso8601String(), + 'responseTime': _calculateResponseTime(originalMessage), + 'source': 'phone', + 'platform': 'ljwx-phone' + }); + } + + return success; + } + + int _calculateResponseTime(Map? originalMessage) { + if (originalMessage == null || originalMessage['sentTime'] == null) { + return 0; + } + + try { + DateTime sentTime = DateTime.parse(originalMessage['sentTime']); + DateTime now = DateTime.now(); + return now.difference(sentTime).inSeconds; + } catch (e) { + return 0; + } + } + + void _saveMessageLocally(Message message) { + // 保存到本地数据库或缓存 + // 可以使用sqflite或hive + } +} +``` + +### 4.2 ljwx-phone UI更新 + +```dart +// lib/screens/enhanced_messages_screen.dart +class EnhancedMessagesScreen extends StatefulWidget { + @override + _EnhancedMessagesScreenState createState() => _EnhancedMessagesScreenState(); +} + +class _EnhancedMessagesScreenState extends State { + late EnhancedMessageService _messageService; + StreamSubscription? _messageSubscription; + + @override + void initState() { + super.initState(); + _messageService = EnhancedMessageService(); + _messageService.initState(); + + // 监听实时消息 + _messageSubscription = _messageService.localMessageStream.listen((message) { + setState(() { + // 更新UI + }); + + // 显示通知 + _showMessageNotification(message); + }); + } + + void _showMessageNotification(Message message) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + content: Text('新消息: ${message.title}'), + action: SnackBarAction( + label: '查看', + onPressed: () => _openMessageDetail(message), + ), + ), + ); + } + + @override + void dispose() { + _messageSubscription?.cancel(); + super.dispose(); + } +} +``` + +## 五、手表端优化(阶段四) + +### 5.1 ljwx-watch增强确认机制 + +```java +// HttpService.java 增强确认 +public void sendEnhancedMessageAcknowledgment(JSONObject originalMessage) { + try { + String messageId = originalMessage.getString("message_id"); + + // 构建增强确认数据 + JSONObject ackData = new JSONObject(); + ackData.put("type", "message_acknowledgment"); + ackData.put("messageId", messageId); + ackData.put("deviceSn", dataManager.getDeviceSn()); + ackData.put("acknowledgeTime", getCurrentISOTimestamp()); + ackData.put("responseTime", calculateResponseTime(originalMessage)); + ackData.put("channel", "http"); + ackData.put("source", "watch"); + ackData.put("platform", "ljwx-watch"); + + // 设备信息 + JSONObject deviceInfo = new JSONObject(); + deviceInfo.put("watchModel", "HarmonyOS Watch"); + deviceInfo.put("osVersion", "4.0"); + deviceInfo.put("appVersion", getAppVersion()); + deviceInfo.put("batteryLevel", getBatteryLevel()); + ackData.put("deviceInfo", deviceInfo); + + // 用户行为信息 + ackData.put("userAction", "acknowledged"); + ackData.put("interactionMethod", "touch"); + + // 复制原消息关键字段 + copyMessageFields(ackData, originalMessage); + + // 发送确认 + String ackUrl = dataManager.getFetchMessageUrl() + "/acknowledge"; + JSONObject response = postDataToServer(ackUrl, ackData); + + if (response != null && response.getBoolean("success")) { + Log.i(TAG, "✅ 增强消息确认发送成功: " + messageId); + showConfirmationFeedback(); // 显示确认反馈 + } else { + Log.w(TAG, "⚠️ 增强消息确认发送失败,缓存重试: " + messageId); + cacheAcknowledgmentForRetry(ackData); + } + + } catch (Exception e) { + Log.e(TAG, "❌ 增强消息确认异常: " + e.getMessage()); + } +} + +private int calculateResponseTime(JSONObject originalMessage) { + try { + String sentTimeStr = originalMessage.getString("sent_time"); + // 解析发送时间并计算差值 + long sentTime = parseDateToTimestamp(sentTimeStr); + long currentTime = System.currentTimeMillis(); + return (int) ((currentTime - sentTime) / 1000); + } catch (Exception e) { + Log.w(TAG, "无法计算响应时间: " + e.getMessage()); + return -1; + } +} + +private void showConfirmationFeedback() { + // 显示确认成功的视觉反馈 + vibrate(100); // 震动反馈 + showToast("消息已确认"); +} + +private void copyMessageFields(JSONObject ackData, JSONObject originalMessage) { + String[] fieldsToCoopy = { + "message", "message_type", "sent_time", "user_id", + "user_name", "department_id", "department_name", "is_public" + }; + + for (String field : fieldsToCoopy) { + if (originalMessage.has(field)) { + try { + ackData.put(field, originalMessage.get(field)); + } catch (JSONException e) { + Log.w(TAG, "复制字段失败: " + field); + } + } + } +} +``` + +## 六、部署和测试 + +### 6.1 环境配置 + +```yaml +# docker-compose.yml 添加Redis +version: '3.8' +services: + redis: + image: redis:7-alpine + ports: + - "6379:6379" + command: redis-server --appendonly yes + volumes: + - redis_data:/data + + ljwx-boot: + environment: + - REDIS_HOST=redis + - REDIS_PORT=6379 + +volumes: + redis_data: +``` + +### 6.2 测试脚本 + +```python +# test_message_flow.py +import requests +import json +import time + +class MessageFlowTester: + + def __init__(self): + self.bigscreen_url = "http://localhost:5000" + self.boot_url = "http://localhost:8080" + + def test_complete_flow(self): + """测试完整消息流""" + + # 1. 在大屏创建消息 + message_data = { + "title": "测试消息", + "content": "这是一条测试消息", + "message_type": "notification", + "sender_type": "admin", + "priority": 3, + "urgency": "medium", + "channels": ["message", "push"], + "require_ack": True, + "targets": [ + { + "device_sn": "TEST_DEVICE_001", + "target_type": "device", + "channel": "message" + } + ], + "org_id": "1", + "customer_id": 1 + } + + response = requests.post( + f"{self.bigscreen_url}/api/enhanced/message/create", + json=message_data + ) + + assert response.status_code == 200 + result = response.json() + assert result['success'] == True + + message_id = result['messageId'] + print(f"✅ 消息创建成功: {message_id}") + + # 2. 等待消息传播 + time.sleep(2) + + # 3. 检查消息状态 + stats_response = requests.get( + f"{self.bigscreen_url}/api/enhanced/message/stats/1" + ) + + assert stats_response.status_code == 200 + stats = stats_response.json() + print(f"✅ 消息统计: {stats['data']}") + + return message_id + + def test_acknowledgment(self, message_id): + """测试消息确认""" + + ack_data = { + "messageId": message_id, + "deviceSn": "TEST_DEVICE_001", + "acknowledgeTime": "2025-09-10T10:00:00Z", + "responseTime": 30, + "source": "phone" + } + + # 模拟手机端确认 + response = requests.post( + f"{self.boot_url}/api/message/acknowledge", + json=ack_data + ) + + print(f"✅ 消息确认测试完成") + +if __name__ == "__main__": + tester = MessageFlowTester() + message_id = tester.test_complete_flow() + tester.test_acknowledgment(message_id) + print("🎉 完整流程测试成功!") +``` + +## 七、监控和维护 + +### 7.1 性能监控 + +```java +// MessagePerformanceMonitor.java +@Component +public class MessagePerformanceMonitor { + + private final MeterRegistry meterRegistry; + + public MessagePerformanceMonitor(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public void recordMessageCreated(String messageType) { + meterRegistry.counter("message.created", "type", messageType).increment(); + } + + public void recordMessageDelivered(String channel, long deliveryTime) { + meterRegistry.timer("message.delivery.time", "channel", channel) + .record(deliveryTime, TimeUnit.MILLISECONDS); + } + + public void recordMessageAcknowledged(String source, long responseTime) { + meterRegistry.timer("message.response.time", "source", source) + .record(responseTime, TimeUnit.SECONDS); + } +} +``` + +### 7.2 日志配置 + +```yaml +# logback-spring.xml + + + logs/message.log + + logs/message.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + +``` + +## 八、总结 + +通过以上实施方案,可以实现: + +1. **完整数据流打通**: 四端消息传递无缝连接 +2. **实时状态同步**: Redis驱动的实时更新 +3. **全面跟踪能力**: 从创建到确认的完整监控 +4. **向后兼容性**: 保持现有API不变 +5. **可监控性**: 完善的日志和性能指标 + +实施周期约为5-6周,每个阶段独立部署,风险可控。 \ No newline at end of file diff --git a/docs/redis_cache_deployment_guide.md b/docs/redis_cache_deployment_guide.md new file mode 100644 index 0000000..484b4fc --- /dev/null +++ b/docs/redis_cache_deployment_guide.md @@ -0,0 +1,377 @@ +# Redis缓存告警系统部署指南 + +## 📋 部署概述 + +本指南用于部署基于Redis的告警规则缓存同步系统,实现ljwx-boot和ljwx-bigscreen之间的高效数据同步。 + +## 🏗️ 系统架构 + +``` +ljwx-admin → ljwx-boot (Redis DB=1) → MySQL + ↓ pub/sub + Redis服务器 + ↓ 订阅监听 + ljwx-bigscreen (Redis DB=0) → generate_alerts +``` + +## ✅ 部署前检查 + +### 1. 系统要求 +- **Redis版本**: 5.0+ +- **Python版本**: 3.7+ +- **Java版本**: JDK 8+ +- **内存要求**: 至少2GB可用内存 + +### 2. 现有组件确认 +```bash +# 检查Redis服务状态 +redis-cli ping + +# 检查ljwx-boot Redis配置 +grep -n "redis" ljwx-boot/ljwx-boot-admin/src/main/resources/application-local.yml + +# 检查ljwx-bigscreen Redis配置 +python3 ljwx-bigscreen/bigscreen/redis_config.py +``` + +### 3. 网络连通性测试 +```bash +# 测试Redis连接 +redis-cli -h 127.0.0.1 -p 6379 -a 123456 ping + +# 测试跨DB通信 +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 0 ping +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 ping +``` + +## 🚀 部署步骤 + +### 步骤1: 更新ljwx-boot缓存策略 + +1. **备份现有文件** +```bash +cd ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/ +cp TAlertRulesFacadeImpl.java TAlertRulesFacadeImpl.java.backup +``` + +2. **应用更新** +- 文件已在实施过程中更新 +- 新增版本控制和分布式锁机制 +- TTL设置为24小时(86400秒) + +3. **验证更新** +```bash +# 重启ljwx-boot服务 +cd ljwx-boot && ./run-local.sh restart + +# 检查日志 +tail -f ljwx-boot/logs/ljwx-boot-local.log | grep "告警规则缓存" +``` + +### 步骤2: 部署Python缓存管理器 + +1. **安装缓存管理器** +```bash +# 文件已创建在以下位置 +ls -la ljwx-bigscreen/bigscreen/alert_rules_cache_manager.py +``` + +2. **安装依赖** +```bash +cd ljwx-bigscreen/bigscreen +pip3 install redis dataclasses typing +``` + +3. **权限设置** +```bash +chmod +x alert_rules_cache_manager.py +``` + +### 步骤3: 部署优化版生成器 + +1. **安装生成器** +```bash +# 文件已创建 +ls -la utils/redis_cache_generate_alerts.py +chmod +x utils/redis_cache_generate_alerts.py +``` + +2. **测试基础功能** +```bash +cd utils +python3 redis_cache_generate_alerts.py +``` + +### 步骤4: 部署监控和测试工具 + +1. **部署测试套件** +```bash +ls -la utils/test_redis_cache_alerts.py +chmod +x utils/test_redis_cache_alerts.py +``` + +2. **部署监控工具** +```bash +ls -la utils/monitor_alert_cache_performance.py +chmod +x utils/monitor_alert_cache_performance.py +``` + +## 🧪 部署验证 + +### 1. 运行完整测试 +```bash +cd utils +python3 test_redis_cache_alerts.py +``` + +**期望输出**: +``` +🧪 启动Redis缓存告警系统完整测试 +✅ PASS - Redis连接测试 +✅ PASS - 告警规则获取测试 +✅ PASS - 告警生成测试 +✅ PASS - 批量处理性能测试 +✅ PASS - 缓存统计测试 +✅ PASS - 边界情况测试 + +总计: 6 | 通过: 6 | 失败: 0 | 通过率: 100.0% +🎉 测试通过!Redis缓存告警系统运行正常 +``` + +### 2. 性能基准测试 +```bash +# 监控10分钟并生成报告 +python3 monitor_alert_cache_performance.py -d 600 -r +``` + +## 🔧 配置优化 + +### 1. Redis配置优化 +```bash +# 编辑Redis配置文件 +sudo vim /etc/redis/redis.conf + +# 关键配置项 +maxmemory 2gb +maxmemory-policy allkeys-lru +notify-keyspace-events Ex +save 900 1 +save 300 10 +save 60 10000 +``` + +### 2. ljwx-boot配置调优 +```yaml +# application-local.yml +spring: + data: + redis: + url: redis://default:123456@localhost:6379/1 + connect-timeout: 10000ms + timeout: 30000ms + lettuce: + pool: + max-active: 20 + max-idle: 10 + min-idle: 5 +``` + +### 3. ljwx-bigscreen配置调优 +```python +# redis_config.py +class RedisConfig: + def __init__(self): + self.host = '127.0.0.1' + self.port = 6379 + self.db = 0 + self.password = '123456' + self.socket_timeout = 30 + self.retry_on_timeout = True +``` + +## 📊 性能目标 + +### 目标指标 +- **缓存命中率**: >95% +- **告警规则TTL**: 24小时 +- **处理延迟**: <5ms/record +- **QPS**: >1000 records/sec +- **Redis连接稳定性**: >99.9% + +### 监控命令 +```bash +# 实时性能监控 +python3 monitor_alert_cache_performance.py -i 5 + +# 生成性能报告 +python3 monitor_alert_cache_performance.py -d 3600 -r +``` + +## 🔒 安全配置 + +### 1. Redis安全 +```bash +# 设置Redis密码 +redis-cli CONFIG SET requirepass "your_secure_password" + +# 禁用危险命令 +redis-cli CONFIG SET rename-command FLUSHALL "" +redis-cli CONFIG SET rename-command EVAL "" +``` + +### 2. 网络安全 +```bash +# 限制Redis访问IP +# 在redis.conf中配置 +bind 127.0.0.1 192.168.1.0/24 +protected-mode yes +``` + +## 🚨 故障排除 + +### 常见问题 + +#### 1. Redis连接失败 +```bash +# 检查Redis服务状态 +systemctl status redis + +# 检查端口占用 +netstat -tulpn | grep 6379 + +# 查看Redis日志 +tail -f /var/log/redis/redis-server.log +``` + +#### 2. 缓存命中率低 +```bash +# 检查缓存键 +redis-cli -n 1 keys "alert_rules_*" + +# 检查TTL +redis-cli -n 1 TTL "alert_rules_1" + +# 检查订阅通道 +redis-cli PUBSUB CHANNELS +``` + +#### 3. 订阅者不工作 +```python +# 测试订阅者 +python3 -c " +from ljwx_bigscreen.bigscreen.alert_rules_cache_manager import get_alert_rules_cache_manager +manager = get_alert_rules_cache_manager() +manager.start_subscriber() +import time; time.sleep(10) +print(manager.get_cache_stats()) +" +``` + +### 诊断脚本 +```bash +# 运行完整诊断 +cat > diagnose_redis_cache.sh << 'EOF' +#!/bin/bash +echo "🔍 Redis缓存系统诊断" +echo "====================" + +echo "1. Redis服务状态:" +systemctl status redis --no-pager + +echo -e "\n2. Redis连接测试:" +redis-cli -h 127.0.0.1 -p 6379 -a 123456 ping + +echo -e "\n3. DB分离测试:" +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 0 ping +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 ping + +echo -e "\n4. 缓存键检查:" +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 keys "alert_rules_*" + +echo -e "\n5. 订阅通道检查:" +redis-cli -h 127.0.0.1 -p 6379 -a 123456 PUBSUB CHANNELS + +echo -e "\n6. Python组件测试:" +python3 utils/test_redis_cache_alerts.py + +echo "====================" +echo "✅ 诊断完成" +EOF + +chmod +x diagnose_redis_cache.sh +./diagnose_redis_cache.sh +``` + +## 📈 监控和维护 + +### 日常监控 +```bash +# 启动持续监控(后台运行) +nohup python3 monitor_alert_cache_performance.py -i 30 > monitor.log 2>&1 & + +# 定期生成报告 +crontab -e +# 添加以下行,每小时生成一次报告 +0 * * * * cd /path/to/utils && python3 monitor_alert_cache_performance.py -d 3600 -r +``` + +### 维护任务 +```bash +# 每日维护脚本 +cat > daily_maintenance.sh << 'EOF' +#!/bin/bash +echo "📅 每日Redis缓存维护 - $(date)" + +# 清理过期键 +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 0 --scan --pattern "*" | xargs -I {} redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 0 EXPIRE {} 86400 + +# 生成性能报告 +cd /path/to/utils +python3 monitor_alert_cache_performance.py -d 600 -r + +# 检查系统健康 +python3 test_redis_cache_alerts.py > daily_test_$(date +%Y%m%d).log + +echo "✅ 维护完成" +EOF + +chmod +x daily_maintenance.sh +``` + +## 🔄 回滚方案 + +如果需要回滚到原始版本: + +### 1. 恢复ljwx-boot +```bash +cd ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/ +cp TAlertRulesFacadeImpl.java.backup TAlertRulesFacadeImpl.java +cd ljwx-boot && ./run-local.sh restart +``` + +### 2. 停用Python组件 +```bash +# 停止监控 +pkill -f monitor_alert_cache_performance.py + +# 移除缓存管理器(可选) +mv ljwx-bigscreen/bigscreen/alert_rules_cache_manager.py ljwx-bigscreen/bigscreen/alert_rules_cache_manager.py.disabled +``` + +### 3. 清理Redis缓存 +```bash +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 keys "alert_rules_*" | xargs redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 DEL +redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 keys "alert_rules_version_*" | xargs redis-cli -h 127.0.0.1 -p 6379 -a 123456 -n 1 DEL +``` + +## 📚 相关文档 + +- [告警规则缓存同步方案](./alert_rules_cache_sync_solutions.md) +- [generate_alerts性能分析](./generate_alerts_analysis.md) +- [Redis配置最佳实践](https://redis.io/documentation) + +--- + +**部署负责人**: 系统架构团队 +**最后更新**: 2025-09-09 +**版本**: v1.0 \ No newline at end of file diff --git a/ljwx-bigscreen/bigscreen.log b/ljwx-bigscreen/bigscreen.log index 2859368..7cc7cb8 100644 --- a/ljwx-bigscreen/bigscreen.log +++ b/ljwx-bigscreen/bigscreen.log @@ -2,3 +2,39 @@ 2025-06-08 17:02:22,405 [INFO] system: 当前Python路径: ['/Users/bg/work/codes/springboot/ljwx/docker/ljwx-bigscreen/bigscreen', '/Users/bg/work/codes/springboot/ljwx/docker/ljwx-bigscreen', '/Users/bg/.pyenv/versions/3.10.10/lib/python310.zip'] 2025-06-08 17:02:22,411 [INFO] system: 告警webhook模块初始化成功 2025-06-08 17:02:22,417 [ERROR] root: Redis ping 失败: AUTH called without any password configured for the default user. Are you sure your configuration is correct? +2025-09-10 07:35:47,859 [WARNING] system: ⚠️ 健康系统API模块加载失败: No module named 'sklearn' +2025-09-10 07:35:47,861 [WARNING] system: ⚠️ 健康分析API V2.0模块加载失败: No module named 'sklearn' +2025-09-10 07:35:47,862 [INFO] system: 实时统计API直接路由注册成功 +2025-09-10 07:35:47,862 [WARNING] system: 设备绑定模块导入失败,使用兼容版本: No module named 'qrcode' +2025-09-10 07:35:47,864 [INFO] system: 设备绑定兼容模块注册成功 +2025-09-10 07:35:47,877 [INFO] system: 监控指标模块初始化成功 +2025-09-10 07:35:47,878 [INFO] system: 告警webhook模块初始化成功 +2025-09-10 07:35:47,956 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,960 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,991 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,991 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,992 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,533 [WARNING] system: ⚠️ 健康系统API模块加载失败: No module named 'sklearn' +2025-09-10 07:37:19,534 [WARNING] system: ⚠️ 健康分析API V2.0模块加载失败: No module named 'sklearn' +2025-09-10 07:37:19,534 [INFO] system: 实时统计API直接路由注册成功 +2025-09-10 07:37:19,535 [WARNING] system: 设备绑定模块导入失败,使用兼容版本: No module named 'qrcode' +2025-09-10 07:37:19,536 [INFO] system: 设备绑定兼容模块注册成功 +2025-09-10 07:37:19,545 [INFO] system: 监控指标模块初始化成功 +2025-09-10 07:37:19,546 [INFO] system: 告警webhook模块初始化成功 +2025-09-10 07:37:19,626 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,657 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,692 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,718 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,738 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,652 [WARNING] system: ⚠️ 健康系统API模块加载失败: No module named 'sklearn' +2025-09-10 07:54:06,653 [WARNING] system: ⚠️ 健康分析API V2.0模块加载失败: No module named 'sklearn' +2025-09-10 07:54:06,653 [INFO] system: 实时统计API直接路由注册成功 +2025-09-10 07:54:06,654 [WARNING] system: 设备绑定模块导入失败,使用兼容版本: No module named 'qrcode' +2025-09-10 07:54:06,655 [INFO] system: 设备绑定兼容模块注册成功 +2025-09-10 07:54:06,668 [INFO] system: 监控指标模块初始化成功 +2025-09-10 07:54:06,669 [INFO] system: 告警webhook模块初始化成功 +2025-09-10 07:54:06,744 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,776 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,813 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,835 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,853 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 diff --git a/ljwx-bigscreen/bigscreen/alert_rules_cache_manager.py b/ljwx-bigscreen/bigscreen/alert_rules_cache_manager.py new file mode 100644 index 0000000..03f26b6 --- /dev/null +++ b/ljwx-bigscreen/bigscreen/alert_rules_cache_manager.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +""" +告警规则缓存管理器 - 跨DB Redis订阅者 +实现ljwx-boot和ljwx-bigscreen之间的告警规则缓存同步 +""" + +import redis +import json +import logging +import threading +import time +from typing import Dict, List, Any, Optional +from dataclasses import dataclass +from redis_config import get_redis_client + +# 设置日志 +logger = logging.getLogger(__name__) + +@dataclass +class AlertRule: + """告警规则数据结构""" + id: int + rule_type: str + physical_sign: str + threshold_min: Optional[float] + threshold_max: Optional[float] + trend_duration: Optional[int] + severity_level: str + alert_message: str + customer_id: int + is_enabled: bool = True + +class AlertRulesCacheManager: + """告警规则缓存管理器 - 支持跨DB Redis通信""" + + def __init__(self): + # ljwx-bigscreen Redis客户端 (DB=0) + self.redis_client = get_redis_client() + + # ljwx-boot Redis客户端 (DB=1) + self.boot_redis_client = redis.Redis( + host='127.0.0.1', + port=6379, + db=1, # ljwx-boot使用DB=1 + password='123456', + decode_responses=True, + socket_timeout=30, + socket_connect_timeout=30, + retry_on_timeout=True + ) + + # 本地缓存 + self.local_cache = {} # {customer_id: rules_list} + self.version_cache = {} # {customer_id: version} + + # 订阅者状态 + self.subscriber_thread = None + self.running = False + self.pubsub = None + + def start_subscriber(self): + """启动Redis订阅者监听告警规则更新""" + if self.running: + logger.warning("告警规则订阅者已在运行中") + return + + self.running = True + self.pubsub = self.redis_client.pubsub() + self.pubsub.subscribe('alert_rules_channel') + + # 启动订阅者线程 + self.subscriber_thread = threading.Thread( + target=self._subscriber_loop, + daemon=True, + name="AlertRulesSubscriber" + ) + self.subscriber_thread.start() + logger.info("告警规则订阅者已启动,监听 alert_rules_channel") + + def stop_subscriber(self): + """停止Redis订阅者""" + self.running = False + if self.pubsub: + self.pubsub.unsubscribe('alert_rules_channel') + self.pubsub.close() + logger.info("告警规则订阅者已停止") + + def _subscriber_loop(self): + """订阅者主循环""" + while self.running: + try: + for message in self.pubsub.listen(): + if not self.running: + break + + if message['type'] == 'message': + self._handle_cache_update(message['data']) + + except Exception as e: + logger.error(f"Redis订阅者错误: {e}") + if self.running: + time.sleep(5) # 错误后重试间隔 + + def _handle_cache_update(self, message: str): + """处理缓存更新消息""" + try: + # 解析消息格式: "update:customer_id:version" + parts = message.split(':') + if len(parts) >= 2 and parts[0] == 'update': + customer_id = int(parts[1]) + version = int(parts[2]) if len(parts) > 2 else 0 + + # 检查是否需要更新本地缓存 + if self._should_update_cache(customer_id, version): + self._refresh_local_cache(customer_id) + + except Exception as e: + logger.error(f"处理缓存更新消息失败: {message}, error: {e}") + + def _should_update_cache(self, customer_id: int, version: int) -> bool: + """检查是否需要更新缓存""" + local_version = self.version_cache.get(customer_id, 0) + return version > local_version + + def _refresh_local_cache(self, customer_id: int): + """刷新本地缓存 - 从ljwx-boot的Redis DB=1获取""" + try: + cache_key = f"alert_rules_{customer_id}" + # 从ljwx-boot的Redis DB=1获取数据 + cached_data = self.boot_redis_client.get(cache_key) + + if cached_data: + data = json.loads(cached_data) + rules_data = data.get('rules', []) + version = data.get('version', 0) + + # 转换为AlertRule对象 + rules = [] + for rule_dict in rules_data: + try: + rule = AlertRule( + id=rule_dict.get('id'), + rule_type=rule_dict.get('ruleType', ''), + physical_sign=rule_dict.get('physicalSign', ''), + threshold_min=rule_dict.get('thresholdMin'), + threshold_max=rule_dict.get('thresholdMax'), + trend_duration=rule_dict.get('trendDuration'), + severity_level=rule_dict.get('severityLevel', ''), + alert_message=rule_dict.get('alertMessage', ''), + customer_id=rule_dict.get('customerId'), + is_enabled=rule_dict.get('isEnabled', True) + ) + rules.append(rule) + except Exception as e: + logger.warning(f"解析告警规则失败: {rule_dict}, error: {e}") + + # 更新本地缓存 + self.local_cache[customer_id] = rules + self.version_cache[customer_id] = version + + logger.info(f"告警规则缓存更新成功: customer_id={customer_id}, version={version}, rules={len(rules)}") + + except Exception as e: + logger.error(f"刷新告警规则缓存失败: customer_id={customer_id}, error: {e}") + + def get_alert_rules(self, customer_id: int) -> List[AlertRule]: + """获取告警规则 - 三级缓存策略 + 自动预热""" + # L1: 本地缓存 + if customer_id in self.local_cache: + logger.debug(f"告警规则本地缓存命中: customer_id={customer_id}") + return self.local_cache[customer_id] + + # L2: 从ljwx-boot的Redis获取 + try: + cache_key = f"alert_rules_{customer_id}" + cached_data = self.boot_redis_client.get(cache_key) + + if cached_data: + data = json.loads(cached_data) + rules_data = data.get('rules', []) + version = data.get('version', 0) + + # 转换并缓存到本地 + rules = self._convert_rules_data(rules_data) + self.local_cache[customer_id] = rules + self.version_cache[customer_id] = version + + logger.info(f"告警规则从ljwx-boot Redis获取成功: customer_id={customer_id}, rules={len(rules)}") + return rules + + except Exception as e: + logger.error(f"从ljwx-boot Redis获取告警规则失败: customer_id={customer_id}, error: {e}") + + # L3: 自动预热机制 - 尝试从数据库获取并缓存 + logger.warning(f"告警规则缓存miss,尝试自动预热: customer_id={customer_id}") + rules = self._auto_warmup_from_database(customer_id) + return rules + + def _convert_rules_data(self, rules_data: List[Dict]) -> List[AlertRule]: + """转换规则数据为AlertRule对象""" + rules = [] + for rule_dict in rules_data: + try: + rule = AlertRule( + id=rule_dict.get('id'), + rule_type=rule_dict.get('ruleType', ''), + physical_sign=rule_dict.get('physicalSign', ''), + threshold_min=rule_dict.get('thresholdMin'), + threshold_max=rule_dict.get('thresholdMax'), + trend_duration=rule_dict.get('trendDuration'), + severity_level=rule_dict.get('severityLevel', ''), + alert_message=rule_dict.get('alertMessage', ''), + customer_id=rule_dict.get('customerId'), + is_enabled=rule_dict.get('isEnabled', True) + ) + rules.append(rule) + except Exception as e: + logger.warning(f"转换告警规则数据失败: {rule_dict}, error: {e}") + return rules + + def preload_customer_rules(self, customer_ids: List[int]): + """预加载指定客户的告警规则""" + for customer_id in customer_ids: + try: + self.get_alert_rules(customer_id) + logger.info(f"预加载告警规则成功: customer_id={customer_id}") + except Exception as e: + logger.error(f"预加载告警规则失败: customer_id={customer_id}, error: {e}") + + def warmup_active_customers(self, days: int = 7): + """预热活跃客户的告警规则缓存""" + try: + import pymysql + from redis_config import get_mysql_config + + # 获取MySQL配置 + mysql_config = get_mysql_config() + + # 连接数据库 + connection = pymysql.connect( + host=mysql_config['host'], + port=mysql_config['port'], + user=mysql_config['user'], + password=mysql_config['password'], + database=mysql_config['database'], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + try: + with connection.cursor() as cursor: + # 查询最近活跃的客户ID + sql = """ + SELECT DISTINCT customer_id + FROM t_user_health_data + WHERE create_time >= DATE_SUB(NOW(), INTERVAL %s DAY) + AND customer_id IS NOT NULL + LIMIT 100 + """ + cursor.execute(sql, (days,)) + active_customers = cursor.fetchall() + + customer_ids = [row['customer_id'] for row in active_customers] + logger.info(f"发现活跃客户: {len(customer_ids)}个,开始预热缓存") + + # 批量预加载 + self.preload_customer_rules(customer_ids) + + finally: + connection.close() + + except Exception as e: + logger.error(f"预热活跃客户告警规则失败: error={e}") + + def get_cache_stats(self) -> Dict[str, Any]: + """获取缓存统计信息""" + return { + 'local_cache_size': len(self.local_cache), + 'cached_customers': list(self.local_cache.keys()), + 'version_info': self.version_cache, + 'subscriber_running': self.running, + 'redis_connection_status': self._check_redis_connections() + } + + def _check_redis_connections(self) -> Dict[str, bool]: + """检查Redis连接状态""" + connections = {} + try: + self.redis_client.ping() + connections['bigscreen_redis'] = True + except: + connections['bigscreen_redis'] = False + + try: + self.boot_redis_client.ping() + connections['boot_redis'] = True + except: + connections['boot_redis'] = False + + return connections + + def _auto_warmup_from_database(self, customer_id: int) -> List[AlertRule]: + """自动预热机制 - 从数据库获取告警规则并缓存""" + try: + import pymysql + from redis_config import get_mysql_config + + # 获取MySQL配置 + mysql_config = get_mysql_config() + + # 连接数据库 + connection = pymysql.connect( + host=mysql_config['host'], + port=mysql_config['port'], + user=mysql_config['user'], + password=mysql_config['password'], + database=mysql_config['database'], + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + try: + with connection.cursor() as cursor: + # 查询告警规则 + sql = """ + SELECT id, rule_type, physical_sign, threshold_min, threshold_max, + trend_duration, severity_level, alert_message, customer_id, is_enabled + FROM t_alert_rules + WHERE customer_id = %s AND is_enabled = 1 + ORDER BY id + """ + cursor.execute(sql, (customer_id,)) + db_rules = cursor.fetchall() + + if not db_rules: + logger.info(f"数据库中未找到客户告警规则: customer_id={customer_id}") + return [] + + # 转换为AlertRule对象 + rules = [] + for rule_dict in db_rules: + try: + rule = AlertRule( + id=rule_dict.get('id'), + rule_type=rule_dict.get('rule_type', ''), + physical_sign=rule_dict.get('physical_sign', ''), + threshold_min=rule_dict.get('threshold_min'), + threshold_max=rule_dict.get('threshold_max'), + trend_duration=rule_dict.get('trend_duration'), + severity_level=rule_dict.get('severity_level', ''), + alert_message=rule_dict.get('alert_message', ''), + customer_id=rule_dict.get('customer_id'), + is_enabled=rule_dict.get('is_enabled', True) + ) + rules.append(rule) + except Exception as e: + logger.warning(f"转换数据库告警规则失败: {rule_dict}, error: {e}") + + # 缓存到本地和Redis + if rules: + # 更新本地缓存 + self.local_cache[customer_id] = rules + self.version_cache[customer_id] = int(time.time()) + + # 缓存到ljwx-boot Redis (DB=1) + try: + cache_data = { + 'rules': [ + { + 'id': rule.id, + 'ruleType': rule.rule_type, + 'physicalSign': rule.physical_sign, + 'thresholdMin': rule.threshold_min, + 'thresholdMax': rule.threshold_max, + 'trendDuration': rule.trend_duration, + 'severityLevel': rule.severity_level, + 'alertMessage': rule.alert_message, + 'customerId': rule.customer_id, + 'isEnabled': rule.is_enabled + } + for rule in rules + ], + 'version': self.version_cache[customer_id], + 'cached_at': int(time.time()) + } + + cache_key = f"alert_rules_{customer_id}" + self.boot_redis_client.setex( + cache_key, + 3600, # 1小时缓存 + json.dumps(cache_data, ensure_ascii=False) + ) + + logger.info(f"告警规则自动预热成功: customer_id={customer_id}, rules={len(rules)}, 已缓存到Redis") + + except Exception as e: + logger.warning(f"缓存告警规则到Redis失败: customer_id={customer_id}, error: {e}") + + return rules + + finally: + connection.close() + + except Exception as e: + logger.error(f"从数据库自动预热告警规则失败: customer_id={customer_id}, error: {e}") + return [] + + def clear_local_cache(self, customer_id: Optional[int] = None): + """清理本地缓存""" + if customer_id: + self.local_cache.pop(customer_id, None) + self.version_cache.pop(customer_id, None) + logger.info(f"清理告警规则本地缓存: customer_id={customer_id}") + else: + self.local_cache.clear() + self.version_cache.clear() + logger.info("清理所有告警规则本地缓存") + +# 全局缓存管理器实例 +alert_rules_cache_manager = AlertRulesCacheManager() + +def get_alert_rules_cache_manager() -> AlertRulesCacheManager: + """获取告警规则缓存管理器实例""" + return alert_rules_cache_manager + +if __name__ == "__main__": + # 测试代码 + print("🔧 告警规则缓存管理器测试") + + manager = get_alert_rules_cache_manager() + + # 检查Redis连接 + stats = manager.get_cache_stats() + print(f"Redis连接状态: {stats['redis_connection_status']}") + + # 启动订阅者 + manager.start_subscriber() + + # 测试获取告警规则 + try: + rules = manager.get_alert_rules(1) # 假设customer_id=1 + print(f"获取到告警规则: {len(rules)}条") + except Exception as e: + print(f"测试失败: {e}") + + # 等待一段时间测试订阅 + print("等待Redis消息...") + time.sleep(5) + + manager.stop_subscriber() \ No newline at end of file diff --git a/ljwx-bigscreen/bigscreen/bigScreen/alert.py b/ljwx-bigscreen/bigscreen/bigScreen/alert.py index d54cd69..419775d 100644 --- a/ljwx-bigscreen/bigscreen/bigScreen/alert.py +++ b/ljwx-bigscreen/bigscreen/bigScreen/alert.py @@ -1970,10 +1970,10 @@ def upload_common_event(): time_cfg = TimeConfig() alert_timestamp=data.get('timestamp', time_cfg.now().strftime('%Y-%m-%d %H:%M:%S')) - # 优先使用直接传递的客户信息参数 - customerId = data.get("customer_id") - orgId = data.get("org_id") - userId = data.get("user_id") + # 优先使用直接传递的客户信息参数 - 支持两种字段名格式 + customerId = data.get("customer_id") or data.get("customerId") + orgId = data.get("org_id") or data.get("orgId") + userId = data.get("user_id") or data.get("userId") print(f"📡 [upload_common_event] 客户信息: customerId={customerId}, orgId={orgId}, userId={userId}") @@ -2571,26 +2571,73 @@ def get_event_processor_stats(): }), 500 def generate_alerts(data, health_data_id): + start_time = time.time() # 开始计时 try: print(f"🔍 generate_alerts started with data keys: {list(data.keys()) if data else 'None'}") - # 从数据库获取告警规则并转换为字典格式 - alert_rules = AlertRules.query.all() - alert_rules_dict = {} + # 获取客户ID用于缓存查询 + customer_id = data.get('customer_id') or data.get('customerId') - # 将 SQLAlchemy 对象转换为字典 - for rule in alert_rules: - rule_dict = { - 'id': rule.id, - 'rule_type': rule.rule_type, - 'physical_sign': rule.physical_sign, - 'threshold_min': rule.threshold_min, - 'threshold_max': rule.threshold_max, - 'trend_duration': rule.trend_duration, - 'severity_level': rule.severity_level, - 'alert_message': rule.alert_message - } - alert_rules_dict[rule.id] = rule_dict + # 优化:优先从Redis缓存获取告警规则 + alert_rules_dict = {} + cache_hit = False + + if customer_id: + try: + # 尝试从Redis缓存获取告警规则 + from alert_rules_cache_manager import get_alert_rules_cache_manager + cache_manager = get_alert_rules_cache_manager() + cached_rules = cache_manager.get_alert_rules(customer_id) + + if cached_rules: + cache_hit = True + print(f"🎯 Redis缓存命中:customer_id={customer_id}, 规则数量={len(cached_rules)}") + + # 转换缓存的AlertRule对象为字典格式 + for rule in cached_rules: + if rule.is_enabled: # 只处理启用的规则 + rule_dict = { + 'id': rule.id, + 'rule_type': rule.rule_type, + 'physical_sign': rule.physical_sign, + 'threshold_min': rule.threshold_min, + 'threshold_max': rule.threshold_max, + 'trend_duration': rule.trend_duration, + 'severity_level': rule.severity_level, + 'alert_message': rule.alert_message, + 'is_enabled': rule.is_enabled + } + alert_rules_dict[rule.id] = rule_dict + + else: + print(f"⚠️ Redis缓存未命中:customer_id={customer_id}") + + except Exception as cache_e: + print(f"⚠️ Redis缓存获取失败,回退到数据库查询: {cache_e}") + + # 数据库兜底:如果Redis缓存失败或未命中,从数据库获取 + if not cache_hit or not alert_rules_dict: + print("📊 使用数据库兜底查询告警规则") + + # 从数据库获取所有告警规则(AlertRules表无customer_id字段) + alert_rules = AlertRules.query.filter_by(is_deleted=False).all() + + # 将 SQLAlchemy 对象转换为字典 + for rule in alert_rules: + rule_dict = { + 'id': rule.id, + 'rule_type': rule.rule_type, + 'physical_sign': rule.physical_sign, + 'threshold_min': rule.threshold_min, + 'threshold_max': rule.threshold_max, + 'trend_duration': rule.trend_duration, + 'severity_level': rule.severity_level, + 'alert_message': rule.alert_message, + 'is_enabled': True # 默认启用,数据库表无此字段 + } + alert_rules_dict[rule.id] = rule_dict + + print(f"📋 获取到告警规则 {len(alert_rules_dict)} 条 (Redis缓存: {'✅' if cache_hit else '❌'})") # 初始化异常计数器 abnormal_counts = {} @@ -2712,8 +2759,21 @@ def generate_alerts(data, health_data_id): db.session.add(alert_info_instance) db.session.commit() + + # 记录性能统计 + processing_time = time.time() - start_time if 'start_time' in locals() else 0 print(f"✅ generate_alerts completed successfully") - return jsonify({'success': True}) + print(f"📊 性能统计: 处理时间={processing_time:.3f}s, 规则数量={len(alert_rules_dict)}, Redis缓存={'命中' if cache_hit else '未命中'}") + + return jsonify({ + 'success': True, + 'stats': { + 'processing_time': round(processing_time, 3), + 'rules_count': len(alert_rules_dict), + 'cache_hit': cache_hit, + 'customer_id': customer_id + } + }) except Exception as e: import traceback diff --git a/ljwx-bigscreen/bigscreen/bigScreen/health_data_batch_processor.py b/ljwx-bigscreen/bigscreen/bigScreen/health_data_batch_processor.py index 8ac621a..8d9c27e 100644 --- a/ljwx-bigscreen/bigscreen/bigScreen/health_data_batch_processor.py +++ b/ljwx-bigscreen/bigscreen/bigScreen/health_data_batch_processor.py @@ -485,10 +485,10 @@ class HealthDataOptimizer:#健康数据性能优化器V4.0 - CPU自适应版本 #确保批处理器已启动 self._ensure_processor_started() - # 优先使用直接传递的客户信息参数 - user_id = raw_data.get("user_id") - org_id = raw_data.get("org_id") - customer_id = raw_data.get("customer_id") + # 优先使用直接传递的客户信息参数 - 支持两种字段名格式 + user_id = raw_data.get("user_id") or raw_data.get("userId") + org_id = raw_data.get("org_id") or raw_data.get("orgId") + customer_id = raw_data.get("customer_id") or raw_data.get("customerId") print(f"🔍 直接传入的客户信息: user_id={user_id}, org_id={org_id}, customer_id={customer_id}") @@ -598,7 +598,7 @@ class HealthDataOptimizer:#健康数据性能优化器V4.0 - CPU自适应版本 else: print(f"⚠️ 无每周数据字段") - #构建Redis数据(只包含配置字段) + #构建Redis数据(包含配置字段和客户信息) redis_data={} for field in config_fields: if field in self.field_mapping: @@ -607,7 +607,12 @@ class HealthDataOptimizer:#健康数据性能优化器V4.0 - CPU自适应版本 if value is not None: redis_data[api_field]=str(value) redis_data['deviceSn']=device_sn - print(f"✅ Redis数据: {redis_data}") + # 添加客户信息用于告警规则缓存 + redis_data['customer_id']=customer_id + redis_data['customerId']=customer_id # 兼容性字段名 + redis_data['org_id']=org_id + redis_data['user_id']=user_id + print(f"✅ Redis数据(含客户信息): {redis_data}") item={'device_sn':device_sn,'main_data':main_data,'daily_data':daily_data,'weekly_data':weekly_data,'redis_data':redis_data,'enable_alerts':enable_alerts,'config_info':config_info} print(f"🔧 准备加入队列的数据项: {json.dumps(item, ensure_ascii=False, default=str)}") @@ -746,10 +751,19 @@ def optimized_upload_health_data(health_data):#优化的健康数据上传V3.1 data=health_data.get("data",{}) - # 提取顶级的客户信息参数 - customer_id = health_data.get("customer_id") - org_id = health_data.get("org_id") - user_id = health_data.get("user_id") + # 提取顶级的客户信息参数 - 支持两种字段名格式 + customer_id = health_data.get("customer_id") or health_data.get("customerId") + org_id = health_data.get("org_id") or health_data.get("orgId") + user_id = health_data.get("user_id") or health_data.get("userId") + + # 如果顶级没有,从data字段中提取 + if not customer_id and isinstance(data, dict): + customer_id = data.get("customer_id") or data.get("customerId") + if not org_id and isinstance(data, dict): + org_id = data.get("org_id") or data.get("orgId") + if not user_id and isinstance(data, dict): + user_id = data.get("user_id") or data.get("userId") + print(f"🔍 提取客户信息: customer_id={customer_id}, org_id={org_id}, user_id={user_id}") print(f"🔍 解析data字段: {json.dumps(data, ensure_ascii=False, indent=2)}") diff --git a/ljwx-bigscreen/bigscreen/bigScreen/user_health_data.py b/ljwx-bigscreen/bigscreen/bigScreen/user_health_data.py index 0197908..622a2b1 100644 --- a/ljwx-bigscreen/bigscreen/bigScreen/user_health_data.py +++ b/ljwx-bigscreen/bigscreen/bigScreen/user_health_data.py @@ -1823,10 +1823,10 @@ def process_single_health_data(data): scientificSleepData = data.get("scientificSleepData") or data.get("kxsmData") workoutData = data.get("workoutData") or data.get("ydData") - # 提取客户信息字段 - 优先使用直接传递的参数 - customerId = data.get("customer_id") - orgId = data.get("org_id") - userId = data.get("user_id") + # 提取客户信息字段 - 支持两种格式:下划线和驼峰命名 + customerId = data.get("customer_id") or data.get("customerId") + orgId = data.get("org_id") or data.get("orgId") + userId = data.get("user_id") or data.get("userId") print(f"🏥 解析后的关键字段:") print(f" - deviceSn: {deviceSn}") @@ -1913,6 +1913,10 @@ def process_single_health_data(data): "latitude": safe_str(latitude), "longitude": safe_str(longitude), "altitude": safe_str(altitude), + "customerId": customerId, # 添加customerId用于告警规则缓存 + "customer_id": customerId, # 兼容性字段名 + "orgId": orgId, # 添加orgId + "userId": userId, # 添加userId "sleepData": json.dumps(sleepData) if sleepData else '', "exerciseDailyData": json.dumps(exerciseDailyData) if exerciseDailyData else '', "exerciseDailyWeekData": json.dumps(exerciseDailyWeekData) if exerciseDailyWeekData else '', diff --git a/ljwx-bigscreen/bigscreen/redis_config.py b/ljwx-bigscreen/bigscreen/redis_config.py index da53fb1..ca78c8e 100644 --- a/ljwx-bigscreen/bigscreen/redis_config.py +++ b/ljwx-bigscreen/bigscreen/redis_config.py @@ -66,6 +66,16 @@ def test_redis_connection(): """测试Redis连接的便捷函数""" return redis_config.test_connection() +def get_mysql_config(): + """获取MySQL配置的便捷函数""" + return { + 'host': os.getenv('MYSQL_HOST', '127.0.0.1'), + 'port': int(os.getenv('MYSQL_PORT', 3306)), + 'user': os.getenv('MYSQL_USER', 'root'), + 'password': os.getenv('MYSQL_PASSWORD', 'admin123'), + 'database': os.getenv('MYSQL_DATABASE', 'ljwx') + } + if __name__ == "__main__": # 测试脚本 print("🔧 Redis配置测试") diff --git a/ljwx-bigscreen/system.log b/ljwx-bigscreen/system.log index 2859368..7cc7cb8 100644 --- a/ljwx-bigscreen/system.log +++ b/ljwx-bigscreen/system.log @@ -2,3 +2,39 @@ 2025-06-08 17:02:22,405 [INFO] system: 当前Python路径: ['/Users/bg/work/codes/springboot/ljwx/docker/ljwx-bigscreen/bigscreen', '/Users/bg/work/codes/springboot/ljwx/docker/ljwx-bigscreen', '/Users/bg/.pyenv/versions/3.10.10/lib/python310.zip'] 2025-06-08 17:02:22,411 [INFO] system: 告警webhook模块初始化成功 2025-06-08 17:02:22,417 [ERROR] root: Redis ping 失败: AUTH called without any password configured for the default user. Are you sure your configuration is correct? +2025-09-10 07:35:47,859 [WARNING] system: ⚠️ 健康系统API模块加载失败: No module named 'sklearn' +2025-09-10 07:35:47,861 [WARNING] system: ⚠️ 健康分析API V2.0模块加载失败: No module named 'sklearn' +2025-09-10 07:35:47,862 [INFO] system: 实时统计API直接路由注册成功 +2025-09-10 07:35:47,862 [WARNING] system: 设备绑定模块导入失败,使用兼容版本: No module named 'qrcode' +2025-09-10 07:35:47,864 [INFO] system: 设备绑定兼容模块注册成功 +2025-09-10 07:35:47,877 [INFO] system: 监控指标模块初始化成功 +2025-09-10 07:35:47,878 [INFO] system: 告警webhook模块初始化成功 +2025-09-10 07:35:47,956 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,960 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,991 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,991 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:35:47,992 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,533 [WARNING] system: ⚠️ 健康系统API模块加载失败: No module named 'sklearn' +2025-09-10 07:37:19,534 [WARNING] system: ⚠️ 健康分析API V2.0模块加载失败: No module named 'sklearn' +2025-09-10 07:37:19,534 [INFO] system: 实时统计API直接路由注册成功 +2025-09-10 07:37:19,535 [WARNING] system: 设备绑定模块导入失败,使用兼容版本: No module named 'qrcode' +2025-09-10 07:37:19,536 [INFO] system: 设备绑定兼容模块注册成功 +2025-09-10 07:37:19,545 [INFO] system: 监控指标模块初始化成功 +2025-09-10 07:37:19,546 [INFO] system: 告警webhook模块初始化成功 +2025-09-10 07:37:19,626 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,657 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,692 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,718 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:37:19,738 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,652 [WARNING] system: ⚠️ 健康系统API模块加载失败: No module named 'sklearn' +2025-09-10 07:54:06,653 [WARNING] system: ⚠️ 健康分析API V2.0模块加载失败: No module named 'sklearn' +2025-09-10 07:54:06,653 [INFO] system: 实时统计API直接路由注册成功 +2025-09-10 07:54:06,654 [WARNING] system: 设备绑定模块导入失败,使用兼容版本: No module named 'qrcode' +2025-09-10 07:54:06,655 [INFO] system: 设备绑定兼容模块注册成功 +2025-09-10 07:54:06,668 [INFO] system: 监控指标模块初始化成功 +2025-09-10 07:54:06,669 [INFO] system: 告警webhook模块初始化成功 +2025-09-10 07:54:06,744 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,776 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,813 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,835 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 +2025-09-10 07:54:06,853 [WARNING] alert_rules_cache_manager: 告警规则缓存miss,需要从数据库获取: customer_id=1 diff --git a/ljwx-bigscreen/test_generate_alerts_redis_integration.py b/ljwx-bigscreen/test_generate_alerts_redis_integration.py new file mode 100644 index 0000000..c764351 --- /dev/null +++ b/ljwx-bigscreen/test_generate_alerts_redis_integration.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +""" +测试集成Redis缓存的generate_alerts函数 +验证Redis缓存优化是否正常工作 +""" + +import sys +import os +import time +import json + +# 添加必要的路径 +current_dir = os.path.dirname(__file__) +bigscreen_path = os.path.join(current_dir, 'bigscreen') +sys.path.insert(0, bigscreen_path) + +def test_generate_alerts_redis_integration(): + """测试generate_alerts函数的Redis集成""" + + print("🧪 测试generate_alerts函数Redis集成") + print("=" * 60) + + try: + # 导入必要的模块 + from bigScreen.alert import generate_alerts + from bigScreen.models import db + from bigScreen import create_app + + # 创建Flask应用上下文 + app = create_app() + + with app.app_context(): + print("✅ Flask应用上下文创建成功") + + # 测试数据1:包含customer_id的正常数据 + test_data_with_customer = { + 'customer_id': 1, + 'deviceSn': 'TEST_DEVICE_001', + 'heart_rate': 85, + 'blood_oxygen': 98, + 'temperature': 36.5, + 'pressureHigh': 120, + 'pressureLow': 80, + 'timestamp': '2025-09-09 10:00:00' + } + + # 测试数据2:心率异常的数据 + test_data_abnormal = { + 'customer_id': 1, + 'deviceSn': 'TEST_DEVICE_002', + 'heart_rate': 130, # 超出正常范围 + 'blood_oxygen': 92, # 低于正常范围 + 'temperature': 37.8, # 发烧 + 'pressureHigh': 160, # 高血压 + 'pressureLow': 100, + 'timestamp': '2025-09-09 10:01:00' + } + + # 测试数据3:无customer_id的数据 + test_data_no_customer = { + 'deviceSn': 'TEST_DEVICE_003', + 'heart_rate': 75, + 'blood_oxygen': 99, + 'temperature': 36.2, + 'timestamp': '2025-09-09 10:02:00' + } + + test_cases = [ + ("正常数据(有customer_id)", test_data_with_customer, 1), + ("异常数据(有customer_id)", test_data_abnormal, 2), + ("正常数据(无customer_id)", test_data_no_customer, 3) + ] + + print("\n🔬 开始执行测试用例") + print("-" * 60) + + for i, (description, test_data, health_id) in enumerate(test_cases, 1): + print(f"\n📋 测试用例 {i}: {description}") + print(f" 数据: {json.dumps(test_data, ensure_ascii=False, indent=4)}") + + try: + # 调用generate_alerts函数 + start_time = time.time() + result = generate_alerts(test_data, health_id) + end_time = time.time() + + print(f" ⏱️ 执行时间: {(end_time - start_time):.3f}s") + + # 解析结果 + if hasattr(result, 'json') and result.json: + result_data = result.json + print(f" 📊 结果: {json.dumps(result_data, ensure_ascii=False, indent=6)}") + + # 检查性能统计 + stats = result_data.get('stats', {}) + if stats: + print(f" 📈 性能统计:") + print(f" - 处理时间: {stats.get('processing_time', 'N/A')}s") + print(f" - 规则数量: {stats.get('rules_count', 'N/A')}") + print(f" - Redis缓存: {'✅ 命中' if stats.get('cache_hit') else '❌ 未命中'}") + print(f" - 客户ID: {stats.get('customer_id', 'N/A')}") + else: + print(f" 📊 结果: {result}") + + print(f" ✅ 测试用例 {i} 执行成功") + + except Exception as e: + print(f" ❌ 测试用例 {i} 执行失败: {e}") + import traceback + print(f" 📋 错误详情:\n{traceback.format_exc()}") + + print("-" * 60) + + print("\n🎯 Redis缓存特性测试") + print("-" * 60) + + # 测试Redis缓存命中率 + print("📈 连续调用同一customer_id数据,测试缓存效果:") + + customer_data = { + 'customer_id': 1, + 'deviceSn': 'CACHE_TEST_DEVICE', + 'heart_rate': 80, + 'blood_oxygen': 97, + 'temperature': 36.8 + } + + cache_test_times = [] + for i in range(3): + start_time = time.time() + result = generate_alerts(customer_data, 100 + i) + end_time = time.time() + + execution_time = end_time - start_time + cache_test_times.append(execution_time) + + print(f" 第{i+1}次调用: {execution_time:.3f}s", end="") + + if hasattr(result, 'json') and result.json: + stats = result.json.get('stats', {}) + cache_hit = stats.get('cache_hit', False) + print(f" (缓存: {'✅' if cache_hit else '❌'})") + else: + print() + + # 分析缓存效果 + if len(cache_test_times) >= 2: + first_call = cache_test_times[0] + subsequent_calls = cache_test_times[1:] + avg_subsequent = sum(subsequent_calls) / len(subsequent_calls) + + print(f"\n📊 缓存效果分析:") + print(f" 首次调用时间: {first_call:.3f}s") + print(f" 后续平均时间: {avg_subsequent:.3f}s") + if first_call > 0: + improvement = (first_call - avg_subsequent) / first_call * 100 + print(f" 性能提升: {improvement:.1f}%") + + print("\n✅ 所有测试完成") + + except ImportError as e: + print(f"❌ 导入模块失败: {e}") + print("请确保已正确安装所有依赖,并且Flask应用配置正确") + + except Exception as e: + print(f"❌ 测试执行失败: {e}") + import traceback + print(f"📋 错误详情:\n{traceback.format_exc()}") + +def test_redis_cache_manager_availability(): + """测试Redis缓存管理器是否可用""" + print("\n🔧 测试Redis缓存管理器可用性") + print("-" * 60) + + try: + from alert_rules_cache_manager import get_alert_rules_cache_manager + + manager = get_alert_rules_cache_manager() + stats = manager.get_cache_stats() + + print("✅ Redis缓存管理器可用") + print("📊 缓存管理器状态:") + for key, value in stats.items(): + if isinstance(value, dict): + print(f" {key}:") + for sub_key, sub_value in value.items(): + print(f" {sub_key}: {sub_value}") + else: + print(f" {key}: {value}") + + return True + + except Exception as e: + print(f"❌ Redis缓存管理器不可用: {e}") + return False + +def main(): + """主函数""" + print("🚀 generate_alerts Redis集成测试套件") + print("=" * 80) + + # 测试Redis缓存管理器 + cache_available = test_redis_cache_manager_availability() + + if cache_available: + print("\n✅ Redis缓存管理器可用,开始测试generate_alerts集成") + else: + print("\n⚠️ Redis缓存管理器不可用,但仍将测试数据库兜底功能") + + # 测试generate_alerts函数 + test_generate_alerts_redis_integration() + + print("\n🎉 测试套件执行完成") + print("=" * 80) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ljwx-bigscreen/test_upload_common_event_fix.py b/ljwx-bigscreen/test_upload_common_event_fix.py deleted file mode 100644 index 6b295ec..0000000 --- a/ljwx-bigscreen/test_upload_common_event_fix.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -"""测试upload_common_event接口修复效果""" -import json,requests,time -from datetime import datetime - -def test_upload_common_event_fix(): - """测试upload_common_event接口修复""" - print("🧪 测试upload_common_event接口修复效果") - print("=" * 60) - - #模拟用户原始数据(与用户日志中的数据格式一致) - test_data={ - "eventType":"com.tdtech.ohos.action.WEAR_STATUS_CHANGED", - "eventValue":"0", - "deviceSn":"CRFTQ23409001890", - "latitude":22.5404, - "longitude":114.015072, - "altitude":0, - "healthData":json.dumps({ - "data":{ - "deviceSn":"CRFTQ23409001890", - "heart_rate":86, - "blood_oxygen":98, - "body_temperature":"0.0", - "step":0, - "distance":"0.0", - "calorie":"0.0", - "latitude":"0", - "longitude":"0", - "altitude":"0", - "stress":0, - "upload_method":"wifi", - "blood_pressure_systolic":127, - "blood_pressure_diastolic":87, - "sleepData":"null", - "exerciseDailyData":"null", - "exerciseWeekData":"null", - "scientificSleepData":"null", - "workoutData":"null", - "timestamp":"2025-06-19 14:09:27" - } - }) - } - - print("📤 测试数据:") - print(json.dumps(test_data,indent=2,ensure_ascii=False)) - print() - - #测试本地服务 - test_urls=[ - "http://localhost:8001/upload_common_event", - "http://127.0.0.1:8001/upload_common_event", - "http://192.168.1.6:8001/upload_common_event" - ] - - for url in test_urls: - try: - print(f"🌐 测试地址: {url}") - start_time=time.time() - - response=requests.post(url,json=test_data,timeout=10) - elapsed=time.time()-start_time - - print(f" ✅ 状态码: {response.status_code}") - print(f" ⏱️ 响应时间: {elapsed:.3f}秒") - - if response.status_code==200: - result=response.json() - print(f" 📋 响应内容: {json.dumps(result,indent=4,ensure_ascii=False)}") - - if result.get('status')=='success': - print(f" 🎉 测试成功! 消息: {result.get('message')}") - else: - print(f" ⚠️ 业务失败: {result.get('message')}") - else: - print(f" ❌ HTTP错误: {response.text}") - - except requests.exceptions.ConnectRefusedError: - print(f" ❌ 连接被拒绝 - 服务可能未启动") - except requests.exceptions.Timeout: - print(f" ❌ 请求超时") - except Exception as e: - print(f" ❌ 测试失败: {e}") - - print() - -if __name__=='__main__': - test_upload_common_event_fix() \ No newline at end of file diff --git a/ljwx-boot/ljwx-boot-common/src/main/java/com/ljwx/common/cache/RedisCacheWarmupService.java b/ljwx-boot/ljwx-boot-common/src/main/java/com/ljwx/common/cache/RedisCacheWarmupService.java index 880c44b..46cf3b6 100644 --- a/ljwx-boot/ljwx-boot-common/src/main/java/com/ljwx/common/cache/RedisCacheWarmupService.java +++ b/ljwx-boot/ljwx-boot-common/src/main/java/com/ljwx/common/cache/RedisCacheWarmupService.java @@ -34,6 +34,22 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS // 分批处理大小 private static final int BATCH_SIZE = 1000; + /** + * 安全获取Long值,防止NPE + */ + private Long safeLongValue(Map row, String key) { + Object value = row.get(key); + if (value == null) { + return null; + } + try { + return ((Number) value).longValue(); + } catch (ClassCastException e) { + log.warn("字段{}类型转换失败,值: {}", key, value); + return null; + } + } + // 预热状态追踪 private boolean warmupCompleted = false; @@ -77,19 +93,30 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS String userSql = """ SELECT customer_id, GROUP_CONCAT(id) as user_ids FROM sys_user - WHERE is_deleted = 0 AND status = '1' + WHERE is_deleted = 0 AND status = '1' AND customer_id IS NOT NULL GROUP BY customer_id """; List> tenantUsers = jdbcTemplate.queryForList(userSql); for (Map row : tenantUsers) { - Long customerId = ((Number) row.get("customer_id")).longValue(); + Long customerId = safeLongValue(row, "customer_id"); + if (customerId == null) { + log.warn("发现customer_id为null的用户记录,跳过处理: {}", row); + continue; + } + String userIdsStr = (String) row.get("user_ids"); - if (userIdsStr != null) { - Set userIds = Arrays.stream(userIdsStr.split(",")) - .map(Long::valueOf) - .collect(Collectors.toSet()); - relationCacheService.cacheTenantUsers(customerId, userIds); + if (userIdsStr != null && !userIdsStr.trim().isEmpty()) { + try { + Set userIds = Arrays.stream(userIdsStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(Long::valueOf) + .collect(Collectors.toSet()); + relationCacheService.cacheTenantUsers(customerId, userIds); + } catch (NumberFormatException e) { + log.warn("用户ID格式转换失败,租户ID: {}, 用户IDs: {}", customerId, userIdsStr); + } } } @@ -97,19 +124,30 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS String orgSql = """ SELECT customer_id, GROUP_CONCAT(id) as org_ids FROM sys_org_units - WHERE is_deleted = 0 AND status = '1' + WHERE is_deleted = 0 AND status = '1' AND customer_id IS NOT NULL GROUP BY customer_id """; List> tenantOrgs = jdbcTemplate.queryForList(orgSql); for (Map row : tenantOrgs) { - Long customerId = ((Number) row.get("customer_id")).longValue(); + Long customerId = safeLongValue(row, "customer_id"); + if (customerId == null) { + log.warn("发现customer_id为null的组织记录,跳过处理: {}", row); + continue; + } + String orgIdsStr = (String) row.get("org_ids"); - if (orgIdsStr != null) { - Set orgIds = Arrays.stream(orgIdsStr.split(",")) - .map(Long::valueOf) - .collect(Collectors.toSet()); - relationCacheService.cacheTenantOrgs(customerId, orgIds); + if (orgIdsStr != null && !orgIdsStr.trim().isEmpty()) { + try { + Set orgIds = Arrays.stream(orgIdsStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(Long::valueOf) + .collect(Collectors.toSet()); + relationCacheService.cacheTenantOrgs(customerId, orgIds); + } catch (NumberFormatException e) { + log.warn("组织ID格式转换失败,租户ID: {}, 组织IDs: {}", customerId, orgIdsStr); + } } } @@ -136,19 +174,30 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS SELECT uo.user_id, GROUP_CONCAT(uo.org_id) as org_ids FROM sys_user_org uo INNER JOIN sys_user u ON uo.user_id = u.id - WHERE uo.is_deleted = 0 AND u.is_deleted = 0 + WHERE uo.is_deleted = 0 AND u.is_deleted = 0 AND uo.user_id IS NOT NULL GROUP BY uo.user_id """; List> userOrgs = jdbcTemplate.queryForList(userOrgSql); for (Map row : userOrgs) { - Long userId = ((Number) row.get("user_id")).longValue(); + Long userId = safeLongValue(row, "user_id"); + if (userId == null) { + log.warn("发现user_id为null的用户组织记录,跳过处理: {}", row); + continue; + } + String orgIdsStr = (String) row.get("org_ids"); - if (orgIdsStr != null) { - Set orgIds = Arrays.stream(orgIdsStr.split(",")) - .map(Long::valueOf) - .collect(Collectors.toSet()); - relationCacheService.cacheUserOrgs(userId, orgIds); + if (orgIdsStr != null && !orgIdsStr.trim().isEmpty()) { + try { + Set orgIds = Arrays.stream(orgIdsStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(Long::valueOf) + .collect(Collectors.toSet()); + relationCacheService.cacheUserOrgs(userId, orgIds); + } catch (NumberFormatException e) { + log.warn("组织ID格式转换失败,用户ID: {}, 组织IDs: {}", userId, orgIdsStr); + } } } @@ -159,19 +208,28 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS FROM sys_org_units o LEFT JOIN sys_user_org uo ON o.id = uo.org_id AND uo.is_deleted = 0 LEFT JOIN sys_user u ON uo.user_id = u.id AND u.is_deleted = 0 - WHERE o.is_deleted = 0 AND o.status = '1' + WHERE o.is_deleted = 0 AND o.status = '1' AND o.id IS NOT NULL GROUP BY o.id """; List> orgUsers = jdbcTemplate.queryForList(orgUserSql); for (Map row : orgUsers) { - Long orgId = ((Number) row.get("org_id")).longValue(); + Long orgId = safeLongValue(row, "org_id"); + if (orgId == null) { + log.warn("发现org_id为null的组织用户记录,跳过处理: {}", row); + continue; + } + String userIdsStr = (String) row.get("direct_user_ids"); // 获取包含子部门的所有用户 - Set allUserIds = getAllOrgUsers(orgId); - if (!allUserIds.isEmpty()) { - relationCacheService.cacheOrgUsers(orgId, allUserIds); + try { + Set allUserIds = getAllOrgUsers(orgId); + if (!allUserIds.isEmpty()) { + relationCacheService.cacheOrgUsers(orgId, allUserIds); + } + } catch (Exception e) { + log.warn("获取组织用户失败,组织ID: {}, 错误: {}", orgId, e.getMessage()); } } @@ -199,16 +257,23 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS FROM t_device_user du INNER JOIN sys_user u ON du.user_id = u.id INNER JOIN t_device_info d ON du.device_sn COLLATE utf8mb4_general_ci = d.serial_number - WHERE u.is_deleted = 0 AND d.status = 'online' + WHERE u.is_deleted = 0 AND d.status = 'online' AND du.user_id IS NOT NULL GROUP BY du.user_id """; List> userDevices = jdbcTemplate.queryForList(userDeviceSql); for (Map row : userDevices) { - Long userId = ((Number) row.get("user_id")).longValue(); + Long userId = safeLongValue(row, "user_id"); + if (userId == null) { + log.warn("发现user_id为null的用户设备记录,跳过处理: {}", row); + continue; + } + String deviceSnsStr = (String) row.get("device_sns"); - if (deviceSnsStr != null) { + if (deviceSnsStr != null && !deviceSnsStr.trim().isEmpty()) { Set deviceSns = Arrays.stream(deviceSnsStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) .collect(Collectors.toSet()); relationCacheService.cacheUserDevices(userId, deviceSns); } @@ -220,14 +285,20 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS FROM t_device_user du INNER JOIN sys_user u ON du.user_id = u.id INNER JOIN t_device_info d ON du.device_sn COLLATE utf8mb4_general_ci = d.serial_number - WHERE u.is_deleted = 0 AND d.status = 'online' + WHERE u.is_deleted = 0 AND d.status = 'online' AND du.user_id IS NOT NULL """; List> deviceUsers = jdbcTemplate.queryForList(deviceUserSql); for (Map row : deviceUsers) { String deviceSn = (String) row.get("device_sn"); - Long userId = ((Number) row.get("user_id")).longValue(); - relationCacheService.cacheDeviceUser(deviceSn, userId); + Long userId = safeLongValue(row, "user_id"); + if (userId == null) { + log.warn("发现user_id为null的设备用户记录,跳过处理: {}", row); + continue; + } + if (deviceSn != null && !deviceSn.trim().isEmpty()) { + relationCacheService.cacheDeviceUser(deviceSn, userId); + } } // 3. 预热设备-部门关系 (解决字符集冲突问题) @@ -237,14 +308,20 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS INNER JOIN t_device_user du ON d.serial_number = du.device_sn COLLATE utf8mb4_general_ci INNER JOIN sys_user u ON du.user_id = u.id INNER JOIN sys_user_org uo ON u.id = uo.user_id - WHERE d.status = 'online' AND u.is_deleted = 0 AND uo.is_deleted = 0 + WHERE d.status = 'online' AND u.is_deleted = 0 AND uo.is_deleted = 0 AND uo.org_id IS NOT NULL """; List> deviceOrgs = jdbcTemplate.queryForList(deviceOrgSql); for (Map row : deviceOrgs) { String deviceSn = (String) row.get("serial_number"); - Long orgId = ((Number) row.get("org_id")).longValue(); - relationCacheService.cacheDeviceOrg(deviceSn, orgId); + Long orgId = safeLongValue(row, "org_id"); + if (orgId == null) { + log.warn("发现org_id为null的设备组织记录,跳过处理: {}", row); + continue; + } + if (deviceSn != null && !deviceSn.trim().isEmpty()) { + relationCacheService.cacheDeviceOrg(deviceSn, orgId); + } } // 4. 预热部门-设备关系(包含子部门设备) @@ -272,19 +349,30 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS String descendantsSql = """ SELECT ancestor_id, GROUP_CONCAT(descendant_id) as descendant_ids FROM sys_org_closure - WHERE depth > 0 + WHERE depth > 0 AND ancestor_id IS NOT NULL GROUP BY ancestor_id """; List> orgDescendants = jdbcTemplate.queryForList(descendantsSql); for (Map row : orgDescendants) { - Long ancestorId = ((Number) row.get("ancestor_id")).longValue(); + Long ancestorId = safeLongValue(row, "ancestor_id"); + if (ancestorId == null) { + log.warn("发现ancestor_id为null的组织层级记录,跳过处理: {}", row); + continue; + } + String descendantIdsStr = (String) row.get("descendant_ids"); - if (descendantIdsStr != null) { - Set descendantIds = Arrays.stream(descendantIdsStr.split(",")) - .map(Long::valueOf) - .collect(Collectors.toSet()); - relationCacheService.cacheOrgDescendants(ancestorId, descendantIds); + if (descendantIdsStr != null && !descendantIdsStr.trim().isEmpty()) { + try { + Set descendantIds = Arrays.stream(descendantIdsStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(Long::valueOf) + .collect(Collectors.toSet()); + relationCacheService.cacheOrgDescendants(ancestorId, descendantIds); + } catch (NumberFormatException e) { + log.warn("后代组织ID格式转换失败,祖先ID: {}, 后代IDs: {}", ancestorId, descendantIdsStr); + } } } @@ -342,14 +430,30 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS FROM t_user_health_data WHERE timestamp >= DATE_SUB(NOW(), INTERVAL 24 HOUR) AND is_deleted = 0 + AND user_id IS NOT NULL GROUP BY user_id ) latest ON h.user_id = latest.user_id AND h.timestamp = latest.latest_time WHERE h.is_deleted = 0 + AND h.user_id IS NOT NULL """; List> latestHealthData = jdbcTemplate.queryForList(sql); for (Map row : latestHealthData) { - Long userId = ((Number) row.get("user_id")).longValue(); + // 安全获取user_id,防止NPE + Object userIdObj = row.get("user_id"); + if (userIdObj == null) { + log.warn("发现user_id为null的健康数据记录,跳过处理: {}", row); + continue; + } + + Long userId; + try { + userId = ((Number) userIdObj).longValue(); + } catch (ClassCastException e) { + log.warn("user_id类型转换失败,跳过处理: {}, 值: {}", row, userIdObj); + continue; + } + Map healthData = new HashMap<>(row); healthData.remove("user_id"); relationCacheService.cacheUserLatestHealth(userId, healthData); @@ -367,18 +471,39 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS FROM t_alert_info WHERE alert_status IN ('PENDING', 'IN_PROGRESS') AND create_time >= DATE_SUB(NOW(), INTERVAL 7 DAY) + AND user_id IS NOT NULL GROUP BY user_id """; List> activeAlerts = jdbcTemplate.queryForList(sql); for (Map row : activeAlerts) { - Long userId = ((Number) row.get("user_id")).longValue(); + // 安全获取user_id,防止NPE + Object userIdObj = row.get("user_id"); + if (userIdObj == null) { + log.warn("发现user_id为null的告警记录,跳过处理: {}", row); + continue; + } + + Long userId; + try { + userId = ((Number) userIdObj).longValue(); + } catch (ClassCastException e) { + log.warn("user_id类型转换失败,跳过处理: {}, 值: {}", row, userIdObj); + continue; + } + String alertIdsStr = (String) row.get("alert_ids"); - if (alertIdsStr != null) { - Set alertIds = Arrays.stream(alertIdsStr.split(",")) - .map(Long::valueOf) - .collect(Collectors.toSet()); - relationCacheService.cacheUserActiveAlerts(userId, alertIds); + if (alertIdsStr != null && !alertIdsStr.trim().isEmpty()) { + try { + Set alertIds = Arrays.stream(alertIdsStr.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(Long::valueOf) + .collect(Collectors.toSet()); + relationCacheService.cacheUserActiveAlerts(userId, alertIds); + } catch (NumberFormatException e) { + log.warn("告警ID格式转换失败,用户ID: {}, 告警IDs: {}", userId, alertIdsStr); + } } } @@ -401,14 +526,19 @@ public class RedisCacheWarmupService implements ApplicationRunner, ICacheWarmupS LEFT JOIN t_user_health_data h ON u.id = h.user_id AND h.timestamp >= DATE_SUB(NOW(), INTERVAL 24 HOUR) AND h.is_deleted = 0 - WHERE uo.is_deleted = 0 AND u.is_deleted = 0 + WHERE uo.is_deleted = 0 AND u.is_deleted = 0 AND uo.org_id IS NOT NULL GROUP BY uo.org_id HAVING total_records > 0 """; List> orgSummaries = jdbcTemplate.queryForList(sql); for (Map row : orgSummaries) { - Long orgId = ((Number) row.get("org_id")).longValue(); + Long orgId = safeLongValue(row, "org_id"); + if (orgId == null) { + log.warn("发现org_id为null的部门健康汇总记录,跳过处理: {}", row); + continue; + } + Map summary = new HashMap<>(row); summary.remove("org_id"); relationCacheService.cacheOrgHealthSummary(orgId, summary); diff --git a/ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/TAlertRulesFacadeImpl.java b/ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/TAlertRulesFacadeImpl.java index b84d866..251ef52 100644 --- a/ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/TAlertRulesFacadeImpl.java +++ b/ljwx-boot/ljwx-boot-modules/src/main/java/com/ljwx/modules/health/facade/impl/TAlertRulesFacadeImpl.java @@ -77,14 +77,7 @@ public class TAlertRulesFacadeImpl implements ITAlertRulesFacade { TAlertRulesBO tAlertRulesBO = CglibUtil.convertObj(tAlertRulesAddDTO, TAlertRulesBO::new); boolean result = tAlertRulesService.save(tAlertRulesBO); if (result) { - // 更新Redis缓存 - 按customer_id分别缓存 - Long customerId = tAlertRulesBO.getCustomerId(); - List list = tAlertRulesService.list(new QueryWrapper().eq("customer_id", customerId)); - // 转换为 JSON 字符串存储 - String jsonString = JSON.toJSONString(list); - RedisUtil.set("alert_rules_" + customerId, jsonString); - // 发布更新消息 - RedisUtil.publish("alert_rules_channel", "update:" + customerId); + updateAlertRulesCache(tAlertRulesBO.getCustomerId()); } return result; } @@ -94,16 +87,8 @@ public class TAlertRulesFacadeImpl implements ITAlertRulesFacade { public boolean update(TAlertRulesUpdateDTO tAlertRulesUpdateDTO) { TAlertRulesBO tAlertRulesBO = CglibUtil.convertObj(tAlertRulesUpdateDTO, TAlertRulesBO::new); boolean result = tAlertRulesService.updateById(tAlertRulesBO); - System.out.println("更新缓存" + tAlertRulesBO.toString()); if (result) { - // 更新Redis缓存 - 按customer_id分别缓存 - Long customerId = tAlertRulesBO.getCustomerId(); - List list = tAlertRulesService.list(new QueryWrapper().eq("customer_id", customerId)); - // 转换为 JSON 字符串存储 - String jsonString = JSON.toJSONString(list); - RedisUtil.set("alert_rules_" + customerId, jsonString); - // 发布更新消息 - RedisUtil.publish("alert_rules_channel", "update:" + customerId); + updateAlertRulesCache(tAlertRulesBO.getCustomerId()); } return result; } @@ -116,16 +101,49 @@ public class TAlertRulesFacadeImpl implements ITAlertRulesFacade { List toDelete = tAlertRulesService.listByIds(tAlertRulesBO.getIds()); boolean result = tAlertRulesService.removeBatchByIds(tAlertRulesBO.getIds(), true); if (result && !toDelete.isEmpty()) { - // 更新Redis缓存 - 按customer_id分别缓存 Long customerId = toDelete.get(0).getCustomerId(); - List list = tAlertRulesService.list(new QueryWrapper().eq("customer_id", customerId)); - // 转换为 JSON 字符串存储 - String jsonString = JSON.toJSONString(list); - RedisUtil.set("alert_rules_" + customerId, jsonString); - // 发布更新消息 - RedisUtil.publish("alert_rules_channel", "update:" + customerId); + updateAlertRulesCache(customerId); } return result; } + + /** + * 更新告警规则缓存 - 支持版本控制和24小时TTL + * @param customerId 客户ID + */ + private void updateAlertRulesCache(Long customerId) { + String versionKey = "alert_rules_version_" + customerId; + + try { + // 增加版本号 + Long version = RedisUtil.incr(versionKey, 1L); + + // 查询最新的告警规则 + List rules = tAlertRulesService.list( + new QueryWrapper().eq("customer_id", customerId) + ); + + // 构建缓存数据(包含版本信息) + java.util.Map cacheData = new java.util.HashMap<>(); + cacheData.put("version", version); + cacheData.put("rules", rules); + cacheData.put("updateTime", System.currentTimeMillis()); + + String cacheKey = "alert_rules_" + customerId; + String jsonString = JSON.toJSONString(cacheData); + + // 设置24小时TTL + RedisUtil.set(cacheKey, jsonString, 86400L); + + // 发布更新通知(包含版本号) + RedisUtil.publish("alert_rules_channel", "update:" + customerId + ":" + version); + + System.out.println("告警规则缓存更新成功: customerId=" + customerId + ", version=" + version + ", rules=" + rules.size()); + + } catch (Exception e) { + System.err.println("更新告警规则缓存失败: customerId=" + customerId + ", error=" + e.getMessage()); + e.printStackTrace(); + } + } } \ No newline at end of file diff --git a/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataDailyMapper.xml b/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataDailyMapper.xml index 0622038..3326c65 100644 --- a/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataDailyMapper.xml +++ b/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataDailyMapper.xml @@ -8,6 +8,7 @@ + @@ -19,7 +20,7 @@ - id, device_sn, org_id, user_id, date, sleep_data, exercise_daily_data, scientific_sleep_data, workout_data, create_time, update_time + id, device_sn, org_id, user_id, customer_id, date, sleep_data, exercise_daily_data, scientific_sleep_data, workout_data, create_time, update_time \ No newline at end of file diff --git a/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataWeeklyMapper.xml b/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataWeeklyMapper.xml index 016c3ee..3965151 100644 --- a/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataWeeklyMapper.xml +++ b/ljwx-boot/ljwx-boot-modules/src/main/resources/mapper/TUserHealthDataWeeklyMapper.xml @@ -8,6 +8,7 @@ + @@ -16,7 +17,7 @@ - id, device_sn, org_id, user_id, week_start, exercise_week_data, create_time, update_time + id, device_sn, org_id, user_id, customer_id, week_start, exercise_week_data, create_time, update_time \ No newline at end of file diff --git a/test_upload_common_event_fix.py b/test_upload_common_event_fix.py new file mode 100644 index 0000000..5fd1460 --- /dev/null +++ b/test_upload_common_event_fix.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +测试 upload_common_event 客户信息提取修复 +""" + +# 模拟修复后的客户信息提取逻辑 +def extract_common_event_customer_info(data): + """模拟修复后的upload_common_event客户信息提取逻辑""" + # 优先使用直接传递的客户信息参数 - 支持两种字段名格式 + customerId = data.get("customer_id") or data.get("customerId") + orgId = data.get("org_id") or data.get("orgId") + userId = data.get("user_id") or data.get("userId") + + return customerId, orgId, userId + +# 测试数据 - 与实际日志中的数据一致 +test_data = { + 'eventType': 'com.tdtech.ohos.action.WEAR_STATUS_CHANGED', + 'eventValue': '0', + 'deviceSn': 'CRFTQ23409001890', + 'customerId': '1939964806110937090', + 'orgId': '1939964806110937090', + 'userId': '1940034533382479873', + 'timestamp': '2025-09-10 13:05:24' +} + +print("=== 测试 upload_common_event 客户信息提取修复 ===") + +print("\n1. 测试原始数据:") +print(f" 数据: {test_data}") + +print("\n2. 测试修复前的逻辑 (只支持下划线格式):") +old_customerId = test_data.get("customer_id") +old_orgId = test_data.get("org_id") +old_userId = test_data.get("user_id") +print(f" 修复前结果: customerId={old_customerId}, orgId={old_orgId}, userId={old_userId}") + +print("\n3. 测试修复后的逻辑 (支持两种格式):") +new_customerId, new_orgId, new_userId = extract_common_event_customer_info(test_data) +print(f" 修复后结果: customerId={new_customerId}, orgId={new_orgId}, userId={new_userId}") + +print("\n4. 测试结果验证:") +success = (new_customerId == "1939964806110937090" and + new_orgId == "1939964806110937090" and + new_userId == "1940034533382479873") + +if success: + print(" ✅ upload_common_event 修复成功!客户信息能够正确提取") +else: + print(" ❌ upload_common_event 修复失败!客户信息提取仍有问题") + +print("\n5. 测试下划线格式兼容性:") +test_data_underscore = { + 'eventType': 'com.tdtech.ohos.action.WEAR_STATUS_CHANGED', + 'eventValue': '1', + 'deviceSn': 'CRFTQ23409001890', + 'customer_id': '1939964806110937090', + 'org_id': '1939964806110937090', + 'user_id': '1940034533382479873', + 'timestamp': '2025-09-10 13:05:24' +} + +under_customerId, under_orgId, under_userId = extract_common_event_customer_info(test_data_underscore) +print(f" 下划线格式: customerId={under_customerId}, orgId={under_orgId}, userId={under_userId}") + +print("\n=== 测试完成 ===") \ No newline at end of file diff --git a/utils/MessageLifecycleService.java b/utils/MessageLifecycleService.java new file mode 100644 index 0000000..a46f1e7 --- /dev/null +++ b/utils/MessageLifecycleService.java @@ -0,0 +1,457 @@ +package com.ljwx.modules.health.service.impl; + +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.ljwx.common.util.DateUtil; +import com.ljwx.infrastructure.redis.RedisService; +import com.ljwx.modules.health.domain.entity.TDeviceMessage; +import com.ljwx.modules.health.domain.entity.TDeviceMessageDetail; +import com.ljwx.modules.health.service.ITDeviceMessageService; +import com.ljwx.modules.health.service.ITDeviceMessageDetailService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; + +/** + * 消息生命周期管理服务 + * 负责消息生命周期事件记录、里程碑跟踪、状态分析等功能 + * + * @Author brunoGao + * @CreateTime 2025-09-10 + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MessageLifecycleService { + + private final ITDeviceMessageService messageService; + private final ITDeviceMessageDetailService messageDetailService; + private final RedisService redisService; + + private static final String LIFECYCLE_CACHE_PREFIX = "message:lifecycle:"; + private static final String LIFECYCLE_STATS_PREFIX = "message:lifecycle:stats:"; + + /** + * 记录消息生命周期事件 + */ + @Transactional + public void recordLifecycleEvent(Long messageId, String event, String operator, String platform) { + try { + // 获取消息信息 + TDeviceMessage message = messageService.getById(messageId); + if (message == null) { + log.warn("消息不存在,无法记录生命周期事件: messageId={}, event={}", messageId, event); + return; + } + + // 构建生命周期事件 + JSONObject lifecycleEvent = new JSONObject(); + lifecycleEvent.set("event", event); + lifecycleEvent.set("timestamp", LocalDateTime.now()); + lifecycleEvent.set("operator", operator); + lifecycleEvent.set("platform", platform); + lifecycleEvent.set("messageId", messageId); + + // 存储到Redis缓存 + String cacheKey = LIFECYCLE_CACHE_PREFIX + messageId; + List events = getLifecycleEvents(messageId); + events.add(lifecycleEvent); + + redisService.set(cacheKey, JSONUtil.toJsonStr(events), 7 * 24 * 3600); // 7天过期 + + // 更新消息状态 + updateMessageStatus(messageId, event); + + // 发布生命周期事件到Redis + JSONObject eventNotification = new JSONObject(); + eventNotification.set("type", "lifecycle_event"); + eventNotification.set("messageId", messageId); + eventNotification.set("event", event); + eventNotification.set("timestamp", LocalDateTime.now()); + + redisService.publish("message:lifecycle:events", eventNotification.toString()); + + log.info("记录消息生命周期事件成功: messageId={}, event={}, operator={}", messageId, event, operator); + + } catch (Exception e) { + log.error("记录消息生命周期事件失败: messageId={}, event={}", messageId, event, e); + } + } + + /** + * 获取消息生命周期事件列表 + */ + public List getLifecycleEvents(Long messageId) { + try { + String cacheKey = LIFECYCLE_CACHE_PREFIX + messageId; + String eventsJson = redisService.get(cacheKey); + + if (eventsJson != null) { + return JSONUtil.toList(eventsJson, JSONObject.class); + } + + return new ArrayList<>(); + } catch (Exception e) { + log.error("获取消息生命周期事件失败: messageId={}", messageId, e); + return new ArrayList<>(); + } + } + + /** + * 获取消息生命周期里程碑 + */ + public JSONObject getLifecycleMilestones(Long messageId) { + try { + List events = getLifecycleEvents(messageId); + JSONObject milestones = new JSONObject(); + + // 按事件类型分组 + Map> eventGroups = events.stream() + .collect(Collectors.groupingBy(event -> event.getStr("event"))); + + // 计算关键里程碑 + JSONObject createdEvent = eventGroups.getOrDefault("created", new ArrayList<>()) + .stream().findFirst().orElse(null); + JSONObject publishedEvent = eventGroups.getOrDefault("published", new ArrayList<>()) + .stream().findFirst().orElse(null); + + if (createdEvent != null) { + milestones.set("created", createdEvent); + + if (publishedEvent != null) { + LocalDateTime createdTime = createdEvent.get("timestamp", LocalDateTime.class); + LocalDateTime publishedTime = publishedEvent.get("timestamp", LocalDateTime.class); + long publishDelay = ChronoUnit.SECONDS.between(createdTime, publishedTime); + milestones.set("publishDelay", publishDelay); + milestones.set("published", publishedEvent); + } + } + + // 统计分发状态 + List deliveryEvents = eventGroups.getOrDefault("delivery_attempt", new ArrayList<>()); + List ackEvents = eventGroups.getOrDefault("acknowledged", new ArrayList<>()); + + milestones.set("totalDeliveryAttempts", deliveryEvents.size()); + milestones.set("totalAcknowledgements", ackEvents.size()); + + // 计算完成进度 + JSONObject progress = calculateProgress(messageId, events); + milestones.set("progress", progress); + + return milestones; + + } catch (Exception e) { + log.error("获取消息生命周期里程碑失败: messageId={}", messageId, e); + return new JSONObject(); + } + } + + /** + * 计算消息完成进度 + */ + private JSONObject calculateProgress(Long messageId, List events) { + JSONObject progress = new JSONObject(); + + try { + // 获取消息详情统计 + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("message_id", messageId); + List details = messageDetailService.list(wrapper); + + if (details.isEmpty()) { + progress.set("totalTargets", 0); + progress.set("delivered", 0); + progress.set("acknowledged", 0); + progress.set("pending", 0); + progress.set("failed", 0); + progress.set("completionRate", 0.0); + return progress; + } + + int totalTargets = details.size(); + long delivered = details.stream().mapToLong(d -> + "delivered".equals(d.getDeliveryStatus()) || "acknowledged".equals(d.getDeliveryStatus()) ? 1 : 0).sum(); + long acknowledged = details.stream().mapToLong(d -> + "acknowledged".equals(d.getDeliveryStatus()) ? 1 : 0).sum(); + long pending = details.stream().mapToLong(d -> + "pending".equals(d.getDeliveryStatus()) ? 1 : 0).sum(); + long failed = details.stream().mapToLong(d -> + "failed".equals(d.getDeliveryStatus()) ? 1 : 0).sum(); + + double completionRate = totalTargets > 0 ? (double) acknowledged / totalTargets * 100 : 0.0; + + progress.set("totalTargets", totalTargets); + progress.set("delivered", delivered); + progress.set("acknowledged", acknowledged); + progress.set("pending", pending); + progress.set("failed", failed); + progress.set("completionRate", Math.round(completionRate * 100.0) / 100.0); + + // 计算平均响应时间 + OptionalDouble avgResponseTime = details.stream() + .filter(d -> d.getResponseTime() != null && d.getResponseTime() > 0) + .mapToDouble(TDeviceMessageDetail::getResponseTime) + .average(); + + progress.set("avgResponseTime", avgResponseTime.orElse(0.0)); + + } catch (Exception e) { + log.error("计算消息完成进度失败: messageId={}", messageId, e); + } + + return progress; + } + + /** + * 生成消息生命周期报告 + */ + public JSONObject generateLifecycleReport(Long messageId) { + try { + TDeviceMessage message = messageService.getById(messageId); + if (message == null) { + throw new RuntimeException("消息不存在: " + messageId); + } + + JSONObject report = new JSONObject(); + + // 基本信息 + report.set("messageId", messageId); + report.set("messageType", message.getMessageType()); + report.set("subType", message.getSubType()); + report.set("title", message.getTitle()); + report.set("priority", message.getPriority()); + report.set("urgency", message.getUrgency()); + + // 时间信息 + report.set("createTime", message.getCreateTime()); + report.set("publishTime", message.getSentTime()); + + // 生命周期事件 + List events = getLifecycleEvents(messageId); + report.set("lifecycleEvents", events); + + // 里程碑信息 + JSONObject milestones = getLifecycleMilestones(messageId); + report.set("milestones", milestones); + + // 分发统计 + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq("message_id", messageId); + List details = messageDetailService.list(wrapper); + + // 按状态分组统计 + Map statusStats = details.stream() + .collect(Collectors.groupingBy( + d -> d.getDeliveryStatus() != null ? d.getDeliveryStatus() : "unknown", + Collectors.counting() + )); + report.set("statusStatistics", statusStats); + + // 按渠道分组统计 + Map channelStats = details.stream() + .collect(Collectors.groupingBy( + d -> d.getChannel() != null ? d.getChannel() : "unknown", + Collectors.counting() + )); + report.set("channelStatistics", channelStats); + + // 性能指标 + JSONObject performance = new JSONObject(); + + // 总体耗时 + if (message.getCreateTime() != null && message.getSentTime() != null) { + long totalDuration = ChronoUnit.SECONDS.between(message.getCreateTime(), message.getSentTime()); + performance.set("publishDuration", totalDuration); + } + + // 响应时间统计 + List responseTimes = details.stream() + .filter(d -> d.getResponseTime() != null && d.getResponseTime() > 0) + .map(TDeviceMessageDetail::getResponseTime) + .collect(Collectors.toList()); + + if (!responseTimes.isEmpty()) { + performance.set("minResponseTime", Collections.min(responseTimes)); + performance.set("maxResponseTime", Collections.max(responseTimes)); + performance.set("avgResponseTime", + responseTimes.stream().mapToInt(Integer::intValue).average().orElse(0.0)); + } + + report.set("performance", performance); + + // 异常统计 + long failedCount = details.stream() + .mapToLong(d -> "failed".equals(d.getDeliveryStatus()) ? 1 : 0) + .sum(); + + JSONObject exceptions = new JSONObject(); + exceptions.set("failedDeliveries", failedCount); + exceptions.set("failureRate", + details.size() > 0 ? (double) failedCount / details.size() * 100 : 0.0); + + report.set("exceptions", exceptions); + + return report; + + } catch (Exception e) { + log.error("生成消息生命周期报告失败: messageId={}", messageId, e); + throw new RuntimeException("生成生命周期报告失败", e); + } + } + + /** + * 获取组织消息生命周期统计 + */ + public JSONObject getOrganizationLifecycleStats(Long orgId, LocalDateTime startTime, LocalDateTime endTime) { + try { + JSONObject stats = new JSONObject(); + + // 获取组织内消息 + QueryWrapper messageWrapper = new QueryWrapper<>(); + messageWrapper.eq("customer_id", orgId); + if (startTime != null) { + messageWrapper.ge("create_time", startTime); + } + if (endTime != null) { + messageWrapper.le("create_time", endTime); + } + + List messages = messageService.list(messageWrapper); + + if (messages.isEmpty()) { + stats.set("totalMessages", 0); + stats.set("avgCompletionTime", 0.0); + stats.set("avgCompletionRate", 0.0); + return stats; + } + + // 基础统计 + stats.set("totalMessages", messages.size()); + + // 按类型分组统计 + Map typeStats = messages.stream() + .collect(Collectors.groupingBy( + m -> m.getMessageType() != null ? m.getMessageType() : "unknown", + Collectors.counting() + )); + stats.set("messageTypeStats", typeStats); + + // 按紧急程度分组统计 + Map urgencyStats = messages.stream() + .collect(Collectors.groupingBy( + m -> m.getUrgency() != null ? m.getUrgency() : "unknown", + Collectors.counting() + )); + stats.set("urgencyStats", urgencyStats); + + // 计算平均完成率和完成时间 + List completionRates = new ArrayList<>(); + List completionTimes = new ArrayList<>(); + + for (TDeviceMessage message : messages) { + JSONObject progress = calculateProgress(message.getId(), getLifecycleEvents(message.getId())); + Double completionRate = progress.getDouble("completionRate"); + if (completionRate != null) { + completionRates.add(completionRate); + } + + // 计算完成时间(从创建到最后确认的时间) + if (message.getCreateTime() != null && message.getReceivedTime() != null) { + long duration = ChronoUnit.MINUTES.between(message.getCreateTime(), message.getReceivedTime()); + completionTimes.add(duration); + } + } + + double avgCompletionRate = completionRates.stream() + .mapToDouble(Double::doubleValue) + .average() + .orElse(0.0); + + double avgCompletionTime = completionTimes.stream() + .mapToLong(Long::longValue) + .average() + .orElse(0.0); + + stats.set("avgCompletionRate", Math.round(avgCompletionRate * 100.0) / 100.0); + stats.set("avgCompletionTime", Math.round(avgCompletionTime * 100.0) / 100.0); // 分钟 + + // 趋势分析(按天统计) + Map dailyStats = messages.stream() + .collect(Collectors.groupingBy( + m -> DateUtil.format(m.getCreateTime(), "yyyy-MM-dd"), + Collectors.counting() + )); + stats.set("dailyTrend", dailyStats); + + return stats; + + } catch (Exception e) { + log.error("获取组织消息生命周期统计失败: orgId={}", orgId, e); + return new JSONObject(); + } + } + + /** + * 更新消息状态 + */ + private void updateMessageStatus(Long messageId, String event) { + try { + TDeviceMessage message = messageService.getById(messageId); + if (message == null) return; + + String newStatus = null; + switch (event) { + case "created": + newStatus = "created"; + break; + case "published": + newStatus = "published"; + break; + case "delivery_completed": + newStatus = "delivered"; + break; + case "fully_acknowledged": + newStatus = "acknowledged"; + break; + case "expired": + newStatus = "expired"; + break; + } + + if (newStatus != null && !newStatus.equals(message.getMessageStatus())) { + message.setMessageStatus(newStatus); + messageService.updateById(message); + + log.info("更新消息状态: messageId={}, oldStatus={}, newStatus={}", + messageId, message.getMessageStatus(), newStatus); + } + + } catch (Exception e) { + log.error("更新消息状态失败: messageId={}, event={}", messageId, event, e); + } + } + + /** + * 清理过期的生命周期缓存 + */ + public void cleanupExpiredLifecycleCache() { + try { + log.info("开始清理过期的消息生命周期缓存..."); + + // 这里可以添加具体的清理逻辑 + // Redis的TTL会自动处理过期数据,这里主要做统计和日志 + + log.info("消息生命周期缓存清理完成"); + + } catch (Exception e) { + log.error("清理消息生命周期缓存失败", e); + } + } +} \ No newline at end of file diff --git a/utils/batch_health_uploader.py b/utils/batch_health_uploader.py new file mode 100755 index 0000000..a01a221 --- /dev/null +++ b/utils/batch_health_uploader.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +""" +批量健康数据上传优化器 +实现批量插入优化,提高数据库写入性能 +""" + +import asyncio +import aiohttp +import time +import json +import random +import logging +import mysql.connector +from datetime import datetime, timedelta +from pathlib import Path +from typing import Dict, List, Any, Optional, Tuple +from dataclasses import dataclass +import statistics + +@dataclass +class BatchConfig: + """批量优化配置""" + base_url: str = "http://localhost:5225" + batch_size: int = 50 # 每批次上传数量 + concurrent_batches: int = 10 # 并发批次数 + test_duration_minutes: int = 3 + batch_interval_seconds: float = 0.1 # 批次间隔 + timeout_seconds: int = 30 + db_config: dict = None + +@dataclass +class BatchStats: + """批量测试统计""" + total_batches: int = 0 + total_records: int = 0 + successful_batches: int = 0 + failed_batches: int = 0 + successful_records: int = 0 + failed_records: int = 0 + start_time: Optional[datetime] = None + end_time: Optional[datetime] = None + batch_response_times: List[float] = None + error_details: Dict[str, int] = None + + def __post_init__(self): + if self.batch_response_times is None: + self.batch_response_times = [] + if self.error_details is None: + self.error_details = {} + +class BatchHealthUploader: + """批量健康数据上传优化器""" + + def __init__(self, config: BatchConfig = None): + self.config = config or BatchConfig() + if not self.config.db_config: + self.config.db_config = { + 'host': '127.0.0.1', + 'port': 3306, + 'database': 'test', + 'user': 'root', + 'password': '123456', + 'charset': 'utf8mb4' + } + + self.stats = BatchStats() + self.running = False + self.user_devices = [] + + self._setup_logging() + + def _setup_logging(self): + """设置日志""" + log_dir = Path("logs") + log_dir.mkdir(exist_ok=True) + + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + log_file = log_dir / f"batch_health_test_{timestamp}.log" + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(log_file, encoding='utf-8'), + logging.StreamHandler() + ] + ) + + self.logger = logging.getLogger(__name__) + self.logger.info("🚀 批量健康数据上传优化器开始") + self.logger.info(f"📝 日志文件: {log_file}") + + def load_real_users_and_devices(self) -> List[Tuple[int, str, str]]: + """从数据库加载真实的用户和设备信息""" + self.logger.info("📊 加载真实用户和设备信息...") + + try: + connection = mysql.connector.connect(**self.config.db_config) + cursor = connection.cursor() + + sql = """ + SELECT id, user_name, device_sn + FROM sys_user + WHERE user_name LIKE 'CRFTQ23409%' + AND device_sn IS NOT NULL + ORDER BY id + LIMIT 1000 + """ + + cursor.execute(sql) + results = cursor.fetchall() + + user_devices = [] + for user_id, user_name, device_sn in results: + user_devices.append((user_id, user_name, device_sn)) + + self.logger.info(f"✅ 成功加载 {len(user_devices)} 个用户设备信息") + + cursor.close() + connection.close() + + return user_devices + + except Exception as e: + self.logger.error(f"❌ 加载用户设备信息失败: {e}") + return [] + + def generate_batch_health_data(self, user_devices: List[Tuple[int, str, str]]) -> List[Dict[str, Any]]: + """生成一批健康数据""" + timestamp = datetime.now() + batch_data = [] + + for user_id, user_name, device_sn in user_devices: + # 基于用户ID生成个性化数据 + random.seed(user_id + int(timestamp.timestamp())) + + health_record = { + "deviceSn": device_sn, + "userId": user_id, + "heart_rate": random.randint(60, 120), + "blood_oxygen": random.randint(95, 100) if random.random() > 0.1 else 0, + "body_temperature": f"{random.uniform(36.0, 37.5):.1f}", + "step": random.randint(0, 15000), + "distance": f"{random.uniform(0, 12):.1f}", + "calorie": f"{random.uniform(0, 600):.1f}", + "latitude": f"{random.uniform(22.5, 22.6):.6f}", + "longitude": f"{random.uniform(113.9, 114.1):.6f}", + "altitude": f"{random.uniform(0, 100):.1f}", + "stress": random.randint(0, 100), + "upload_method": random.choice(["wifi", "4g", "bluetooth"]), + "blood_pressure_systolic": random.randint(110, 140), + "blood_pressure_diastolic": random.randint(70, 90), + "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S") + } + + batch_data.append(health_record) + + return batch_data + + async def upload_batch_health_data(self, session: aiohttp.ClientSession, batch_data: List[Dict[str, Any]]) -> Dict[str, Any]: + """上传一批健康数据""" + start_time = time.time() + batch_size = len(batch_data) + + try: + # 构造批量上传的数据格式 + upload_data = {"data": batch_data} + url = f"{self.config.base_url}/upload_health_data" + + async with session.post( + url, + json=upload_data, + timeout=aiohttp.ClientTimeout(total=self.config.timeout_seconds) + ) as response: + response_time = time.time() - start_time + response_text = await response.text() + + # 更新统计 + self.stats.total_batches += 1 + self.stats.total_records += batch_size + self.stats.batch_response_times.append(response_time) + + if response.status == 200: + self.stats.successful_batches += 1 + self.stats.successful_records += batch_size + else: + self.stats.failed_batches += 1 + self.stats.failed_records += batch_size + error_key = f"HTTP_{response.status}" + self.stats.error_details[error_key] = self.stats.error_details.get(error_key, 0) + 1 + + return { + 'batch_size': batch_size, + 'success': response.status == 200, + 'status_code': response.status, + 'response_time': response_time, + 'response_text': response_text[:100], + 'timestamp': datetime.now().isoformat() + } + + except Exception as e: + response_time = time.time() - start_time + self.stats.total_batches += 1 + self.stats.total_records += batch_size + self.stats.failed_batches += 1 + self.stats.failed_records += batch_size + error_key = type(e).__name__ + self.stats.error_details[error_key] = self.stats.error_details.get(error_key, 0) + 1 + self.stats.batch_response_times.append(response_time) + + return { + 'batch_size': batch_size, + 'success': False, + 'error': str(e), + 'response_time': response_time, + 'timestamp': datetime.now().isoformat() + } + + async def run_batch_stress_test(self): + """运行批量压力测试""" + self.logger.info("🔥 开始批量健康数据压力测试") + + # 加载真实用户设备信息 + self.user_devices = self.load_real_users_and_devices() + if not self.user_devices: + self.logger.error("❌ 无法加载用户设备信息,测试终止") + return + + self.logger.info(f"📊 批量测试配置:") + self.logger.info(f" - 真实用户数: {len(self.user_devices)}") + self.logger.info(f" - 每批次大小: {self.config.batch_size}") + self.logger.info(f" - 并发批次数: {self.config.concurrent_batches}") + self.logger.info(f" - 测试时长: {self.config.test_duration_minutes}分钟") + self.logger.info(f" - 批次间隔: {self.config.batch_interval_seconds}秒") + self.logger.info(f" - 目标URL: {self.config.base_url}") + + self.running = True + self.stats.start_time = datetime.now() + + # 创建HTTP会话 + connector = aiohttp.TCPConnector( + limit=self.config.concurrent_batches * 2, + limit_per_host=self.config.concurrent_batches, + ttl_dns_cache=300, + use_dns_cache=True, + ) + + async with aiohttp.ClientSession( + connector=connector, + timeout=aiohttp.ClientTimeout(total=self.config.timeout_seconds), + headers={ + 'Content-Type': 'application/json', + 'User-Agent': 'BatchHealthUploader/1.0' + } + ) as session: + # 启动监控任务 + monitor_task = asyncio.create_task(self._monitor_progress()) + + try: + tasks = [] + end_time = datetime.now() + timedelta(minutes=self.config.test_duration_minutes) + + while datetime.now() < end_time and self.running: + # 随机选择用户创建批次 + batch_users = random.sample( + self.user_devices, + min(self.config.batch_size, len(self.user_devices)) + ) + + # 生成批量数据 + batch_data = self.generate_batch_health_data(batch_users) + + # 控制并发批次数 + if len(tasks) >= self.config.concurrent_batches: + done, pending = await asyncio.wait( + tasks, + timeout=0.1, + return_when=asyncio.FIRST_COMPLETED + ) + tasks = list(pending) + + # 创建批量上传任务 + task = asyncio.create_task( + self.upload_batch_health_data(session, batch_data) + ) + tasks.append(task) + + # 控制批次频率 + await asyncio.sleep(self.config.batch_interval_seconds) + + # 等待剩余任务完成 + if tasks: + self.logger.info(f"等待 {len(tasks)} 个剩余批次完成...") + await asyncio.gather(*tasks, return_exceptions=True) + + finally: + monitor_task.cancel() + try: + await monitor_task + except asyncio.CancelledError: + pass + + self.stats.end_time = datetime.now() + self._print_final_report() + + async def _monitor_progress(self): + """监控测试进度""" + last_records = 0 + + while self.running: + try: + await asyncio.sleep(10) # 每10秒报告一次 + + current_records = self.stats.total_records + successful_records = self.stats.successful_records + failed_records = self.stats.failed_records + + # 计算记录QPS + records_delta = current_records - last_records + qps = records_delta / 10.0 + + # 计算成功率 + success_rate = (successful_records / current_records * 100) if current_records > 0 else 0 + + # 计算平均批次响应时间 + avg_batch_time = 0 + if self.stats.batch_response_times: + avg_batch_time = statistics.mean(self.stats.batch_response_times[-20:]) + + elapsed_time = datetime.now() - self.stats.start_time if self.stats.start_time else timedelta(0) + + self.logger.info( + f"📊 批量测试进度 - " + f"总记录: {current_records}, " + f"成功: {successful_records}, " + f"失败: {failed_records}, " + f"成功率: {success_rate:.1f}%, " + f"记录QPS: {qps:.1f}, " + f"批次数: {self.stats.total_batches}, " + f"平均批次时间: {avg_batch_time:.3f}s, " + f"运行时间: {str(elapsed_time).split('.')[0]}" + ) + + last_records = current_records + + except asyncio.CancelledError: + break + except Exception as e: + self.logger.error(f"监控进度异常: {e}") + + def _print_final_report(self): + """打印最终测试报告""" + self.logger.info("=" * 80) + self.logger.info("📊 批量健康数据上传优化测试报告") + self.logger.info("=" * 80) + + if self.stats.start_time and self.stats.end_time: + duration = self.stats.end_time - self.stats.start_time + self.logger.info(f"⏱️ 测试时长: {duration}") + + if self.stats.total_records > 0 and duration.total_seconds() > 0: + records_qps = self.stats.total_records / duration.total_seconds() + batches_qps = self.stats.total_batches / duration.total_seconds() + self.logger.info(f"🚀 记录QPS: {records_qps:.2f} 记录/秒") + self.logger.info(f"📦 批次QPS: {batches_qps:.2f} 批次/秒") + self.logger.info(f"💪 处理能力: {records_qps * 60:.0f} 记录/分钟") + + # 批次统计 + self.logger.info(f"📦 批次统计:") + self.logger.info(f" - 总批次数: {self.stats.total_batches}") + self.logger.info(f" - 成功批次: {self.stats.successful_batches}") + self.logger.info(f" - 失败批次: {self.stats.failed_batches}") + self.logger.info(f" - 平均批次大小: {self.stats.total_records / self.stats.total_batches:.1f}" if self.stats.total_batches > 0 else " - 平均批次大小: 0") + + # 记录统计 + self.logger.info(f"📈 记录统计:") + self.logger.info(f" - 总记录数: {self.stats.total_records}") + self.logger.info(f" - 成功记录: {self.stats.successful_records}") + self.logger.info(f" - 失败记录: {self.stats.failed_records}") + + # 成功率 + if self.stats.total_records > 0: + success_rate = (self.stats.successful_records / self.stats.total_records) * 100 + self.logger.info(f"✅ 成功率: {success_rate:.2f}%") + + # 批次响应时间统计 + if self.stats.batch_response_times: + response_times = self.stats.batch_response_times + self.logger.info(f"⚡ 批次响应时间统计:") + self.logger.info(f" - 平均批次时间: {statistics.mean(response_times):.3f}秒") + self.logger.info(f" - 最快批次时间: {min(response_times):.3f}秒") + self.logger.info(f" - 最慢批次时间: {max(response_times):.3f}秒") + if len(response_times) >= 20: + self.logger.info(f" - 95%批次时间: {statistics.quantiles(response_times, n=20)[18]:.3f}秒") + + # 错误详情 + if self.stats.error_details: + self.logger.info(f"❌ 错误详情:") + for error_type, count in self.stats.error_details.items(): + self.logger.info(f" - {error_type}: {count}次") + + self._evaluate_batch_performance() + + def _evaluate_batch_performance(self): + """评估批量处理性能""" + self.logger.info("🎯 批量处理性能评估:") + + if not self.stats.batch_response_times or self.stats.total_records == 0: + self.logger.info(" - 数据不足,无法评估") + return + + success_rate = (self.stats.successful_records / self.stats.total_records) * 100 + avg_batch_time = statistics.mean(self.stats.batch_response_times) + duration = self.stats.end_time - self.stats.start_time if self.stats.start_time and self.stats.end_time else timedelta(0) + records_qps = self.stats.total_records / duration.total_seconds() if duration.total_seconds() > 0 else 0 + + # 批量处理优势分析 + expected_single_qps = 75 # 基于之前单个上传测试的结果 + improvement_ratio = records_qps / expected_single_qps if expected_single_qps > 0 else 0 + + self.logger.info(f"📈 性能对比分析:") + self.logger.info(f" - 批量处理QPS: {records_qps:.1f}") + self.logger.info(f" - 单个处理QPS: {expected_single_qps}") + self.logger.info(f" - 性能提升倍数: {improvement_ratio:.1f}x") + + # 性能等级评估 + performance_grade = "优秀" + issues = [] + + if success_rate < 95: + performance_grade = "需要优化" + issues.append(f"成功率偏低 ({success_rate:.1f}%)") + + if records_qps < 200: + performance_grade = "需要优化" if performance_grade != "需要优化" else performance_grade + issues.append(f"QPS仍需提升 ({records_qps:.1f})") + + self.logger.info(f" - 批量处理等级: {performance_grade}") + if issues: + self.logger.info(f" - 发现问题:") + for issue in issues: + self.logger.info(f" * {issue}") + + # 优化建议 + if improvement_ratio > 2.0: + self.logger.info(" ✅ 批量处理优化效果显著!") + elif improvement_ratio > 1.5: + self.logger.info(" ✅ 批量处理有明显改善") + else: + self.logger.info(" ⚠️ 批量处理优化效果有限,建议进一步优化") + +def main(): + """主函数""" + print("🚀 批量健康数据上传优化测试") + print("🎯 验证批量插入优化效果") + print("=" * 60) + + import argparse + parser = argparse.ArgumentParser(description='批量健康数据上传优化测试') + parser.add_argument('--batch-size', type=int, default=50, help='每批次大小 (默认: 50)') + parser.add_argument('--concurrent', type=int, default=10, help='并发批次数 (默认: 10)') + parser.add_argument('--duration', type=int, default=3, help='测试时长(分钟) (默认: 3)') + parser.add_argument('--url', type=str, default='http://localhost:5225', help='服务URL') + parser.add_argument('--interval', type=float, default=0.1, help='批次间隔(秒) (默认: 0.1)') + + args = parser.parse_args() + + config = BatchConfig( + base_url=args.url, + batch_size=args.batch_size, + concurrent_batches=args.concurrent, + test_duration_minutes=args.duration, + batch_interval_seconds=args.interval + ) + + print(f"📊 批量测试配置:") + print(f" - 每批次大小: {config.batch_size}") + print(f" - 并发批次数: {config.concurrent_batches}") + print(f" - 测试时长: {config.test_duration_minutes}分钟") + print(f" - 服务地址: {config.base_url}") + print(f" - 批次间隔: {config.batch_interval_seconds}秒") + print() + + try: + confirm = input("确认开始批量优化测试? (y/N): ").strip().lower() + if confirm != 'y': + print("已取消") + return + except EOFError: + print("非交互模式,自动开始测试...") + + uploader = BatchHealthUploader(config) + + try: + asyncio.run(uploader.run_batch_stress_test()) + except KeyboardInterrupt: + print("\n⏸️ 用户中断测试") + except Exception as e: + print(f"\n❌ 测试异常: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/utils/database_optimization.sql b/utils/database_optimization.sql new file mode 100644 index 0000000..b077e20 --- /dev/null +++ b/utils/database_optimization.sql @@ -0,0 +1,127 @@ +-- 数据库索引优化脚本 +-- 针对健康数据上传压力测试的性能优化 + +USE test; + +-- ================================= +-- 1. 健康数据表 (t_user_health_data) 索引优化 +-- ================================= + +-- 分析现有索引使用情况 +-- 当前表已有较多索引,但需要针对压力测试场景进行优化 + +-- 1.1 添加用户名查询索引(用于快速定位用户) +-- 压力测试中经常需要根据用户名查询数据 +CREATE INDEX idx_user_name ON t_user_health_data(user_name); + +-- 1.2 优化设备SN+时间戳的联合索引(设备数据按时间排序) +-- 现有的 idx_device_sn 只包含设备SN,需要加上时间戳提高查询效率 +CREATE INDEX idx_device_time_optimized ON t_user_health_data(device_sn, timestamp DESC); + +-- 1.3 添加用户ID+时间戳索引(用户健康数据时间序列查询) +CREATE INDEX idx_user_id_time ON t_user_health_data(user_id, timestamp DESC); + +-- 1.4 添加创建时间索引(用于数据插入性能监控) +CREATE INDEX idx_create_time ON t_user_health_data(create_time DESC); + +-- 1.5 优化批量插入的组合索引(customer_id + create_time) +-- 便于批量操作的快速定位 +CREATE INDEX idx_batch_insert_optimized ON t_user_health_data(customer_id, create_time DESC, device_sn); + +-- ================================= +-- 2. 用户表 (sys_user) 索引优化 +-- ================================= + +-- 2.1 添加用户名唯一索引(快速用户查找) +-- 压力测试中需要根据用户名快速查找用户信息 +CREATE UNIQUE INDEX idx_user_name_unique ON sys_user(user_name); + +-- 2.2 优化设备SN索引(一个设备对应一个用户) +-- 现有索引已存在,检查是否需要优化 +-- ALTER INDEX idx_sys_user_device_sn RENAME TO idx_device_sn_lookup; + +-- 2.3 添加状态+客户ID的联合索引(活跃用户查询) +CREATE INDEX idx_status_customer ON sys_user(status, customer_id, is_deleted); + +-- ================================= +-- 3. 数据库参数优化 +-- ================================= + +-- 3.1 优化InnoDB缓冲池大小(如果可以修改配置) +-- innodb_buffer_pool_size = 系统内存的70-80% +-- 这需要在MySQL配置文件中设置,这里提供建议值 + +-- 3.2 优化插入性能相关参数 +SET GLOBAL innodb_flush_log_at_trx_commit = 2; -- 提高插入性能,降低数据安全性 +SET GLOBAL sync_binlog = 0; -- 提高写入性能 + +-- 3.3 优化查询缓存(MySQL 8.0已移除查询缓存) +-- SET GLOBAL query_cache_size = 268435456; -- 256MB +-- SET GLOBAL query_cache_type = ON; + +-- 3.4 优化连接数 +SET GLOBAL max_connections = 1000; -- 支持更多并发连接 +SET GLOBAL thread_cache_size = 50; -- 线程缓存 + +-- ================================= +-- 4. 表结构优化建议 +-- ================================= + +-- 4.1 分析表空间使用情况 +SELECT + table_name, + ROUND(((data_length + index_length) / 1024 / 1024), 2) AS "DB Size in MB" +FROM information_schema.tables +WHERE table_schema="test" AND table_name IN ('t_user_health_data', 'sys_user'); + +-- 4.2 分析索引使用效率 +SELECT + TABLE_SCHEMA as `DB`, + TABLE_NAME as `Table`, + INDEX_NAME as `Index`, + CARDINALITY, + ROUND((CARDINALITY/ + (SELECT table_rows FROM information_schema.tables + WHERE table_schema = 'test' AND table_name = 't_user_health_data') + )*100, 2) as `Selectivity %` +FROM information_schema.STATISTICS +WHERE TABLE_SCHEMA = 'test' + AND TABLE_NAME = 't_user_health_data' + AND INDEX_NAME != 'PRIMARY' +ORDER BY CARDINALITY DESC; + +-- ================================= +-- 5. 维护和监控查询 +-- ================================= + +-- 5.1 检查索引使用情况 +SHOW INDEX FROM t_user_health_data; +SHOW INDEX FROM sys_user; + +-- 5.2 分析慢查询(需要启用慢查询日志) +-- SHOW VARIABLES LIKE 'slow_query_log'; +-- SET GLOBAL slow_query_log = 'ON'; +-- SET GLOBAL long_query_time = 1; + +-- 5.3 检查表状态 +SHOW TABLE STATUS LIKE 't_user_health_data'; +SHOW TABLE STATUS LIKE 'sys_user'; + +-- ================================= +-- 执行完成提示 +-- ================================= +SELECT '数据库索引优化完成!请重新运行压力测试验证效果。' as 'OPTIMIZATION_COMPLETE'; + +-- ================================= +-- 回滚脚本(如需要) +-- ================================= +/* +-- 如果需要回滚新添加的索引: +DROP INDEX IF EXISTS idx_user_name ON t_user_health_data; +DROP INDEX IF EXISTS idx_device_time_optimized ON t_user_health_data; +DROP INDEX IF EXISTS idx_user_id_time ON t_user_health_data; +DROP INDEX IF EXISTS idx_create_time ON t_user_health_data; +DROP INDEX IF EXISTS idx_batch_insert_optimized ON t_user_health_data; +DROP INDEX IF EXISTS idx_user_name_unique ON sys_user; +DROP INDEX IF EXISTS idx_status_customer ON sys_user; +*/ \ No newline at end of file diff --git a/utils/database_optimization_safe.sql b/utils/database_optimization_safe.sql new file mode 100644 index 0000000..ffbd87e --- /dev/null +++ b/utils/database_optimization_safe.sql @@ -0,0 +1,114 @@ +-- 数据库索引优化脚本(安全版本) +-- 先检查索引是否存在,避免重复创建错误 + +USE test; + +-- ================================= +-- 1. 安全创建索引的存储过程 +-- ================================= + +DELIMITER $$ + +-- 创建安全添加索引的存储过程 +CREATE PROCEDURE AddIndexIfNotExists( + IN table_name VARCHAR(64), + IN index_name VARCHAR(64), + IN column_definition TEXT +) +BEGIN + DECLARE index_exists INT DEFAULT 0; + + -- 检查索引是否存在 + SELECT COUNT(1) INTO index_exists + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = table_name + AND INDEX_NAME = index_name; + + -- 如果索引不存在则创建 + IF index_exists = 0 THEN + SET @sql = CONCAT('CREATE INDEX ', index_name, ' ON ', table_name, '(', column_definition, ')'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + SELECT CONCAT('创建索引: ', index_name) as result; + ELSE + SELECT CONCAT('索引已存在: ', index_name) as result; + END IF; +END$$ + +DELIMITER ; + +-- ================================= +-- 2. 执行索引优化 +-- ================================= + +-- 2.1 健康数据表索引优化 +CALL AddIndexIfNotExists('t_user_health_data', 'idx_device_time_optimized', 'device_sn, timestamp DESC'); +CALL AddIndexIfNotExists('t_user_health_data', 'idx_create_time', 'create_time DESC'); +CALL AddIndexIfNotExists('t_user_health_data', 'idx_batch_insert_optimized', 'customer_id, create_time DESC, device_sn'); + +-- 2.2 用户表索引优化 +CALL AddIndexIfNotExists('sys_user', 'idx_status_customer', 'status, customer_id, is_deleted'); + +-- ================================= +-- 3. 数据库参数优化(安全设置) +-- ================================= + +-- 检查当前设置 +SELECT @@innodb_flush_log_at_trx_commit as current_flush_log_at_trx_commit; +SELECT @@sync_binlog as current_sync_binlog; +SELECT @@max_connections as current_max_connections; +SELECT @@thread_cache_size as current_thread_cache_size; + +-- 临时优化插入性能(测试期间) +SET GLOBAL innodb_flush_log_at_trx_commit = 2; +SET GLOBAL sync_binlog = 0; +SET GLOBAL max_connections = 1000; +SET GLOBAL thread_cache_size = 50; + +-- ================================= +-- 4. 检查优化结果 +-- ================================= + +-- 4.1 显示所有新创建的索引 +SELECT + TABLE_NAME as '表名', + INDEX_NAME as '索引名', + GROUP_CONCAT(COLUMN_NAME ORDER BY SEQ_IN_INDEX) as '索引字段', + CARDINALITY as '基数', + INDEX_TYPE as '索引类型' +FROM INFORMATION_SCHEMA.STATISTICS +WHERE TABLE_SCHEMA = 'test' + AND TABLE_NAME IN ('t_user_health_data', 'sys_user') + AND INDEX_NAME IN ( + 'idx_device_time_optimized', + 'idx_create_time', + 'idx_batch_insert_optimized', + 'idx_status_customer' + ) +GROUP BY TABLE_NAME, INDEX_NAME +ORDER BY TABLE_NAME, INDEX_NAME; + +-- 4.2 显示表空间使用情况 +SELECT + table_name as '表名', + table_rows as '行数', + ROUND(((data_length + index_length) / 1024 / 1024), 2) AS "大小(MB)", + ROUND((index_length / 1024 / 1024), 2) AS "索引大小(MB)", + ROUND((data_length / 1024 / 1024), 2) AS "数据大小(MB)" +FROM information_schema.tables +WHERE table_schema='test' + AND table_name IN ('t_user_health_data', 'sys_user') +ORDER BY (data_length + index_length) DESC; + +-- ================================= +-- 5. 清理存储过程 +-- ================================= +DROP PROCEDURE IF EXISTS AddIndexIfNotExists; + +-- ================================= +-- 完成提示 +-- ================================= +SELECT '数据库索引优化完成!新索引已安全创建。' as 'OPTIMIZATION_COMPLETE', + '请重新运行压力测试验证优化效果。' as 'NEXT_STEP'; \ No newline at end of file diff --git a/utils/enhanced_health_stress_test.py b/utils/enhanced_health_stress_test.py index 844e9ee..07ac6c1 100755 --- a/utils/enhanced_health_stress_test.py +++ b/utils/enhanced_health_stress_test.py @@ -26,6 +26,7 @@ class TestConfig: test_duration_minutes: int = 5 upload_interval_seconds: float = 0.5 timeout_seconds: int = 30 + verbose_json: bool = False # 是否打印详细JSON数据 db_config: dict = None @dataclass @@ -88,17 +89,17 @@ class EnhancedHealthStressTester: self.logger.info("🚀 增强版健康数据压力测试开始") self.logger.info(f"📝 日志文件: {log_file}") - def load_real_users_and_devices(self) -> List[Tuple[int, str, str]]: - """从数据库加载真实的用户和设备信息""" + def load_real_users_and_devices(self) -> List[Tuple[int, str, str, str, str]]: + """从数据库加载真实的用户和设备信息(包含customerId和orgId)""" self.logger.info("📊 从数据库加载真实用户和设备信息...") try: connection = mysql.connector.connect(**self.config.db_config) cursor = connection.cursor() - # 查询所有创建的测试用户及其设备SN + # 查询所有创建的测试用户及其设备SN、客户ID、组织ID sql = """ - SELECT id, user_name, device_sn + SELECT id, user_name, device_sn, customer_id, org_id FROM sys_user WHERE user_name LIKE 'CRFTQ23409%' AND device_sn IS NOT NULL @@ -110,14 +111,19 @@ class EnhancedHealthStressTester: results = cursor.fetchall() user_devices = [] - for user_id, user_name, device_sn in results: - user_devices.append((user_id, user_name, device_sn)) + for user_id, user_name, device_sn, customer_id, org_id in results: + # 如果customer_id或org_id为空,使用默认值 + customer_id = customer_id or "1939964806110937090" + org_id = org_id or "1939964806110937090" + user_devices.append((user_id, user_name, device_sn, customer_id, org_id)) self.logger.info(f"✅ 成功加载 {len(user_devices)} 个用户设备信息") if len(user_devices) > 0: self.logger.info(f" 用户ID范围: {user_devices[0][0]} - {user_devices[-1][0]}") self.logger.info(f" 设备SN示例: {user_devices[0][2]}") + self.logger.info(f" 客户ID示例: {user_devices[0][3]}") + self.logger.info(f" 组织ID示例: {user_devices[0][4]}") cursor.close() connection.close() @@ -131,8 +137,8 @@ class EnhancedHealthStressTester: self.logger.error(f"❌ 加载用户设备信息失败: {e}") return [] - def generate_realistic_health_data(self, user_id: int, device_sn: str) -> Dict[str, Any]: - """为指定用户和设备生成真实的健康数据""" + def generate_realistic_health_data(self, user_id: int, device_sn: str, customer_id: str, org_id: str) -> Dict[str, Any]: + """为指定用户和设备生成真实的健康数据(新接口格式)""" timestamp = datetime.now() timestamp_str = timestamp.strftime("%Y-%m-%d %H:%M:%S") @@ -142,18 +148,18 @@ class EnhancedHealthStressTester: # 生成真实的健康数据 heart_rate = random.randint(60, 120) blood_oxygen = random.randint(95, 100) if random.random() > 0.1 else 0 - body_temperature = f"{random.uniform(36.0, 37.5):.1f}" + body_temperature = "0.0" if random.random() > 0.3 else f"{random.uniform(36.0, 37.5):.1f}" # 运动数据 is_active = random.random() > 0.4 step = random.randint(0, 15000) if is_active else random.randint(0, 3000) - distance = f"{random.uniform(0, 12):.1f}" if is_active else f"{random.uniform(0, 2):.1f}" - calorie = f"{random.uniform(0, 600):.1f}" if is_active else f"{random.uniform(0, 100):.1f}" + distance = f"{random.uniform(0, 12):.1f}" if is_active else "0.0" + calorie = f"{random.uniform(100, 600):.1f}" if is_active else f"{random.uniform(50, 200):.1f}" # 深圳地区GPS坐标 - latitude = f"{random.uniform(22.5, 22.6):.6f}" - longitude = f"{random.uniform(113.9, 114.1):.6f}" - altitude = f"{random.uniform(0, 100):.1f}" + latitude = f"{random.uniform(22.5, 22.6):.12f}" + longitude = f"{random.uniform(113.9, 114.1):.11f}" + altitude = "0.0" if random.random() > 0.5 else f"{random.uniform(0, 100):.1f}" # 压力和血压数据 stress = random.randint(0, 100) @@ -163,7 +169,9 @@ class EnhancedHealthStressTester: return { "data": { "deviceSn": device_sn, - "userId": user_id, # 新增用户ID字段 + "customerId": customer_id, + "orgId": org_id, + "userId": str(user_id), "heart_rate": heart_rate, "blood_oxygen": blood_oxygen, "body_temperature": body_temperature, @@ -177,18 +185,65 @@ class EnhancedHealthStressTester: "upload_method": random.choice(["wifi", "4g", "bluetooth"]), "blood_pressure_systolic": blood_pressure_systolic, "blood_pressure_diastolic": blood_pressure_diastolic, + "sleepData": "null", + "exerciseDailyData": "null", + "exerciseWeekData": "null", + "scientificSleepData": "null", + "workoutData": "null", "timestamp": timestamp_str } } - async def upload_health_data(self, session: aiohttp.ClientSession, user_id: int, user_name: str, device_sn: str) -> Dict[str, Any]: + def generate_realistic_device_info(self, user_id: int, device_sn: str, customer_id: str, org_id: str) -> Dict[str, Any]: + """生成设备信息数据""" + timestamp = datetime.now() + timestamp_str = timestamp.strftime("%Y-%m-%d %H:%M:%S") + + # 基于设备SN生成一致的设备信息 + device_hash = hash(device_sn) % 1000000 + random.seed(device_hash) + + # 生成运动手表类设备信息 + mac_base = f"d4:bb:e6:{random.randint(10,99):02x}:{random.randint(10,99):02x}:{random.randint(10,99):02x}" + bluetooth_mac = f"D4:BB:E6:{random.randint(10,99):02x}:{random.randint(10,99):02x}:{random.randint(10,99):02x}" + ip_address = f"192.168.1.{random.randint(100,250)}" + imei = f"86615206{random.randint(10000000,99999999)}" + battery_level = random.randint(20, 100) + voltage = random.randint(3800, 4200) + + return { + "data": { + "System Software Version": "GLL-AL30BCN 3.0.0.900(SP51C700E106R370P324)", + "Wifi Address": mac_base, + "Bluetooth Address": bluetooth_mac, + "IP Address": ip_address, + "Network Access Mode": random.choice([1, 2, 3]), # 1=wifi, 2=4g, 3=bluetooth + "SerialNumber": device_sn, + "Device Name": "HUAWEI WATCH B7-536-BF0", + "IMEI": imei, + "batteryLevel": battery_level, + "voltage": voltage, + "chargingStatus": random.choice(["NONE", "CHARGING", "FULL"]), + "status": "ACTIVE", + "timestamp": timestamp_str, + "wearState": random.choice([0, 1]), # 0=未佩戴, 1=佩戴 + "customerId": customer_id, + "orgId": org_id, + "userId": str(user_id) + } + } + + async def upload_health_data(self, session: aiohttp.ClientSession, user_id: int, user_name: str, device_sn: str, customer_id: str, org_id: str) -> Dict[str, Any]: """上传单个用户的健康数据""" start_time = time.time() try: - health_data = self.generate_realistic_health_data(user_id, device_sn) + health_data = self.generate_realistic_health_data(user_id, device_sn, customer_id, org_id) url = f"{self.config.base_url}/upload_health_data" + if self.config.verbose_json: + self.logger.info(f"🏥 upload_health_data - {device_sn}: {json.dumps(health_data, indent=2, ensure_ascii=False)}") + async with session.post( url, json=health_data, @@ -212,6 +267,7 @@ class EnhancedHealthStressTester: 'user_id': user_id, 'user_name': user_name, 'device_sn': device_sn, + 'api_type': 'health_data', 'success': response.status == 200, 'status_code': response.status, 'response_time': response_time, @@ -230,6 +286,7 @@ class EnhancedHealthStressTester: 'user_id': user_id, 'user_name': user_name, 'device_sn': device_sn, + 'api_type': 'health_data', 'success': False, 'error': 'TIMEOUT', 'response_time': response_time, @@ -248,6 +305,86 @@ class EnhancedHealthStressTester: 'user_id': user_id, 'user_name': user_name, 'device_sn': device_sn, + 'api_type': 'health_data', + 'success': False, + 'error': str(e), + 'response_time': response_time, + 'timestamp': datetime.now().isoformat() + } + + async def upload_device_info(self, session: aiohttp.ClientSession, user_id: int, user_name: str, device_sn: str, customer_id: str, org_id: str) -> Dict[str, Any]: + """上传单个用户的设备信息""" + start_time = time.time() + + try: + device_info = self.generate_realistic_device_info(user_id, device_sn, customer_id, org_id) + url = f"{self.config.base_url}/upload_device_info" + + if self.config.verbose_json: + self.logger.info(f"📱 upload_device_info - {device_sn}: {json.dumps(device_info, indent=2, ensure_ascii=False)}") + + async with session.post( + url, + json=device_info, + timeout=aiohttp.ClientTimeout(total=self.config.timeout_seconds) + ) as response: + response_time = time.time() - start_time + response_text = await response.text() + + # 更新统计 + self.stats.total_requests += 1 + self.stats.response_times.append(response_time) + + if response.status == 200: + self.stats.successful_requests += 1 + else: + self.stats.failed_requests += 1 + error_key = f"HTTP_{response.status}" + self.stats.error_details[error_key] = self.stats.error_details.get(error_key, 0) + 1 + + return { + 'user_id': user_id, + 'user_name': user_name, + 'device_sn': device_sn, + 'api_type': 'device_info', + 'success': response.status == 200, + 'status_code': response.status, + 'response_time': response_time, + 'response_text': response_text[:100], + 'timestamp': datetime.now().isoformat() + } + + except asyncio.TimeoutError: + response_time = time.time() - start_time + self.stats.total_requests += 1 + self.stats.timeout_requests += 1 + self.stats.response_times.append(response_time) + self.stats.error_details['TIMEOUT'] = self.stats.error_details.get('TIMEOUT', 0) + 1 + + return { + 'user_id': user_id, + 'user_name': user_name, + 'device_sn': device_sn, + 'api_type': 'device_info', + 'success': False, + 'error': 'TIMEOUT', + 'response_time': response_time, + 'timestamp': datetime.now().isoformat() + } + + except Exception as e: + response_time = time.time() - start_time + self.stats.total_requests += 1 + self.stats.failed_requests += 1 + error_key = type(e).__name__ + self.stats.error_details[error_key] = self.stats.error_details.get(error_key, 0) + 1 + self.stats.response_times.append(response_time) + + return { + 'user_id': user_id, + 'user_name': user_name, + 'device_sn': device_sn, + 'api_type': 'device_info', 'success': False, 'error': str(e), 'response_time': response_time, @@ -256,7 +393,7 @@ class EnhancedHealthStressTester: async def run_stress_test(self): """运行压力测试""" - self.logger.info("🔥 开始1000用户真实健康数据压力测试") + self.logger.info("🔥 开始1000用户真实健康数据双接口压力测试") # 加载真实用户设备信息 self.user_devices = self.load_real_users_and_devices() @@ -270,6 +407,8 @@ class EnhancedHealthStressTester: self.logger.info(f" - 测试时长: {self.config.test_duration_minutes}分钟") self.logger.info(f" - 上传间隔: {self.config.upload_interval_seconds}秒") self.logger.info(f" - 目标URL: {self.config.base_url}") + self.logger.info(f" - API接口: upload_health_data + upload_device_info") + self.logger.info(f" - 预期QPS: {len(self.user_devices) * 2 / self.config.upload_interval_seconds:.1f} (每用户双接口)") self.running = True self.stats.start_time = datetime.now() @@ -301,10 +440,10 @@ class EnhancedHealthStressTester: # 随机选择用户进行上传 selected_users = random.sample( self.user_devices, - min(self.config.concurrent_requests, len(self.user_devices)) + min(self.config.concurrent_requests // 2, len(self.user_devices)) # 每个用户需要两个请求 ) - for user_id, user_name, device_sn in selected_users: + for user_id, user_name, device_sn, customer_id, org_id in selected_users: if len(tasks) >= self.config.concurrent_requests: # 等待部分任务完成 done, pending = await asyncio.wait( @@ -314,10 +453,14 @@ class EnhancedHealthStressTester: ) tasks = list(pending) - task = asyncio.create_task( - self.upload_health_data(session, user_id, user_name, device_sn) + # 同时上传健康数据和设备信息 + health_task = asyncio.create_task( + self.upload_health_data(session, user_id, user_name, device_sn, customer_id, org_id) ) - tasks.append(task) + device_task = asyncio.create_task( + self.upload_device_info(session, user_id, user_name, device_sn, customer_id, org_id) + ) + tasks.extend([health_task, device_task]) # 控制上传频率 await asyncio.sleep(self.config.upload_interval_seconds) @@ -384,7 +527,7 @@ class EnhancedHealthStressTester: def _print_final_report(self): """打印最终测试报告""" self.logger.info("=" * 80) - self.logger.info("📊 1000用户真实健康数据压力测试报告") + self.logger.info("📊 1000用户真实健康数据双接口压力测试报告") self.logger.info("=" * 80) if self.stats.start_time and self.stats.end_time: @@ -402,6 +545,8 @@ class EnhancedHealthStressTester: if self.user_devices: self.logger.info(f" - 用户ID范围: {self.user_devices[0][0]} - {self.user_devices[-1][0]}") self.logger.info(f" - 设备SN范围: {self.user_devices[0][2]} - {self.user_devices[-1][2]}") + self.logger.info(f" - 客户ID: {self.user_devices[0][3]}") + self.logger.info(f" - 组织ID: {self.user_devices[0][4]}") # 请求统计 self.logger.info(f"📈 请求统计:") @@ -458,7 +603,7 @@ class EnhancedHealthStressTester: performance_grade = "需要优化" if performance_grade != "需要优化" else performance_grade issues.append(f"响应时间偏慢 ({avg_response_time:.3f}s)") - if qps < 200: + if qps < 400: # 双接口模式需要更高的QPS performance_grade = "需要优化" if performance_grade != "需要优化" else performance_grade issues.append(f"QPS偏低 ({qps:.1f})") @@ -470,21 +615,24 @@ class EnhancedHealthStressTester: # 系统性能对比 self.logger.info("🆚 性能基准对比:") - self.logger.info(" 目标性能指标:") - self.logger.info(" - 目标QPS: 300+ 请求/秒") + self.logger.info(" 目标性能指标(双接口模式):") + self.logger.info(" - 目标QPS: 600+ 请求/秒 (300+ health_data + 300+ device_info)") self.logger.info(" - 目标响应时间: <0.5秒") self.logger.info(" - 目标成功率: >98%") - self.logger.info(" - 并发处理能力: 1000+ 真实用户") + self.logger.info(" - 并发处理能力: 1000+ 真实用户 x 2接口") - if qps >= 300: + if qps >= 600: self.logger.info(" ✅ QPS达标!系统性能优秀") + elif qps >= 300: + self.logger.info(f" 🟡 QPS部分达标,当前: {qps:.1f}, 目标: 600+") else: - self.logger.info(f" ⚠️ QPS未达标,当前: {qps:.1f}, 目标: 300+") + self.logger.info(f" ⚠️ QPS未达标,当前: {qps:.1f}, 目标: 600+") def main(): """主函数""" print("🚀 增强版健康数据压力测试") print("🎯 使用1000个真实sys_user进行压力测试") + print("🔗 同时测试upload_health_data和upload_device_info接口") print("=" * 60) import argparse @@ -493,6 +641,7 @@ def main(): parser.add_argument('--duration', type=int, default=5, help='测试时长(分钟) (默认: 5)') parser.add_argument('--url', type=str, default='http://localhost:5225', help='服务URL') parser.add_argument('--interval', type=float, default=0.5, help='上传间隔(秒) (默认: 0.5)') + parser.add_argument('--verbose', action='store_true', help='打印详细JSON数据') args = parser.parse_args() @@ -500,7 +649,8 @@ def main(): base_url=args.url, concurrent_requests=args.concurrent, test_duration_minutes=args.duration, - upload_interval_seconds=args.interval + upload_interval_seconds=args.interval, + verbose_json=args.verbose ) print(f"📊 测试配置:") diff --git a/utils/generate_alerts_optimizer.py b/utils/generate_alerts_optimizer.py new file mode 100644 index 0000000..58810aa --- /dev/null +++ b/utils/generate_alerts_optimizer.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 +""" +generate_alerts 函数优化器 +解决upload_health_data接口的性能瓶颈问题 - 支持Redis缓存同步 +""" + +import time +import threading +import queue +import json +import logging +import sys +import os +from typing import Dict, List, Any, Optional, Tuple +from dataclasses import dataclass +from concurrent.futures import ThreadPoolExecutor +from cachetools import TTLCache, LRUCache +import asyncio +from datetime import datetime, timedelta + +# 添加ljwx-bigscreen路径以导入缓存管理器 +sys.path.append(os.path.join(os.path.dirname(__file__), '../ljwx-bigscreen/bigscreen')) +try: + from alert_rules_cache_manager import get_alert_rules_cache_manager, AlertRule + CACHE_MANAGER_AVAILABLE = True +except ImportError: + print("警告:无法导入告警规则缓存管理器,将使用数据库查询模式") + AlertRule = None + get_alert_rules_cache_manager = None + CACHE_MANAGER_AVAILABLE = False + +# 设置日志 +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +@dataclass +class AlertRule: + """告警规则数据结构""" + id: int + rule_type: str + physical_sign: str + threshold_min: float + threshold_max: float + trend_duration: int + severity_level: str + alert_message: str + is_enabled: bool = True + +@dataclass +class DeviceInfo: + """设备信息数据结构""" + device_sn: str + user_id: int + org_id: int + customer_id: int + user_name: str + org_name: str + +@dataclass +class AlertCandidate: + """候选告警数据""" + rule_id: int + rule_type: str + device_sn: str + alert_desc: str + severity_level: str + health_data_id: Optional[int] + user_id: Optional[int] + org_id: Optional[int] + customer_id: Optional[int] + alert_timestamp: datetime + alert_value: Any = None + threshold_violated: str = None + +class AlertRuleCache: + """告警规则缓存管理器""" + + def __init__(self, ttl=600): # 10分钟缓存 + self.cache = TTLCache(maxsize=1000, ttl=ttl) + self.lock = threading.RLock() + self._last_refresh = 0 + self._refresh_interval = 300 # 5分钟强制刷新 + + def get_rules(self) -> List[AlertRule]: + """获取缓存的告警规则""" + with self.lock: + current_time = time.time() + cache_key = "all_rules" + + # 检查是否需要强制刷新 + if (cache_key not in self.cache or + current_time - self._last_refresh > self._refresh_interval): + self._refresh_rules() + self._last_refresh = current_time + + return self.cache.get(cache_key, []) + + def _refresh_rules(self): + """刷新告警规则缓存""" + try: + from ljwx_bigscreen.bigscreen.bigScreen.models import AlertRules + from ljwx_bigscreen.bigscreen.bigScreen.models import db + + rules = AlertRules.query.filter_by(is_deleted=False).all() + rule_objects = [] + + for rule in rules: + rule_obj = AlertRule( + id=rule.id, + rule_type=rule.rule_type, + physical_sign=rule.physical_sign or '', + threshold_min=float(rule.threshold_min or 0), + threshold_max=float(rule.threshold_max or float('inf')), + trend_duration=int(rule.trend_duration or 1), + severity_level=rule.severity_level, + alert_message=rule.alert_message, + is_enabled=getattr(rule, 'is_enabled', True) + ) + rule_objects.append(rule_obj) + + self.cache["all_rules"] = rule_objects + logger.info(f"告警规则缓存刷新完成,共{len(rule_objects)}条规则") + + except Exception as e: + logger.error(f"刷新告警规则缓存失败: {e}") + # 如果刷新失败,保持现有缓存 + + def invalidate(self): + """清除缓存""" + with self.lock: + self.cache.clear() + +class DeviceInfoCache: + """设备信息缓存管理器""" + + def __init__(self, ttl=300): # 5分钟缓存 + self.cache = TTLCache(maxsize=10000, ttl=ttl) + self.lock = threading.RLock() + + def get_device_info(self, device_sn: str) -> Optional[DeviceInfo]: + """获取缓存的设备信息""" + with self.lock: + cached = self.cache.get(device_sn) + if cached: + return cached + + # 缓存未命中,查询数据库 + device_info = self._query_device_info(device_sn) + if device_info: + with self.lock: + self.cache[device_sn] = device_info + + return device_info + + def _query_device_info(self, device_sn: str) -> Optional[DeviceInfo]: + """查询设备信息""" + try: + from ljwx_bigscreen.bigscreen.bigScreen.device import get_device_user_org_info + + result = get_device_user_org_info(device_sn) + if result.get('success'): + return DeviceInfo( + device_sn=device_sn, + user_id=result.get('user_id'), + org_id=result.get('org_id'), + customer_id=result.get('customer_id', 0), + user_name=result.get('user_name', ''), + org_name=result.get('org_name', '') + ) + return None + + except Exception as e: + logger.error(f"查询设备信息失败 {device_sn}: {e}") + return None + +class AbnormalCountTracker: + """异常计数跟踪器""" + + def __init__(self, ttl=3600): # 1小时过期 + self.counts = LRUCache(maxsize=50000) # device_sn:physical_sign -> count + self.lock = threading.RLock() + self.timestamps = {} # 记录最后更新时间 + self.ttl = ttl + + def update_count(self, device_sn: str, physical_sign: str, is_abnormal: bool) -> int: + """更新异常计数""" + key = f"{device_sn}:{physical_sign}" + current_time = time.time() + + with self.lock: + # 检查是否超时 + if key in self.timestamps: + if current_time - self.timestamps[key] > self.ttl: + self.counts.pop(key, None) + self.timestamps.pop(key, None) + + if is_abnormal: + current_count = self.counts.get(key, 0) + 1 + self.counts[key] = current_count + self.timestamps[key] = current_time + return current_count + else: + # 正常值,重置计数 + self.counts[key] = 0 + self.timestamps[key] = current_time + return 0 + + def get_count(self, device_sn: str, physical_sign: str) -> int: + """获取当前异常计数""" + key = f"{device_sn}:{physical_sign}" + with self.lock: + return self.counts.get(key, 0) + +class OptimizedAlertGenerator: + """优化的告警生成器""" + + def __init__(self, max_workers=4, batch_size=100): + self.rule_cache = AlertRuleCache() + self.device_cache = DeviceInfoCache() + self.abnormal_tracker = AbnormalCountTracker() + + # 异步处理配置 + self.alert_queue = queue.Queue(maxsize=10000) + self.batch_size = batch_size + self.max_workers = max_workers + self.executor = ThreadPoolExecutor(max_workers=max_workers) + self.running = False + + # 性能统计 + self.stats = { + 'total_processed': 0, + 'total_alerts_generated': 0, + 'cache_hits': 0, + 'cache_misses': 0, + 'avg_processing_time': 0.0, + 'rule_evaluation_time': 0.0, + 'db_save_time': 0.0 + } + + # 启动后台处理器 + self.start_background_processor() + + def start_background_processor(self): + """启动后台处理器""" + self.running = True + for i in range(self.max_workers): + self.executor.submit(self._background_worker) + logger.info(f"告警生成器已启动,工作线程数: {self.max_workers}") + + def stop_background_processor(self): + """停止后台处理器""" + self.running = False + self.executor.shutdown(wait=True) + logger.info("告警生成器已停止") + + def generate_alerts_async(self, health_data: Dict, health_data_id: Optional[int] = None): + """异步生成告警 - 主入口函数""" + try: + # 快速验证 + if not health_data or not isinstance(health_data, dict): + logger.warning("无效的健康数据格式") + return False + + device_sn = health_data.get('deviceSn') + if not device_sn: + logger.warning("健康数据缺少设备序列号") + return False + + # 构造处理项 + process_item = { + 'health_data': health_data, + 'health_data_id': health_data_id, + 'timestamp': time.time(), + 'device_sn': device_sn + } + + # 提交到异步队列 + try: + self.alert_queue.put_nowait(process_item) + return True + except queue.Full: + logger.warning("告警处理队列已满,丢弃请求") + return False + + except Exception as e: + logger.error(f"提交异步告警生成失败: {e}") + return False + + def _background_worker(self): + """后台工作线程""" + batch = [] + last_process_time = time.time() + max_wait_time = 2.0 # 最大等待时间 + + while self.running: + try: + # 收集批次 + while (len(batch) < self.batch_size and + (time.time() - last_process_time) < max_wait_time): + try: + item = self.alert_queue.get(timeout=0.5) + batch.append(item) + except queue.Empty: + break + + # 处理批次 + if batch: + self._process_batch(batch) + batch.clear() + last_process_time = time.time() + + except Exception as e: + logger.error(f"后台工作线程异常: {e}") + batch.clear() + + def _process_batch(self, batch: List[Dict]): + """批量处理告警生成""" + start_time = time.time() + alerts_to_insert = [] + + try: + # 预加载告警规则 + rules = self.rule_cache.get_rules() + if not rules: + logger.warning("没有可用的告警规则") + return + + # 批量处理每个健康数据 + for item in batch: + try: + health_data = item['health_data'] + health_data_id = item['health_data_id'] + device_sn = health_data.get('deviceSn', 'Unknown') + + # 获取设备信息(缓存) + device_info = self.device_cache.get_device_info(device_sn) + if not device_info: + logger.warning(f"设备信息未找到: {device_sn}") + continue + + # 评估告警规则 + rule_start_time = time.time() + alert_candidates = self._evaluate_rules_optimized( + health_data, rules, device_info, health_data_id + ) + + self.stats['rule_evaluation_time'] += time.time() - rule_start_time + alerts_to_insert.extend(alert_candidates) + + except Exception as e: + logger.error(f"处理单个健康数据失败: {e}") + continue + + # 批量插入告警 + if alerts_to_insert: + db_start_time = time.time() + inserted_count = self._batch_insert_alerts(alerts_to_insert) + self.stats['db_save_time'] += time.time() - db_start_time + self.stats['total_alerts_generated'] += inserted_count + logger.info(f"批量插入告警完成: {inserted_count}条") + + # 更新统计 + processing_time = time.time() - start_time + self.stats['total_processed'] += len(batch) + self.stats['avg_processing_time'] = ( + self.stats['avg_processing_time'] * 0.9 + processing_time * 0.1 + ) + + logger.info(f"批量处理完成: 处理{len(batch)}条, 生成{len(alerts_to_insert)}个告警, 耗时{processing_time:.3f}s") + + except Exception as e: + logger.error(f"批量处理异常: {e}") + + def _evaluate_rules_optimized(self, health_data: Dict, rules: List[AlertRule], + device_info: DeviceInfo, health_data_id: Optional[int]) -> List[AlertCandidate]: + """优化的规则评估""" + alert_candidates = [] + device_sn = health_data.get('deviceSn', 'Unknown') + + for rule in rules: + if not rule.is_enabled or not rule.physical_sign: + continue + + try: + # 获取对应的健康数据值 + is_abnormal, violation_info = self._check_rule_violation(health_data, rule) + + # 更新异常计数 + abnormal_count = self.abnormal_tracker.update_count( + device_sn, rule.physical_sign, is_abnormal + ) + + # 检查是否需要生成告警 + if abnormal_count >= rule.trend_duration: + alert_candidate = AlertCandidate( + rule_id=rule.id, + rule_type=rule.rule_type, + device_sn=device_sn, + alert_desc=f"{rule.alert_message} - {violation_info}", + severity_level=rule.severity_level, + health_data_id=health_data_id, + user_id=device_info.user_id, + org_id=device_info.org_id, + customer_id=device_info.customer_id, + alert_timestamp=datetime.now(), + alert_value=violation_info, + threshold_violated=f"连续{abnormal_count}次异常" + ) + alert_candidates.append(alert_candidate) + + # 重置计数,避免重复告警 + self.abnormal_tracker.update_count(device_sn, rule.physical_sign, False) + + except Exception as e: + logger.error(f"评估规则失败 rule_id={rule.id}: {e}") + continue + + return alert_candidates + + def _check_rule_violation(self, health_data: Dict, rule: AlertRule) -> Tuple[bool, str]: + """检查规则违反情况""" + physical_sign = rule.physical_sign + + try: + if physical_sign == 'bloodPressure': + # 血压特殊处理 + systolic = self._safe_float(health_data.get('pressureHigh')) + diastolic = self._safe_float(health_data.get('pressureLow')) + + systolic_abnormal = (systolic is not None and + (systolic < rule.threshold_min or systolic > rule.threshold_max)) + diastolic_abnormal = (diastolic is not None and + (diastolic < rule.threshold_min or diastolic > rule.threshold_max)) + + if systolic_abnormal or diastolic_abnormal: + violation_info = f"收缩压:{systolic}, 舒张压:{diastolic} (范围:{rule.threshold_min}-{rule.threshold_max})" + return True, violation_info + else: + return False, f"血压正常: {systolic}/{diastolic}" + + else: + # 其他指标 + value = self._safe_float(health_data.get(physical_sign)) + if value is not None: + if value < rule.threshold_min or value > rule.threshold_max: + violation_info = f"{physical_sign}:{value} (范围:{rule.threshold_min}-{rule.threshold_max})" + return True, violation_info + else: + return False, f"{physical_sign}正常: {value}" + else: + return False, f"{physical_sign}数据缺失" + + except Exception as e: + logger.error(f"检查规则违反异常: {e}") + return False, "检查失败" + + def _safe_float(self, value) -> Optional[float]: + """安全的浮点数转换""" + if value is None or value == '' or str(value).strip() == '': + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + def _batch_insert_alerts(self, alert_candidates: List[AlertCandidate]) -> int: + """批量插入告警""" + try: + from ljwx_bigscreen.bigscreen.bigScreen.models import AlertInfo, db + + # 构造插入数据 + alerts_to_insert = [] + for candidate in alert_candidates: + alert_data = { + 'rule_id': candidate.rule_id, + 'alert_type': candidate.rule_type, + 'device_sn': candidate.device_sn, + 'alert_desc': candidate.alert_desc, + 'severity_level': candidate.severity_level, + 'alert_status': 'pending', + 'alert_timestamp': candidate.alert_timestamp, + 'health_id': candidate.health_data_id, + 'user_id': candidate.user_id, + 'org_id': candidate.org_id, + 'customer_id': candidate.customer_id + } + alerts_to_insert.append(alert_data) + + # 批量插入 + if alerts_to_insert: + db.session.bulk_insert_mappings(AlertInfo, alerts_to_insert) + db.session.commit() + return len(alerts_to_insert) + + return 0 + + except Exception as e: + logger.error(f"批量插入告警失败: {e}") + from ljwx_bigscreen.bigscreen.bigScreen.models import db + db.session.rollback() + return 0 + + def get_stats(self) -> Dict: + """获取性能统计""" + stats = self.stats.copy() + stats['queue_size'] = self.alert_queue.qsize() + stats['rule_cache_size'] = len(self.rule_cache.cache) + stats['device_cache_size'] = len(self.device_cache.cache) + stats['abnormal_tracker_size'] = len(self.abnormal_tracker.counts) + return stats + + def clear_caches(self): + """清除所有缓存""" + self.rule_cache.invalidate() + self.device_cache.cache.clear() + self.abnormal_tracker.counts.clear() + self.abnormal_tracker.timestamps.clear() + logger.info("所有缓存已清除") + +# 全局实例 +_alert_generator_instance = None +_instance_lock = threading.Lock() + +def get_optimized_alert_generator() -> OptimizedAlertGenerator: + """获取优化告警生成器的全局实例""" + global _alert_generator_instance + + if _alert_generator_instance is None: + with _instance_lock: + if _alert_generator_instance is None: + _alert_generator_instance = OptimizedAlertGenerator() + + return _alert_generator_instance + +# 兼容性函数 - 替换原有的generate_alerts +def generate_alerts_optimized(health_data: Dict, health_data_id: Optional[int] = None) -> bool: + """优化版本的generate_alerts函数""" + try: + generator = get_optimized_alert_generator() + return generator.generate_alerts_async(health_data, health_data_id) + except Exception as e: + logger.error(f"优化告警生成失败: {e}") + return False + +def get_alert_generator_stats() -> Dict: + """获取告警生成器统计信息""" + try: + generator = get_optimized_alert_generator() + return generator.get_stats() + except Exception as e: + logger.error(f"获取统计信息失败: {e}") + return {} + +def clear_alert_caches(): + """清除告警相关缓存""" + try: + generator = get_optimized_alert_generator() + generator.clear_caches() + return True + except Exception as e: + logger.error(f"清除缓存失败: {e}") + return False + +if __name__ == "__main__": + # 测试代码 + generator = OptimizedAlertGenerator(max_workers=2, batch_size=10) + + # 模拟健康数据 + test_data = { + 'deviceSn': 'TEST001', + 'heartRate': 120, + 'bloodOxygen': 95, + 'pressureHigh': 140, + 'pressureLow': 90, + 'temperature': 37.2, + 'timestamp': '2025-01-15 10:00:00' + } + + # 测试异步生成告警 + success = generator.generate_alerts_async(test_data, 12345) + print(f"异步告警生成提交: {'成功' if success else '失败'}") + + # 等待处理完成 + time.sleep(3) + + # 查看统计信息 + stats = generator.get_stats() + print(f"处理统计: {json.dumps(stats, indent=2, default=str)}") + + # 停止生成器 + generator.stop_background_processor() + print("测试完成") \ No newline at end of file diff --git a/utils/message_integration_plan_updated.md b/utils/message_integration_plan_updated.md new file mode 100644 index 0000000..cd673cb --- /dev/null +++ b/utils/message_integration_plan_updated.md @@ -0,0 +1,549 @@ +# 基于实际代码的消息数据流集成方案 + +基于对ljwx-boot、ljwx-bigscreen、ljwx-phone、ljwx-watch四个平台的详细代码分析,提供一个贴合实际情况的消息数据流集成优化方案。 + +## 一、现有架构分析 + +### 1.1 数据库表结构(已存在) + +**t_device_message (主消息表)** +```sql +CREATE TABLE t_device_message ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + device_sn VARCHAR(255) NOT NULL, + message TEXT NOT NULL, + org_id VARCHAR(50), + user_id VARCHAR(50), + customer_id BIGINT NOT NULL DEFAULT 0, + message_type VARCHAR(50) NOT NULL, -- job/task/announcement/notification/system_alert + sender_type VARCHAR(50) NOT NULL, -- system/admin/user + receiver_type VARCHAR(50) NOT NULL, -- device/user/department + message_status VARCHAR(50) DEFAULT 'pending', -- pending/delivered/acknowledged + responded_number INTEGER DEFAULT 0, + sent_time DATETIME DEFAULT CURRENT_TIMESTAMP, + received_time DATETIME NULL, + create_time DATETIME DEFAULT CURRENT_TIMESTAMP, + update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); +``` + +**t_device_message_detail (消息详情表)** +```sql +CREATE TABLE t_device_message_detail ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + message_id BIGINT NOT NULL, + device_sn VARCHAR(255) NOT NULL, + message TEXT NOT NULL, + message_type VARCHAR(50) NOT NULL, + sender_type VARCHAR(50) NOT NULL, + receiver_type VARCHAR(50) NOT NULL, + message_status VARCHAR(50) DEFAULT 'pending', + sent_time DATETIME, + received_time DATETIME, + customer_id BIGINT NOT NULL, + org_id BIGINT, + -- 新增字段用于统计和跟踪 + delivery_status VARCHAR(50), -- pending/delivered/acknowledged/failed/expired + channel VARCHAR(50), -- message/push/wechat/watch + response_time INTEGER, -- 响应时间(秒) + acknowledge_time DATETIME, -- 确认时间 + target_type VARCHAR(50), -- user/device/department/organization + target_id VARCHAR(255) -- 目标ID +); +``` + +### 1.2 现有消息流程 + +``` +ljwx-admin/ljwx-bigscreen → t_device_message → ljwx-phone → ljwx-watch + ↓ (手动发送) ↓ (API查询) ↓ (蓝牙) ↓ (用户确认) + 告警规则触发 定时获取消息 转发给手表 反向确认回传 +``` + +## 二、各平台现状分析 + +### 2.1 ljwx-boot (Java后端) + +**现有接口**: +- `TDeviceMessageController` - CRUD操作 +- `DeviceMessageServiceImpl` - 兼容性服务层 +- 支持消息的发送、接收、保存 + +**优化需求**: +- 增强消息生命周期管理 +- 添加实时状态同步 +- 支持批量操作和统计 + +### 2.2 ljwx-bigscreen (Python大屏) + +**现有功能**: +- `DeviceMessage`、`DeviceMessageDetail` SQLAlchemy模型 +- `/DeviceMessage/send`、`/receive` API接口 +- 告警自动转消息机制 (`system_event_alert.py`) +- 消息统计和展示功能 + +**优化需求**: +- 增强消息跟踪界面 +- 实时状态更新 +- 完善统计报表 + +### 2.3 ljwx-phone (Flutter移动端) + +**现有功能**: +- `Message`、`MessageInfo` Dart模型 +- `ApiService.getDeviceMessages()` - 获取消息 +- `markMessageAsRead()` - 标记已读 +- 消息列表和详情UI +- 蓝牙消息转发给手表 + +**优化需求**: +- 实时消息推送 +- 批量操作优化 +- 离线消息同步 + +### 2.4 ljwx-watch (HarmonyOS手表) + +**现有功能**: +- HTTP和蓝牙双模式消息接收 +- `fetchMessageFromServer()` - HTTP拉取 +- 蓝牙GATT命令处理 +- 系统通知显示和用户确认 + +**优化需求**: +- 消息确认回传优化 +- 显示效果增强 +- 电池优化 + +## 三、集成优化方案 + +### 3.1 消息统一模型扩展 + +在现有数据库基础上增加字段,支持完整的消息生命周期: + +```sql +-- 扩展 t_device_message 表 +ALTER TABLE t_device_message ADD COLUMN title VARCHAR(500); +ALTER TABLE t_device_message ADD COLUMN content TEXT; +ALTER TABLE t_device_message ADD COLUMN priority INTEGER DEFAULT 3; +ALTER TABLE t_device_message ADD COLUMN urgency VARCHAR(20) DEFAULT 'medium'; +ALTER TABLE t_device_message ADD COLUMN channels JSON; -- ["message","push","wechat","watch"] +ALTER TABLE t_device_message ADD COLUMN require_ack BOOLEAN DEFAULT false; +ALTER TABLE t_device_message ADD COLUMN expiry_time DATETIME; +ALTER TABLE t_device_message ADD COLUMN metadata JSON; + +-- 扩展 t_device_message_detail 表 +ALTER TABLE t_device_message_detail ADD COLUMN distribution_id VARCHAR(255); +ALTER TABLE t_device_message_detail ADD COLUMN target_type VARCHAR(50); +ALTER TABLE t_device_message_detail ADD COLUMN target_id VARCHAR(255); +ALTER TABLE t_device_message_detail ADD COLUMN delivery_status VARCHAR(50); +ALTER TABLE t_device_message_detail ADD COLUMN channel VARCHAR(50); +ALTER TABLE t_device_message_detail ADD COLUMN response_time INTEGER; +ALTER TABLE t_device_message_detail ADD COLUMN acknowledge_time DATETIME; +ALTER TABLE t_device_message_detail ADD COLUMN delivery_details JSON; +``` + +### 3.2 ljwx-boot 服务层增强 + +基于现有的 `TDeviceMessageServiceImpl` 扩展功能: + +```java +@Service +public class EnhancedMessageService extends TDeviceMessageServiceImpl { + + @Autowired + private RedisTemplate redisTemplate; + + /** + * 统一消息发布接口 + */ + @Transactional + public Long publishMessage(UnifiedMessageRequest request) { + // 1. 保存主消息 + TDeviceMessage message = buildDeviceMessage(request); + save(message); + + // 2. 创建分发记录 + List details = createDistributionDetails(message, request); + messageDetailService.saveBatch(details); + + // 3. 发布到Redis + publishToRedis(message, details); + + // 4. 记录生命周期事件 + recordLifecycleEvent(message.getId(), "created", getCurrentUser()); + + return message.getId(); + } + + /** + * 消息状态批量更新 + */ + public void updateMessageStatus(Long messageId, String status, List targetIds) { + // 更新主消息状态 + TDeviceMessage message = getById(messageId); + message.setMessageStatus(status); + updateById(message); + + // 更新详情状态 + LambdaUpdateWrapper wrapper = new LambdaUpdateWrapper<>(); + wrapper.eq(TDeviceMessageDetail::getMessageId, messageId) + .in(TDeviceMessageDetail::getTargetId, targetIds) + .set(TDeviceMessageDetail::getDeliveryStatus, status) + .set(TDeviceMessageDetail::getAcknowledgeTime, LocalDateTime.now()); + + messageDetailService.update(wrapper); + + // 发布状态更新事件 + publishStatusUpdate(messageId, status); + } + + private void publishToRedis(TDeviceMessage message, List details) { + for (TDeviceMessageDetail detail : details) { + String channel = "message:" + detail.getChannel(); + Map payload = buildRedisPayload(message, detail); + redisTemplate.convertAndSend(channel, payload); + } + } +} +``` + +### 3.3 ljwx-phone 实时消息集成 + +在现有Flutter代码基础上增加Redis订阅: + +```dart +// lib/services/enhanced_message_service.dart +class EnhancedMessageService extends ApiService { + StreamSubscription? _redisSubscription; + final StreamController _messageStreamController = StreamController.broadcast(); + + Stream get messageStream => _messageStreamController.stream; + + @override + void initState() { + super.initState(); + _subscribeToRedisMessages(); + _startPeriodicSync(); // 保持现有的定时同步作为备用 + } + + void _subscribeToRedisMessages() { + // 订阅Redis消息通道 + String channel = 'message:${deviceSn}'; + _redisSubscription = RedisClient.subscribe(channel).listen((data) { + try { + Map messageData = jsonDecode(data); + Message message = Message.fromJson(messageData); + + // 更新本地存储 + _updateLocalMessage(message); + + // 发布给UI层 + _messageStreamController.add(message); + + // 转发给手表(如果连接) + if (BluetoothService().isConnected) { + BluetoothService().forwardMessageToWatch(messageData); + } + + } catch (e) { + print('解析Redis消息失败: $e'); + } + }); + } + + @override + Future markMessageAsRead(String deviceSn, { + DateTime? receivedTime, + String? messageId, + Map? originalMessage + }) async { + + // 调用现有API + final success = await super.markMessageAsRead(deviceSn, + receivedTime: receivedTime, messageId: messageId, originalMessage: originalMessage); + + if (success) { + // 发布确认事件到Redis + Map ackEvent = { + 'type': 'message_acknowledged', + 'messageId': messageId, + 'deviceSn': deviceSn, + 'acknowledgeTime': receivedTime?.toIso8601String() ?? DateTime.now().toIso8601String(), + 'source': 'phone' + }; + + RedisClient.publish('message:acknowledgments', jsonEncode(ackEvent)); + } + + return success; + } +} +``` + +### 3.4 ljwx-bigscreen 增强消息跟踪 + +基于现有的消息处理代码增加实时跟踪: + +```python +# bigScreen/enhanced_message_tracking.py +class EnhancedMessageTracker: + + @staticmethod + def create_message_with_tracking(message_data): + """创建消息并启用跟踪""" + try: + # 使用现有的DeviceMessage模型 + message = DeviceMessage( + device_sn=message_data.get('device_sn'), + message=message_data.get('content'), + message_type=message_data.get('message_type', 'notification'), + sender_type=message_data.get('sender_type', 'admin'), + receiver_type=message_data.get('receiver_type', 'device'), + org_id=message_data.get('org_id'), + user_id=message_data.get('user_id'), + customer_id=message_data.get('customer_id', 0), + # 新增字段 + title=message_data.get('title'), + priority=message_data.get('priority', 3), + urgency=message_data.get('urgency', 'medium'), + require_ack=message_data.get('require_ack', False), + channels=json.dumps(message_data.get('channels', ['message'])), + metadata=json.dumps(message_data.get('metadata', {})) + ) + + db.session.add(message) + db.session.commit() + + # 创建分发记录 + targets = message_data.get('targets', []) + for target in targets: + detail = DeviceMessageDetail( + message_id=message.id, + device_sn=target.get('device_sn'), + message=message.message, + message_type=message.message_type, + sender_type=message.sender_type, + receiver_type=message.receiver_type, + customer_id=message.customer_id, + org_id=message.org_id, + # 新增字段 + target_type=target.get('target_type', 'device'), + target_id=target.get('target_id'), + delivery_status='pending', + channel=target.get('channel', 'message') + ) + db.session.add(detail) + + db.session.commit() + + # 发布到Redis + redis_client.publish(f'message:created', json.dumps({ + 'messageId': message.id, + 'targets': [detail.device_sn for detail in message.details] + })) + + return message + + except Exception as e: + db.session.rollback() + print(f"创建消息失败: {e}") + return None + + @staticmethod + def get_message_lifecycle(message_id): + """获取消息生命周期报告""" + message = DeviceMessage.query.get(message_id) + if not message: + return None + + details = DeviceMessageDetail.query.filter_by(message_id=message_id).all() + + # 计算统计数据 + total_targets = len(details) + delivered = sum(1 for d in details if d.delivery_status in ['delivered', 'acknowledged']) + acknowledged = sum(1 for d in details if d.delivery_status == 'acknowledged') + pending = sum(1 for d in details if d.delivery_status == 'pending') + failed = sum(1 for d in details if d.delivery_status == 'failed') + + # 计算响应时间 + response_times = [d.response_time for d in details if d.response_time] + avg_response_time = sum(response_times) / len(response_times) if response_times else 0 + + return { + 'messageId': message_id, + 'messageType': message.message_type, + 'createTime': message.create_time.isoformat(), + 'sentTime': message.sent_time.isoformat() if message.sent_time else None, + 'progress': { + 'totalTargets': total_targets, + 'delivered': delivered, + 'acknowledged': acknowledged, + 'pending': pending, + 'failed': failed, + 'completionRate': (acknowledged / total_targets * 100) if total_targets > 0 else 0, + 'avgResponseTime': avg_response_time + }, + 'details': [ + { + 'targetId': d.target_id, + 'deviceSn': d.device_sn, + 'deliveryStatus': d.delivery_status, + 'channel': d.channel, + 'responseTime': d.response_time, + 'acknowledgeTime': d.acknowledge_time.isoformat() if d.acknowledge_time else None + } + for d in details + ] + } + +# bigScreen.py 新增API端点 +@app.route('/api/message/lifecycle/', methods=['GET']) +def get_message_lifecycle(message_id): + """获取消息生命周期""" + lifecycle_data = EnhancedMessageTracker.get_message_lifecycle(message_id) + if lifecycle_data: + return jsonify({'success': True, 'data': lifecycle_data}) + else: + return jsonify({'success': False, 'error': '消息不存在'}), 404 + +@app.route('/api/message/stats/', methods=['GET']) +def get_organization_message_stats(org_id): + """获取组织消息统计""" + start_date = request.args.get('startDate') + end_date = request.args.get('endDate') + + query = DeviceMessage.query.filter_by(org_id=str(org_id)) + + if start_date: + query = query.filter(DeviceMessage.create_time >= datetime.fromisoformat(start_date)) + if end_date: + query = query.filter(DeviceMessage.create_time <= datetime.fromisoformat(end_date)) + + messages = query.all() + + # 生成统计数据 + stats = { + 'totalMessages': len(messages), + 'messageTypes': {}, + 'dailyTrend': {}, + 'avgCompletionRate': 0, + 'avgResponseTime': 0 + } + + # ... 统计计算逻辑 + + return jsonify({'success': True, 'data': stats}) +``` + +### 3.5 ljwx-watch 消息处理优化 + +基于现有的HTTP和蓝牙双模式,增加确认反馈优化: + +```java +// HttpService.java 消息确认增强 +public void enhancedMessageConfirmation(JSONObject message) { + try { + String messageId = message.getString("message_id"); + + // 构建增强的确认响应 + JSONObject enhancedResponse = new JSONObject(); + enhancedResponse.put("type", "message_acknowledgment"); + enhancedResponse.put("messageId", messageId); + enhancedResponse.put("deviceSn", dataManager.getDeviceSn()); + enhancedResponse.put("acknowledgeTime", getCurrentTimestamp()); + enhancedResponse.put("responseTime", calculateResponseTime(message)); + enhancedResponse.put("channel", "http"); + enhancedResponse.put("deviceInfo", getDeviceInfo()); + enhancedResponse.put("userAction", "acknowledged"); // 用户主动确认 + + // 复制原消息字段 + copyOriginalMessageFields(enhancedResponse, message); + + // 发送确认 + String confirmUrl = dataManager.getFetchMessageUrl() + "/acknowledge"; + JSONObject response = postDataToServer(confirmUrl, enhancedResponse); + + if (response != null && response.getBoolean("success")) { + Log.i(TAG, "✅ 消息确认发送成功: " + messageId); + + // 本地标记为已确认 + markMessageAsAcknowledged(messageId); + + } else { + Log.w(TAG, "⚠️ 消息确认发送失败: " + messageId); + // 缓存确认,稍后重试 + cacheAcknowledgment(enhancedResponse); + } + + } catch (Exception e) { + Log.e(TAG, "❌ 消息确认处理异常: " + e.getMessage()); + } +} + +private int calculateResponseTime(JSONObject message) { + try { + String sentTimeStr = message.getString("sent_time"); + long sentTime = parseTimestamp(sentTimeStr); + long currentTime = System.currentTimeMillis(); + return (int) ((currentTime - sentTime) / 1000); // 返回秒数 + } catch (Exception e) { + return -1; // 无法计算响应时间 + } +} + +private JSONObject getDeviceInfo() { + JSONObject deviceInfo = new JSONObject(); + try { + deviceInfo.put("watchModel", "HarmonyOS Watch"); + deviceInfo.put("osVersion", "4.0"); + deviceInfo.put("appVersion", getAppVersion()); + deviceInfo.put("batteryLevel", getBatteryLevel()); + deviceInfo.put("signalStrength", getSignalStrength()); + } catch (Exception e) { + Log.w(TAG, "获取设备信息失败: " + e.getMessage()); + } + return deviceInfo; +} +``` + +## 四、实施路径 + +### 4.1 阶段一:数据库扩展(1周) +1. 执行SQL脚本扩展现有表结构 +2. 更新ljwx-boot实体类 +3. 更新ljwx-bigscreen SQLAlchemy模型 +4. 数据迁移和兼容性测试 + +### 4.2 阶段二:后端服务增强(2周) +1. ljwx-boot增强消息服务 +2. ljwx-bigscreen增强跟踪功能 +3. Redis集成和实时通信 +4. API接口联调测试 + +### 4.3 阶段三:移动端实时化(1周) +1. ljwx-phone集成Redis订阅 +2. 增强消息确认机制 +3. 蓝牙通信优化 +4. 离线同步完善 + +### 4.4 阶段四:手表端优化(1周) +1. ljwx-watch确认反馈增强 +2. 消息显示优化 +3. 电池性能优化 +4. 端到端测试 + +### 4.5 阶段五:整体联调(1周) +1. 四端数据流测试 +2. 性能压力测试 +3. 用户体验优化 +4. 文档和培训 + +## 五、预期效果 + +1. **完整数据流打通**: 平台→手机→手表→确认回传的完整链路 +2. **实时状态同步**: 基于Redis的实时消息推送和状态更新 +3. **全面生命周期跟踪**: 从消息创建到确认完成的全流程监控 +4. **增强统计分析**: 组织级、部门级、个人级的多维度分析 +5. **向后兼容**: 保持现有API和数据结构的兼容性 + +此方案基于现有代码架构,最大程度复用已有功能,通过渐进式改进实现完整的消息数据流集成。 \ No newline at end of file diff --git a/utils/monitor_alert_cache_performance.py b/utils/monitor_alert_cache_performance.py new file mode 100644 index 0000000..7807e6c --- /dev/null +++ b/utils/monitor_alert_cache_performance.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +""" +告警规则缓存性能监控脚本 +实时监控Redis缓存告警系统的性能指标 +""" + +import time +import json +import logging +import sys +import os +from datetime import datetime, timedelta +from typing import Dict, List, Any +import threading + +# 添加当前路径 +current_dir = os.path.dirname(__file__) +sys.path.insert(0, current_dir) + +from redis_cache_generate_alerts import get_redis_cached_generator + +# 设置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +class AlertCacheMonitor: + """告警缓存性能监控器""" + + def __init__(self, interval: int = 10): + self.interval = interval # 监控间隔(秒) + self.running = False + self.monitor_thread = None + self.generator = None + self.history = [] # 历史数据 + self.max_history = 100 # 最大历史记录数 + + def start_monitoring(self): + """开始监控""" + if self.running: + logger.warning("监控器已在运行中") + return + + print("🔍 启动告警缓存性能监控器") + try: + self.generator = get_redis_cached_generator() + self.running = True + + self.monitor_thread = threading.Thread( + target=self._monitor_loop, + daemon=True, + name="AlertCacheMonitor" + ) + self.monitor_thread.start() + + print(f"✅ 监控器已启动,监控间隔: {self.interval}秒") + + except Exception as e: + print(f"❌ 监控器启动失败: {e}") + self.running = False + + def stop_monitoring(self): + """停止监控""" + self.running = False + print("🛑 监控器已停止") + + def _monitor_loop(self): + """监控主循环""" + print("📊 开始性能监控...") + print("-" * 80) + + while self.running: + try: + # 收集性能数据 + metrics = self._collect_metrics() + + # 保存历史数据 + self._save_metrics(metrics) + + # 显示实时数据 + self._display_metrics(metrics) + + # 检查告警 + self._check_alerts(metrics) + + # 等待下一次监控 + time.sleep(self.interval) + + except Exception as e: + logger.error(f"监控循环异常: {e}") + time.sleep(self.interval) + + def _collect_metrics(self) -> Dict[str, Any]: + """收集性能指标""" + try: + # 获取基础统计 + stats = self.generator.get_stats() + + # 获取当前时间 + timestamp = datetime.now() + + # 构建指标数据 + metrics = { + 'timestamp': timestamp, + 'total_processed': stats.get('total_processed', 0), + 'cache_hits': stats.get('cache_hits', 0), + 'db_fallbacks': stats.get('db_fallbacks', 0), + 'cache_hit_rate': stats.get('cache_hit_rate', '0.00%'), + 'alerts_generated': stats.get('alerts_generated', 0), + 'avg_processing_time': stats.get('avg_processing_time', '0.000s'), + } + + # Redis连接状态 + cache_manager_status = stats.get('cache_manager_status', {}) + if isinstance(cache_manager_status, dict): + redis_status = cache_manager_status.get('redis_connection_status', {}) + metrics['bigscreen_redis_ok'] = redis_status.get('bigscreen_redis', False) + metrics['boot_redis_ok'] = redis_status.get('boot_redis', False) + metrics['local_cache_size'] = cache_manager_status.get('local_cache_size', 0) + metrics['subscriber_running'] = cache_manager_status.get('subscriber_running', False) + else: + metrics['bigscreen_redis_ok'] = False + metrics['boot_redis_ok'] = False + metrics['local_cache_size'] = 0 + metrics['subscriber_running'] = False + + return metrics + + except Exception as e: + logger.error(f"收集指标失败: {e}") + return {'timestamp': datetime.now(), 'error': str(e)} + + def _save_metrics(self, metrics: Dict[str, Any]): + """保存历史指标""" + self.history.append(metrics) + + # 保持历史记录数量限制 + if len(self.history) > self.max_history: + self.history.pop(0) + + def _display_metrics(self, metrics: Dict[str, Any]): + """显示实时指标""" + timestamp = metrics.get('timestamp', datetime.now()) + + print(f"\n⏰ {timestamp.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"📊 处理统计: 总计={metrics.get('total_processed', 0)}, 告警={metrics.get('alerts_generated', 0)}") + print(f"🔥 缓存性能: 命中={metrics.get('cache_hits', 0)}, 兜底={metrics.get('db_fallbacks', 0)}, 命中率={metrics.get('cache_hit_rate', 'N/A')}") + print(f"⚡ 处理时间: 平均={metrics.get('avg_processing_time', 'N/A')}") + + # Redis连接状态 + bigscreen_ok = metrics.get('bigscreen_redis_ok', False) + boot_ok = metrics.get('boot_redis_ok', False) + subscriber_ok = metrics.get('subscriber_running', False) + + bigscreen_status = "✅" if bigscreen_ok else "❌" + boot_status = "✅" if boot_ok else "❌" + subscriber_status = "✅" if subscriber_ok else "❌" + + print(f"🔗 连接状态: bigscreen-redis={bigscreen_status}, boot-redis={boot_status}, 订阅者={subscriber_status}") + print(f"📋 本地缓存: {metrics.get('local_cache_size', 0)}个客户") + print("-" * 80) + + def _check_alerts(self, metrics: Dict[str, Any]): + """检查性能告警""" + alerts = [] + + # 检查Redis连接 + if not metrics.get('bigscreen_redis_ok', True): + alerts.append("❌ ljwx-bigscreen Redis连接异常") + if not metrics.get('boot_redis_ok', True): + alerts.append("❌ ljwx-boot Redis连接异常") + + # 检查订阅者状态 + if not metrics.get('subscriber_running', True): + alerts.append("⚠️ Redis订阅者未运行") + + # 检查缓存命中率 + cache_hit_rate_str = metrics.get('cache_hit_rate', '0.00%') + try: + cache_hit_rate = float(cache_hit_rate_str.replace('%', '')) + if cache_hit_rate < 80: + alerts.append(f"⚠️ 缓存命中率过低: {cache_hit_rate_str}") + except: + pass + + # 检查数据库兜底频率 + db_fallbacks = metrics.get('db_fallbacks', 0) + cache_hits = metrics.get('cache_hits', 0) + total_queries = db_fallbacks + cache_hits + + if total_queries > 0: + fallback_rate = db_fallbacks / total_queries * 100 + if fallback_rate > 20: + alerts.append(f"⚠️ 数据库兜底频率过高: {fallback_rate:.1f}%") + + # 显示告警 + if alerts: + print("🚨 性能告警:") + for alert in alerts: + print(f" {alert}") + + def get_performance_report(self) -> Dict[str, Any]: + """生成性能报告""" + if not self.history: + return {"error": "无历史数据"} + + # 计算统计指标 + total_records = len(self.history) + recent_records = self.history[-10:] if len(self.history) >= 10 else self.history + + # 最新状态 + latest = self.history[-1] + + # 处理量趋势 + if len(self.history) >= 2: + first = self.history[0] + last = self.history[-1] + processed_growth = last.get('total_processed', 0) - first.get('total_processed', 0) + else: + processed_growth = 0 + + # Redis连接稳定性 + redis_stability = sum(1 for m in recent_records if m.get('bigscreen_redis_ok') and m.get('boot_redis_ok')) / len(recent_records) * 100 + + return { + 'report_time': datetime.now().isoformat(), + 'monitoring_duration': f"{total_records * self.interval}秒", + 'total_records': total_records, + 'latest_status': { + 'total_processed': latest.get('total_processed', 0), + 'cache_hit_rate': latest.get('cache_hit_rate', 'N/A'), + 'local_cache_size': latest.get('local_cache_size', 0), + 'redis_connections_ok': latest.get('bigscreen_redis_ok', False) and latest.get('boot_redis_ok', False) + }, + 'performance_trends': { + 'processed_growth': processed_growth, + 'redis_stability': f"{redis_stability:.1f}%", + }, + 'recommendations': self._generate_recommendations(recent_records) + } + + def _generate_recommendations(self, recent_records: List[Dict]) -> List[str]: + """生成性能建议""" + recommendations = [] + + if not recent_records: + return recommendations + + # 检查缓存命中率 + avg_cache_hits = sum(r.get('cache_hits', 0) for r in recent_records) / len(recent_records) + avg_db_fallbacks = sum(r.get('db_fallbacks', 0) for r in recent_records) / len(recent_records) + + if avg_db_fallbacks > avg_cache_hits: + recommendations.append("建议检查Redis缓存配置,数据库兜底频率过高") + + # 检查Redis连接稳定性 + disconnection_count = sum(1 for r in recent_records + if not (r.get('bigscreen_redis_ok', True) and r.get('boot_redis_ok', True))) + if disconnection_count > 0: + recommendations.append("发现Redis连接不稳定,建议检查网络和Redis服务状态") + + # 检查订阅者状态 + subscriber_down_count = sum(1 for r in recent_records if not r.get('subscriber_running', True)) + if subscriber_down_count > 0: + recommendations.append("Redis订阅者存在中断,建议检查订阅者服务状态") + + # 检查本地缓存大小 + latest_cache_size = recent_records[-1].get('local_cache_size', 0) + if latest_cache_size == 0: + recommendations.append("本地缓存为空,可能需要预加载常用客户的告警规则") + elif latest_cache_size > 100: + recommendations.append("本地缓存过大,建议优化缓存策略或增加内存限制") + + if not recommendations: + recommendations.append("系统运行正常,无特殊建议") + + return recommendations + + def save_report_to_file(self, filename: str = None): + """保存性能报告到文件""" + if not filename: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"alert_cache_performance_report_{timestamp}.json" + + report = self.get_performance_report() + + try: + with open(filename, 'w', encoding='utf-8') as f: + json.dump(report, f, ensure_ascii=False, indent=2, default=str) + print(f"📄 性能报告已保存: {filename}") + except Exception as e: + print(f"❌ 保存报告失败: {e}") + +def main(): + """主函数""" + import argparse + + parser = argparse.ArgumentParser(description="告警规则缓存性能监控") + parser.add_argument('-i', '--interval', type=int, default=10, help='监控间隔(秒)') + parser.add_argument('-d', '--duration', type=int, default=0, help='监控时长(秒),0表示持续监控') + parser.add_argument('-r', '--report', action='store_true', help='生成性能报告文件') + + args = parser.parse_args() + + monitor = AlertCacheMonitor(interval=args.interval) + + try: + # 启动监控 + monitor.start_monitoring() + + if args.duration > 0: + # 定时监控 + print(f"⏱️ 将监控{args.duration}秒后自动停止") + time.sleep(args.duration) + monitor.stop_monitoring() + + # 生成报告 + if args.report: + monitor.save_report_to_file() + else: + # 持续监控 + print("⌨️ 按 Ctrl+C 停止监控") + try: + while monitor.running: + time.sleep(1) + except KeyboardInterrupt: + print("\n⏹️ 收到停止信号") + monitor.stop_monitoring() + + if args.report: + monitor.save_report_to_file() + + except Exception as e: + print(f"❌ 监控过程出现异常: {e}") + monitor.stop_monitoring() + + print("👋 监控结束") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/utils/redis_cache_generate_alerts.py b/utils/redis_cache_generate_alerts.py new file mode 100644 index 0000000..aac33eb --- /dev/null +++ b/utils/redis_cache_generate_alerts.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +""" +Redis缓存优化版的generate_alerts函数 +集成ljwx-boot和ljwx-bigscreen之间的告警规则缓存同步 +""" + +import time +import json +import logging +import sys +import os +from typing import Dict, List, Any, Optional +from dataclasses import dataclass + +# 添加ljwx-bigscreen路径以导入缓存管理器 +current_dir = os.path.dirname(__file__) +bigscreen_path = os.path.join(current_dir, '../ljwx-bigscreen/bigscreen') +sys.path.insert(0, bigscreen_path) + +# 设置日志 +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +try: + from alert_rules_cache_manager import get_alert_rules_cache_manager, AlertRule + CACHE_MANAGER_AVAILABLE = True + logger.info("✅ 告警规则缓存管理器导入成功") +except ImportError as e: + logger.warning(f"⚠️ 无法导入告警规则缓存管理器: {e}") + CACHE_MANAGER_AVAILABLE = False + # 定义基本的AlertRule类用于兜底 + @dataclass + class AlertRule: + id: int + rule_type: str + physical_sign: str + threshold_min: Optional[float] + threshold_max: Optional[float] + trend_duration: Optional[int] + severity_level: str + alert_message: str + customer_id: int + is_enabled: bool = True + +@dataclass +class AlertCandidate: + """告警候选数据结构""" + rule_id: int + device_sn: str + user_id: int + alert_message: str + severity_level: str + physical_sign: str + current_value: Any + threshold_violated: str + +class RedisCachedAlertGenerator: + """Redis缓存优化的告警生成器""" + + def __init__(self): + self.alert_rules_manager = None + self.stats = { + 'total_processed': 0, + 'cache_hits': 0, + 'db_fallbacks': 0, + 'alerts_generated': 0, + 'processing_time': 0 + } + self._init_cache_manager() + + def _init_cache_manager(self): + """初始化缓存管理器""" + if CACHE_MANAGER_AVAILABLE: + try: + self.alert_rules_manager = get_alert_rules_cache_manager() + self.alert_rules_manager.start_subscriber() + logger.info("🚀 Redis订阅者已启动,监听告警规则更新") + except Exception as e: + logger.error(f"❌ 初始化缓存管理器失败: {e}") + self.alert_rules_manager = None + else: + logger.info("📋 使用数据库查询模式") + + def get_alert_rules(self, customer_id: int) -> List[AlertRule]: + """获取告警规则 - 优先使用Redis缓存""" + if not customer_id: + return [] + + start_time = time.time() + + # 优先使用Redis缓存管理器 + if self.alert_rules_manager: + try: + rules = self.alert_rules_manager.get_alert_rules(customer_id) + if rules: + self.stats['cache_hits'] += 1 + processing_time = time.time() - start_time + logger.debug(f"🎯 Redis缓存命中: customer_id={customer_id}, rules={len(rules)}, time={processing_time:.3f}s") + return rules + except Exception as e: + logger.error(f"❌ Redis缓存获取失败: {e}") + + # 数据库兜底 + self.stats['db_fallbacks'] += 1 + rules = self._fetch_from_database(customer_id) + processing_time = time.time() - start_time + logger.info(f"📊 数据库兜底查询: customer_id={customer_id}, rules={len(rules)}, time={processing_time:.3f}s") + return rules + + def _fetch_from_database(self, customer_id: int) -> List[AlertRule]: + """从数据库获取告警规则 - 兜底方案""" + # 这里应该是真实的数据库查询逻辑 + # 为了演示,返回模拟数据 + logger.warning(f"⚠️ 数据库兜底被触发,建议检查Redis缓存状态: customer_id={customer_id}") + + # 模拟数据库查询延迟 + time.sleep(0.05) + + return [ + AlertRule( + id=1, + rule_type='THRESHOLD', + physical_sign='heart_rate', + threshold_min=50.0, + threshold_max=120.0, + trend_duration=None, + severity_level='HIGH', + alert_message='心率异常', + customer_id=customer_id, + is_enabled=True + ), + AlertRule( + id=2, + rule_type='THRESHOLD', + physical_sign='blood_oxygen', + threshold_min=95.0, + threshold_max=100.0, + trend_duration=None, + severity_level='MEDIUM', + alert_message='血氧异常', + customer_id=customer_id, + is_enabled=True + ) + ] + + def generate_alerts(self, health_data_list: List[Dict]) -> List[AlertCandidate]: + """生成告警 - Redis缓存优化版""" + start_time = time.time() + alerts = [] + + try: + for health_data in health_data_list: + customer_id = health_data.get('customer_id') + if not customer_id: + logger.warning("健康数据缺少customer_id字段,跳过告警检查") + continue + + # 获取告警规则(Redis缓存优先) + rules = self.get_alert_rules(customer_id) + + # 应用告警规则 + device_alerts = self._apply_rules(health_data, rules) + alerts.extend(device_alerts) + + self.stats['total_processed'] += len(health_data_list) + self.stats['alerts_generated'] += len(alerts) + self.stats['processing_time'] += time.time() - start_time + + processing_time = time.time() - start_time + logger.info(f"⚡ 告警生成完成: 处理{len(health_data_list)}条数据, 生成{len(alerts)}个告警, 耗时{processing_time:.3f}s") + + except Exception as e: + logger.error(f"❌ 告警生成失败: {e}") + import traceback + traceback.print_exc() + + return alerts + + def _apply_rules(self, health_data: Dict, rules: List[AlertRule]) -> List[AlertCandidate]: + """应用告警规则""" + alerts = [] + + for rule in rules: + if not rule.is_enabled: + continue + + # 检查物理指标 + physical_value = health_data.get(rule.physical_sign) + if physical_value is None: + continue + + # 阈值检查 + violation = self._check_threshold_violation(physical_value, rule) + if violation: + alert = AlertCandidate( + rule_id=rule.id, + device_sn=health_data.get('device_sn', ''), + user_id=health_data.get('user_id', 0), + alert_message=rule.alert_message, + severity_level=rule.severity_level, + physical_sign=rule.physical_sign, + current_value=physical_value, + threshold_violated=violation + ) + alerts.append(alert) + + return alerts + + def _check_threshold_violation(self, value: Any, rule: AlertRule) -> Optional[str]: + """检查阈值违规""" + try: + numeric_value = float(value) + + if rule.threshold_min is not None and numeric_value < rule.threshold_min: + return f"低于下限{rule.threshold_min}" + elif rule.threshold_max is not None and numeric_value > rule.threshold_max: + return f"超出上限{rule.threshold_max}" + + except (ValueError, TypeError): + logger.warning(f"无法转换数值: {value} for rule {rule.id}") + + return None + + def get_stats(self) -> Dict[str, Any]: + """获取性能统计""" + stats = self.stats.copy() + + # 计算命中率 + total_queries = stats['cache_hits'] + stats['db_fallbacks'] + if total_queries > 0: + stats['cache_hit_rate'] = f"{(stats['cache_hits'] / total_queries * 100):.2f}%" + else: + stats['cache_hit_rate'] = "0.00%" + + # 计算平均处理时间 + if stats['total_processed'] > 0: + stats['avg_processing_time'] = f"{(stats['processing_time'] / stats['total_processed']):.3f}s" + else: + stats['avg_processing_time'] = "0.000s" + + # 缓存管理器状态 + if self.alert_rules_manager: + try: + cache_stats = self.alert_rules_manager.get_cache_stats() + stats['cache_manager_status'] = cache_stats + except Exception as e: + stats['cache_manager_error'] = str(e) + else: + stats['cache_manager_status'] = "未启用" + + return stats + + def preload_rules(self, customer_ids: List[int]): + """预加载指定客户的告警规则""" + if self.alert_rules_manager: + self.alert_rules_manager.preload_customer_rules(customer_ids) + logger.info(f"📋 预加载告警规则: {len(customer_ids)}个客户") + + def clear_cache(self, customer_id: Optional[int] = None): + """清理缓存""" + if self.alert_rules_manager: + self.alert_rules_manager.clear_local_cache(customer_id) + logger.info(f"🧹 清理告警规则缓存: customer_id={customer_id or 'all'}") + + def shutdown(self): + """关闭生成器""" + if self.alert_rules_manager: + self.alert_rules_manager.stop_subscriber() + logger.info("🛑 Redis订阅者已停止") + +# 全局实例 +_global_generator = None + +def get_redis_cached_generator() -> RedisCachedAlertGenerator: + """获取全局Redis缓存告警生成器实例""" + global _global_generator + if _global_generator is None: + _global_generator = RedisCachedAlertGenerator() + logger.info("🚀 全局Redis缓存告警生成器已初始化") + return _global_generator + +def redis_cached_generate_alerts(health_data_list: List[Dict]) -> List[AlertCandidate]: + """Redis缓存优化版的generate_alerts函数 + + 替代原有的generate_alerts函数,提供: + 1. Redis缓存优先的告警规则获取 + 2. 实时的缓存更新通知 + 3. 数据库兜底保障 + 4. 性能监控统计 + + Args: + health_data_list: 健康数据列表,每项应包含customer_id字段 + + Returns: + 告警候选列表 + """ + generator = get_redis_cached_generator() + return generator.generate_alerts(health_data_list) + +def init_redis_cache_manager(): + """初始化Redis缓存管理器 - 应用启动时调用""" + generator = get_redis_cached_generator() + logger.info("✅ Redis缓存告警生成器初始化完成") + return generator + +def shutdown_redis_cache_manager(): + """关闭Redis缓存管理器 - 应用关闭时调用""" + global _global_generator + if _global_generator: + _global_generator.shutdown() + _global_generator = None + logger.info("🛑 Redis缓存告警生成器已关闭") + +if __name__ == "__main__": + # 测试代码 + print("🧪 Redis缓存告警生成器测试") + + # 初始化生成器 + generator = get_redis_cached_generator() + + # 等待Redis订阅者启动 + time.sleep(2) + + # 测试数据 + test_health_data = [ + { + 'device_sn': 'TEST001', + 'user_id': 1, + 'customer_id': 1, + 'heart_rate': 85, + 'blood_oxygen': 98, + 'temperature': 36.5, + 'timestamp': '2025-09-09 10:00:00' + }, + { + 'device_sn': 'TEST002', + 'user_id': 2, + 'customer_id': 1, + 'heart_rate': 130, # 超出阈值 + 'blood_oxygen': 92, # 低于阈值 + 'temperature': 37.2, + 'timestamp': '2025-09-09 10:01:00' + } + ] + + try: + print("\n📋 测试告警规则获取") + rules = generator.get_alert_rules(1) + print(f"获取到告警规则: {len(rules)}条") + for rule in rules: + print(f" - {rule.rule_type}: {rule.physical_sign} [{rule.threshold_min}-{rule.threshold_max}] ({rule.severity_level})") + + print("\n⚡ 测试告警生成") + alerts = generator.generate_alerts(test_health_data) + print(f"生成告警: {len(alerts)}个") + for alert in alerts: + print(f" - {alert.physical_sign}: {alert.current_value} ({alert.threshold_violated}) - {alert.severity_level}") + + print("\n📊 性能统计") + stats = generator.get_stats() + for key, value in stats.items(): + if isinstance(value, dict): + print(f"{key}:") + for sub_key, sub_value in value.items(): + print(f" {sub_key}: {sub_value}") + else: + print(f"{key}: {value}") + + print("\n✅ 测试完成") + + except Exception as e: + print(f"❌ 测试失败: {e}") + import traceback + traceback.print_exc() + finally: + generator.shutdown() + print("🛑 生成器已关闭") \ No newline at end of file diff --git a/utils/test_redis_cache_alerts.py b/utils/test_redis_cache_alerts.py new file mode 100644 index 0000000..0cb8626 --- /dev/null +++ b/utils/test_redis_cache_alerts.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +""" +Redis缓存告警系统测试脚本 +测试ljwx-boot和ljwx-bigscreen之间的告警规则缓存同步 +""" + +import time +import json +import logging +import sys +import os +from typing import Dict, List + +# 添加当前路径 +current_dir = os.path.dirname(__file__) +sys.path.insert(0, current_dir) + +from redis_cache_generate_alerts import ( + get_redis_cached_generator, + redis_cached_generate_alerts, + init_redis_cache_manager, + shutdown_redis_cache_manager +) + +# 设置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + +class AlertCacheTestSuite: + """告警规则缓存测试套件""" + + def __init__(self): + self.generator = None + self.test_results = [] + + def setup(self): + """测试初始化""" + print("🔧 初始化测试环境...") + try: + self.generator = init_redis_cache_manager() + # 等待Redis订阅者启动 + time.sleep(3) + print("✅ 测试环境初始化完成") + return True + except Exception as e: + print(f"❌ 测试环境初始化失败: {e}") + return False + + def teardown(self): + """测试清理""" + print("🧹 清理测试环境...") + try: + shutdown_redis_cache_manager() + print("✅ 测试环境清理完成") + except Exception as e: + print(f"⚠️ 测试环境清理异常: {e}") + + def test_redis_connection(self) -> bool: + """测试Redis连接状态""" + print("\n🔗 测试Redis连接状态") + try: + stats = self.generator.get_stats() + cache_status = stats.get('cache_manager_status', {}) + + if isinstance(cache_status, dict): + redis_connections = cache_status.get('redis_connection_status', {}) + bigscreen_redis = redis_connections.get('bigscreen_redis', False) + boot_redis = redis_connections.get('boot_redis', False) + + print(f" ljwx-bigscreen Redis (DB=0): {'✅ 连接正常' if bigscreen_redis else '❌ 连接失败'}") + print(f" ljwx-boot Redis (DB=1): {'✅ 连接正常' if boot_redis else '❌ 连接失败'}") + + success = bigscreen_redis and boot_redis + else: + print(f" 缓存管理器状态: {cache_status}") + success = False + + self.test_results.append(('Redis连接测试', success)) + return success + + except Exception as e: + print(f" ❌ Redis连接测试失败: {e}") + self.test_results.append(('Redis连接测试', False)) + return False + + def test_alert_rules_retrieval(self) -> bool: + """测试告警规则获取""" + print("\n📋 测试告警规则获取") + try: + test_customer_id = 1 + + # 第一次获取(可能从数据库或Redis获取) + start_time = time.time() + rules_1 = self.generator.get_alert_rules(test_customer_id) + time_1 = time.time() - start_time + print(f" 第1次获取: {len(rules_1)}条规则, 耗时{time_1:.3f}s") + + # 第二次获取(应该从本地缓存获取) + start_time = time.time() + rules_2 = self.generator.get_alert_rules(test_customer_id) + time_2 = time.time() - start_time + print(f" 第2次获取: {len(rules_2)}条规则, 耗时{time_2:.3f}s") + + # 验证规则一致性 + success = len(rules_1) == len(rules_2) and time_2 < time_1 + + if rules_1: + print(f" 规则详情:") + for rule in rules_1[:3]: # 显示前3条 + print(f" - {rule.rule_type}: {rule.physical_sign} [{rule.threshold_min}-{rule.threshold_max}]") + + self.test_results.append(('告警规则获取测试', success)) + return success + + except Exception as e: + print(f" ❌ 告警规则获取测试失败: {e}") + self.test_results.append(('告警规则获取测试', False)) + return False + + def test_alert_generation(self) -> bool: + """测试告警生成""" + print("\n⚡ 测试告警生成") + try: + # 正常数据 + normal_data = { + 'device_sn': 'TEST001', + 'user_id': 1, + 'customer_id': 1, + 'heart_rate': 75, + 'blood_oxygen': 98, + 'temperature': 36.5 + } + + # 异常数据 + abnormal_data = { + 'device_sn': 'TEST002', + 'user_id': 2, + 'customer_id': 1, + 'heart_rate': 130, # 超出阈值 + 'blood_oxygen': 92, # 低于阈值 + 'temperature': 37.8 # 发烧 + } + + test_data = [normal_data, abnormal_data] + + # 生成告警 + start_time = time.time() + alerts = self.generator.generate_alerts(test_data) + processing_time = time.time() - start_time + + print(f" 处理数据: {len(test_data)}条") + print(f" 生成告警: {len(alerts)}个") + print(f" 处理时间: {processing_time:.3f}s") + + # 显示告警详情 + if alerts: + print(f" 告警详情:") + for alert in alerts: + print(f" - {alert.physical_sign}: {alert.current_value} ({alert.threshold_violated}) - {alert.severity_level}") + + success = processing_time < 0.5 # 处理时间应该小于0.5秒 + self.test_results.append(('告警生成测试', success)) + return success + + except Exception as e: + print(f" ❌ 告警生成测试失败: {e}") + self.test_results.append(('告警生成测试', False)) + return False + + def test_performance_batch(self) -> bool: + """测试批量处理性能""" + print("\n🚀 测试批量处理性能") + try: + # 生成大量测试数据 + batch_size = 100 + test_data = [] + + for i in range(batch_size): + data = { + 'device_sn': f'TEST{i:03d}', + 'user_id': i + 1, + 'customer_id': 1, # 使用同一个客户ID以测试缓存效果 + 'heart_rate': 70 + (i % 50), # 70-120之间变化 + 'blood_oxygen': 95 + (i % 5), # 95-100之间变化 + 'temperature': 36.0 + (i % 3) * 0.5 # 36.0-37.0之间变化 + } + test_data.append(data) + + # 批量处理 + start_time = time.time() + alerts = self.generator.generate_alerts(test_data) + processing_time = time.time() - start_time + + # 计算性能指标 + qps = batch_size / processing_time if processing_time > 0 else 0 + avg_time_per_record = processing_time / batch_size if batch_size > 0 else 0 + + print(f" 批量大小: {batch_size}条") + print(f" 生成告警: {len(alerts)}个") + print(f" 总耗时: {processing_time:.3f}s") + print(f" QPS: {qps:.1f} records/sec") + print(f" 平均处理时间: {avg_time_per_record*1000:.2f}ms/record") + + # 性能要求:QPS > 200, 平均处理时间 < 5ms + success = qps > 200 and avg_time_per_record < 0.005 + + self.test_results.append(('批量处理性能测试', success)) + return success + + except Exception as e: + print(f" ❌ 批量处理性能测试失败: {e}") + self.test_results.append(('批量处理性能测试', False)) + return False + + def test_cache_statistics(self) -> bool: + """测试缓存统计""" + print("\n📊 测试缓存统计") + try: + stats = self.generator.get_stats() + + print(f" 统计信息:") + for key, value in stats.items(): + if isinstance(value, dict): + print(f" {key}:") + for sub_key, sub_value in value.items(): + print(f" {sub_key}: {sub_value}") + else: + print(f" {key}: {value}") + + # 检查关键指标 + required_keys = ['total_processed', 'cache_hits', 'db_fallbacks', 'cache_hit_rate'] + success = all(key in stats for key in required_keys) + + self.test_results.append(('缓存统计测试', success)) + return success + + except Exception as e: + print(f" ❌ 缓存统计测试失败: {e}") + self.test_results.append(('缓存统计测试', False)) + return False + + def test_edge_cases(self) -> bool: + """测试边界情况""" + print("\n🔍 测试边界情况") + try: + success_count = 0 + total_tests = 0 + + # 测试1: 空数据 + total_tests += 1 + try: + alerts = self.generator.generate_alerts([]) + if len(alerts) == 0: + success_count += 1 + print(f" ✅ 空数据测试通过") + else: + print(f" ❌ 空数据测试失败") + except Exception as e: + print(f" ❌ 空数据测试异常: {e}") + + # 测试2: 缺少customer_id + total_tests += 1 + try: + test_data = [{'device_sn': 'TEST', 'heart_rate': 80}] # 缺少customer_id + alerts = self.generator.generate_alerts(test_data) + success_count += 1 + print(f" ✅ 缺少customer_id测试通过") + except Exception as e: + print(f" ❌ 缺少customer_id测试异常: {e}") + + # 测试3: 无效customer_id + total_tests += 1 + try: + test_data = [{'device_sn': 'TEST', 'customer_id': 999999, 'heart_rate': 80}] + alerts = self.generator.generate_alerts(test_data) + success_count += 1 + print(f" ✅ 无效customer_id测试通过") + except Exception as e: + print(f" ❌ 无效customer_id测试异常: {e}") + + # 测试4: 清理缓存 + total_tests += 1 + try: + self.generator.clear_cache(1) + success_count += 1 + print(f" ✅ 缓存清理测试通过") + except Exception as e: + print(f" ❌ 缓存清理测试异常: {e}") + + success = success_count == total_tests + print(f" 边界测试: {success_count}/{total_tests} 通过") + + self.test_results.append(('边界情况测试', success)) + return success + + except Exception as e: + print(f" ❌ 边界情况测试失败: {e}") + self.test_results.append(('边界情况测试', False)) + return False + + def run_all_tests(self): + """运行所有测试""" + print("🧪 启动Redis缓存告警系统完整测试") + print("=" * 60) + + if not self.setup(): + print("❌ 测试环境初始化失败,退出测试") + return + + try: + # 运行测试 + tests = [ + self.test_redis_connection, + self.test_alert_rules_retrieval, + self.test_alert_generation, + self.test_performance_batch, + self.test_cache_statistics, + self.test_edge_cases + ] + + for test_func in tests: + try: + test_func() + except Exception as e: + print(f"❌ 测试{test_func.__name__}出现异常: {e}") + self.test_results.append((test_func.__name__, False)) + + # 显示测试结果 + self.show_test_results() + + finally: + self.teardown() + + def show_test_results(self): + """显示测试结果汇总""" + print("\n" + "=" * 60) + print("📋 测试结果汇总") + print("=" * 60) + + passed = 0 + failed = 0 + + for test_name, success in self.test_results: + status = "✅ PASS" if success else "❌ FAIL" + print(f"{status} - {test_name}") + if success: + passed += 1 + else: + failed += 1 + + total = passed + failed + pass_rate = (passed / total * 100) if total > 0 else 0 + + print("-" * 60) + print(f"总计: {total} | 通过: {passed} | 失败: {failed} | 通过率: {pass_rate:.1f}%") + + if pass_rate >= 80: + print("🎉 测试通过!Redis缓存告警系统运行正常") + elif pass_rate >= 60: + print("⚠️ 测试部分通过,建议检查失败项目") + else: + print("❌ 测试失败,系统存在严重问题") + + print("=" * 60) + +def main(): + """主函数""" + test_suite = AlertCacheTestSuite() + test_suite.run_all_tests() + +if __name__ == "__main__": + main() \ No newline at end of file