From 5f90c88c63235b73bc521c19b0086bf6d2c564ac Mon Sep 17 00:00:00 2001 From: gao Date: Mon, 15 Dec 2025 21:50:13 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0K8s=20P0=E7=9B=91?= =?UTF-8?q?=E6=8E=A7=E5=AE=8C=E6=95=B4=E9=83=A8=E7=BD=B2=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 新增K8s配置文件 **Prometheus Operator集成**: - servicemonitor.yml: 自动发现ljwx-boot服务，30秒采集间隔 - prometheusrule.yml: 18条告警规则，覆盖P0-3/P0-4所有场景 - grafana-dashboard-configmap.yml: Dashboard导入配置 **应用配置**: - configmap.yml: 添加P0-3 Redis去重和P0-4缓存失效配置 - application.yml: 更新P0功能配置和Actuator暴露设置 **部署工具**: - deploy-p0.sh: 一键部署脚本，支持dry-run和健康检查 - P0-K8S-DEPLOYMENT.md: 完整K8s部署指南（18KB） ## Docker Compose开发环境 **混合模式配置**: - docker-compose-p0-dev.yml: 基础设施Docker化（MySQL/Redis/Prometheus/Grafana） - prometheus-p0-dev.yml: 使用host.docker.internal访问本地ljwx-boot - .env.p0: 环境变量配置模板 - verify-p0-dev.sh: 开发环境验证脚本 **完整模式配置**: - docker-compose-p0.yml: 包含ljwx-boot在内的完整Docker部署 - prometheus-p0.yml: Docker网络内部服务发现 ## 文档 **快速开始指南**: - K8S-P0-QUICK-START.md: K8s部署5分钟快速指南 - P0-DEV-QUICK-START.md: 本地开发环境快速启动指南 **监控Dashboard**: - grafana-dashboard-p0.json: 修复Dashboard JSON格式问题 ## 核心功能 ✅ **P0-3 Redis去重**: 分布式去重，24小时TTL ✅ **P0-4 缓存失效**: 事件驱动自动失效，1秒延迟 ✅ **完整监控**: 14个Grafana面板，18条告警规则 ✅ **多环境支持**: 本地开发、Docker Compose、K8s生产环境 ## 告警规则（6组18条） 1. 服务可用性 (2条): 服务不可用、高错误率 2. P0-3 Redis去重 (3条): Redis连接失败、内存告警、Key激增 3. P0-4 缓存失效 (3条): 功能禁用、失败率高、响应慢 4. JVM资源 (2条): 内存不足、GC时间长 5. 数据库连接池 (2条): 连接池耗尽、无空闲连接 6. 应用性能 (2条): API响应慢、并发数高 7. Pod资源 (3条): CPU/内存高、频繁重启 ## 部署方式 **K8s生产环境**: ```bash cd ljwx-boot/k8s ./deploy-p0.sh ``` **本地开发环境**: ```bash docker-compose -f docker-compose-p0-dev.yml up -d ./ljwx-boot/run-local.sh ``` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .env.p0 | 73 ++ K8S-P0-QUICK-START.md | 365 ++++++ P0-DEV-QUICK-START.md | 404 +++++++ docker-compose-p0-dev.yml | 189 +++ docker-compose-p0.yml | 272 +++++ ljwx-boot/k8s/P0-K8S-DEPLOYMENT.md | 556 +++++++++ ljwx-boot/k8s/configmap.yml | 27 +- ljwx-boot/k8s/deploy-p0.sh | 253 ++++ ljwx-boot/k8s/grafana-dashboard-configmap.yml | 52 + ljwx-boot/k8s/prometheusrule.yml | 256 ++++ ljwx-boot/k8s/servicemonitor.yml | 40 + .../src/main/resources/application.yml | 7 + .../monitoring/grafana-dashboard-p0.json | 1073 ++++++++++------- ljwx-boot/monitoring/prometheus-p0-dev.yml | 41 + ljwx-boot/monitoring/prometheus-p0.yml | 4 +- scripts/verify-p0-dev.sh | 220 ++++ 16 files changed, 3395 insertions(+), 437 deletions(-) create mode 100644 .env.p0 create mode 100644 K8S-P0-QUICK-START.md create mode 100644 P0-DEV-QUICK-START.md create mode 100644 docker-compose-p0-dev.yml create mode 100644 docker-compose-p0.yml create mode 100644 ljwx-boot/k8s/P0-K8S-DEPLOYMENT.md create mode 100755 ljwx-boot/k8s/deploy-p0.sh create mode 100644 ljwx-boot/k8s/grafana-dashboard-configmap.yml create mode 100644 ljwx-boot/k8s/prometheusrule.yml create mode 100644 ljwx-boot/k8s/servicemonitor.yml create mode 100644 ljwx-boot/monitoring/prometheus-p0-dev.yml create mode 100755 scripts/verify-p0-dev.sh diff --git a/.env.p0 b/.env.p0 new file mode 100644 index 00000000..ed1d5d64 --- /dev/null +++ b/.env.p0 @@ -0,0 +1,73 @@ +# ============================================== +# P0 监控完整环境配置文件 +# 用于 docker-compose-p0.yml +# ============================================== + +# ---------------------------------------------- +# 基础设施端口配置 +# ---------------------------------------------- +MYSQL_PORT=3306 +REDIS_PORT=6379 +REDIS_EXPORTER_PORT=9121 + +# ---------------------------------------------- +# 业务应用端口配置 +# ---------------------------------------------- +LJWX_BOOT_PORT=9998 +LJWX_ADMIN_PORT=5173 + +# ---------------------------------------------- +# 监控系统端口配置 +# ---------------------------------------------- +PROMETHEUS_PORT=9090 +GRAFANA_PORT=3000 + +# ---------------------------------------------- +# MySQL 数据库配置 +# ---------------------------------------------- +MYSQL_ROOT_PASSWORD=123456 +MYSQL_DATABASE=test +MYSQL_USER=root +MYSQL_PASSWORD=123456 + +# ---------------------------------------------- +# Redis 配置 +# ---------------------------------------------- +REDIS_PASSWORD=123456 +REDIS_DATABASE=1 +REDIS_MAXMEMORY=2gb + +# ---------------------------------------------- +# ljwx-boot 应用配置 +# ---------------------------------------------- +SPRING_PROFILES_ACTIVE=prod + +# P0-3 Redis去重配置 +HEALTH_DEDUP_USE_REDIS=true +HEALTH_DEDUP_TTL_HOURS=24 +HEALTH_DEDUP_USE_CONTENT_HASH=true + +# P0-4 缓存失效配置 +HEALTH_CACHE_AUTO_INVALIDATION=true +HEALTH_CACHE_INVALIDATION_DELAY_MS=1000 + +# Prometheus Metrics +MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE=* +MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED=true + +# ---------------------------------------------- +# ljwx-admin 前端配置 +# ---------------------------------------------- +NODE_ENV=production +VITE_USER_NODE_ENV=production + +# ---------------------------------------------- +# Grafana 配置 +# ---------------------------------------------- +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=admin123 + +# ---------------------------------------------- +# 时区配置 +# ---------------------------------------------- +TZ=Asia/Shanghai diff --git a/K8S-P0-QUICK-START.md b/K8S-P0-QUICK-START.md new file mode 100644 index 00000000..4b6773c6 --- /dev/null +++ b/K8S-P0-QUICK-START.md @@ -0,0 +1,365 @@ +# K8s P0监控 - 快速开始指南 + +**目标**: 将ljwx-health业务系统部署到K8s的`ljwx-health` namespace，集成现有monitoring组件 + +**适用场景**: 生产环境、测试环境 + +**完成时间**: 约10-15分钟 + +--- + +## 快速部署 + +### 方式1: 使用自动化脚本（推荐） + +```bash +cd /Users/brunogao/work/codes/ljwx/ljwx-health/ljwx-boot/k8s + +# 1. 预览部署（DRY-RUN） +./deploy-p0.sh --dry-run + +# 2. 实际部署 +./deploy-p0.sh + +# 3. 部署到其他namespace +./deploy-p0.sh --namespace=production +``` + +### 方式2: 手动部署 + +```bash +cd /Users/brunogao/work/codes/ljwx/ljwx-health/ljwx-boot/k8s + +# Step 1: 创建基础资源 +kubectl apply -f secret.yml -n ljwx-health +kubectl apply -f configmap.yml -n ljwx-health +kubectl apply -f service.yml -n ljwx-health + +# Step 2: 部署应用 +kubectl apply -f deployment.yml -n ljwx-health + +# Step 3: 部署监控组件 +kubectl apply -f servicemonitor.yml -n ljwx-health +kubectl apply -f prometheusrule.yml -n ljwx-health +kubectl apply -f grafana-dashboard-configmap.yml -n ljwx-health + +# Step 4: 等待Pod就绪 +kubectl wait --for=condition=ready pod -l app=ljwx-boot -n ljwx-health --timeout=300s +``` + +--- + +## 验证部署 + +### 1. 检查应用状态 + +```bash +# 查看所有资源 +kubectl get all,servicemonitor,prometheusrule -n ljwx-health -l app=ljwx-boot + +# 预期输出: +# pod/ljwx-boot-xxxxxxxxx-xxxxx 1/1 Running 0 2m +# pod/ljwx-boot-xxxxxxxxx-yyyyy 1/1 Running 0 2m +# service/ljwx-boot ClusterIP 10.96.xxx.xxx 9998/TCP +# deployment.apps/ljwx-boot 2/2 2 2 5m +# servicemonitor.../ljwx-boot-p0-monitor 30s +# prometheusrule.../ljwx-boot-p0-alerts 18个告警规则 +``` + +### 2. 验证健康检查 + +```bash +# 获取Pod名称 +POD=$(kubectl get pod -n ljwx-health -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') + +# 执行健康检查 +kubectl exec -n ljwx-health $POD -- curl -sf http://localhost:9998/actuator/health + +# 预期返回: {"status":"UP"} +``` + +### 3. 验证P0 Metrics暴露 + +```bash +# 检查P0指标 +kubectl exec -n ljwx-health $POD -- \ + curl -s http://localhost:9998/actuator/prometheus | \ + grep -E "health_cache|health_dedup" + +# 预期看到: +# health_cache_invalidation_enabled{...} 1.0 +# health_cache_invalidation_total{...} 0.0 +# health_dedup_mode{...} 1.0 +``` + +### 4. 验证Prometheus采集 + +```bash +# 端口转发到Prometheus +kubectl port-forward -n monitoring svc/prometheus 9090:9090 & + +# 访问 http://localhost:9090/targets +# 查找 ljwx-boot target，状态应为 UP + +# 查询P0指标 +# 在Prometheus UI执行: health_cache_invalidation_enabled +``` + +--- + +## Grafana Dashboard配置 + +### 快速导入Dashboard + +```bash +# 1. 端口转发到Grafana +kubectl port-forward -n monitoring svc/grafana 3000:3000 & + +# 2. 访问 http://localhost:3000 +# 登录: admin / + +# 3. 导入Dashboard +# - Dashboards → + Import +# - Upload JSON file +# - 选择: ljwx-boot/monitoring/grafana-dashboard-p0.json +# - 数据源: Prometheus +# - 点击 Import + +# 4. 查看Dashboard +# 应该看到14个面板显示数据: +# ✓ 缓存失效监控 (6个面板) +# ✓ Redis去重监控 (4个面板) +# ✓ JVM和数据库监控 (3个面板) +# ✓ 系统告警 (1个面板) +``` + +--- + +## 文件清单 + +| 文件名 | 大小 | 用途 | +|--------|------|------| +| **deployment.yml** | 5KB | 应用Deployment配置（2副本，滚动更新）| +| **service.yml** | 1KB | Service配置（ClusterIP + Headless）| +| **configmap.yml** | 2KB | 应用配置（含P0-3/P0-4配置）| +| **secret.yml.template** | 1KB | Secret模板（数据库/Redis密码）| +| **servicemonitor.yml** | 1KB | Prometheus ServiceMonitor配置 | +| **prometheusrule.yml** | 13KB | Prometheus告警规则（18条规则）| +| **grafana-dashboard-configmap.yml** | 1KB | Grafana Dashboard导入说明 | +| **ingress.yml** | 2KB | Ingress配置（可选）| +| **deploy-p0.sh** | 7KB | 一键部署脚本 | +| **P0-K8S-DEPLOYMENT.md** | 40KB | 完整部署文档 | + +--- + +## 关键配置说明 + +### P0-3 Redis去重配置 + +在`configmap.yml`中配置: +```yaml +health: + dedup: + use-redis: true # 使用Redis去重 + ttl-hours: 24 # Key TTL 24小时 + use-content-hash: true # 启用内容哈希 +``` + +### P0-4 缓存失效配置 + +在`configmap.yml`中配置: +```yaml +health: + cache: + auto-invalidation: true # 启用自动失效 + invalidation-delay-ms: 1000 # 延迟1秒 +``` + +### Prometheus配置 + +`servicemonitor.yml`中的关键配置: +```yaml +spec: + endpoints: + - port: http + path: /actuator/prometheus + interval: 30s # 30秒采集一次 + scrapeTimeout: 10s # 10秒超时 +``` + +--- + +## 告警规则 + +已配置18条告警规则，分为6个组: + +1. **服务可用性** (2条): 服务不可用、高错误率 +2. **P0-3 Redis去重** (3条): Redis连接失败、内存告警、Key激增 +3. **P0-4 缓存失效** (3条): 功能禁用、失败率高、响应慢 +4. **JVM资源** (2条): 内存不足、GC时间长 +5. **数据库连接池** (2条): 连接池耗尽、无空闲连接 +6. **应用性能** (2条): API响应慢、并发数高 +7. **Pod资源** (3条): CPU/内存高、频繁重启 + +--- + +## 常见问题 + +### Q1: Pod启动失败 (CrashLoopBackOff) + +**解决方案**: +```bash +# 查看日志 +kubectl logs -n ljwx-health -l app=ljwx-boot --tail=100 + +# 检查ConfigMap配置 +kubectl get configmap ljwx-boot-config -n ljwx-health -o yaml + +# 验证依赖服务 +kubectl get svc -n ljwx-health | grep -E "mysql|redis" +``` + +### Q2: Prometheus无法采集Metrics + +**解决方案**: +```bash +# 检查ServiceMonitor +kubectl describe servicemonitor ljwx-boot-p0-monitor -n ljwx-health + +# 检查Prometheus Targets +kubectl port-forward -n monitoring svc/prometheus 9090:9090 +# 访问: http://localhost:9090/targets + +# 检查metrics端点 +kubectl exec -n ljwx-health $POD -- curl http://localhost:9998/actuator/prometheus +``` + +### Q3: Dashboard显示"No Data" + +**解决方案**: +```bash +# 1. 确认Prometheus在采集数据 +# Prometheus UI执行查询: health_cache_invalidation_total + +# 2. 检查Grafana数据源 +# Grafana → Configuration → Data Sources → Prometheus + +# 3. 验证P0代码激活 +kubectl exec -n ljwx-health $POD -- \ + curl -s http://localhost:9998/actuator/prometheus | grep health_cache +``` + +--- + +## 运维命令 + +### 查看日志 + +```bash +# 所有Pod日志 +kubectl logs -f -n ljwx-health -l app=ljwx-boot + +# 特定Pod日志 +kubectl logs -f -n ljwx-health ljwx-boot-xxx-yyy +``` + +### 扩缩容 + +```bash +# 扩容到3副本 +kubectl scale deployment ljwx-boot -n ljwx-health --replicas=3 + +# 验证 +kubectl get pods -n ljwx-health -l app=ljwx-boot +``` + +### 滚动更新 + +```bash +# 更新镜像 +kubectl set image deployment/ljwx-boot \ + ljwx-boot=harbor.omniverseai.net/ljwx-health/ljwx-boot:v2.0 \ + -n ljwx-health + +# 查看更新状态 +kubectl rollout status deployment/ljwx-boot -n ljwx-health + +# 回滚 +kubectl rollout undo deployment/ljwx-boot -n ljwx-health +``` + +### 更新配置 + +```bash +# 编辑ConfigMap +kubectl edit configmap ljwx-boot-config -n ljwx-health + +# 重启Pod使配置生效 +kubectl rollout restart deployment ljwx-boot -n ljwx-health +``` + +### 删除部署 + +```bash +# 只删除监控组件 +kubectl delete servicemonitor,prometheusrule -n ljwx-health -l app=ljwx-boot + +# 删除整个应用 +kubectl delete deployment,service,configmap,secret -n ljwx-health -l app=ljwx-boot +``` + +--- + +## 监控指标速查 + +### P0核心指标 + +| 指标 | 说明 | 告警阈值 | +|------|------|---------| +| `health_dedup_mode` | Redis去重模式（1=Redis, 0=内存）| = 0 (critical) | +| `health_cache_invalidation_enabled` | 缓存失效启用状态 | = 0 (warning) | +| `redis_db_keys{db="database1"}` | Redis去重Key数量 | 增速>1000/min (warning) | +| `health_cache_invalidation_total` | 缓存失效次数 | 失败率>5% (warning) | + +### 性能指标 + +| 指标 | 说明 | SLA目标 | +|------|------|---------| +| P50响应时间 | 50%请求响应时间 | < 50ms | +| P95响应时间 | 95%请求响应时间 | < 100ms | +| P99响应时间 | 99%请求响应时间 | < 200ms | +| QPS | 每秒请求数 | ≥ 1000 | +| 成功率 | 请求成功率 | ≥ 99% | + +--- + +## 相关文档 + +- **[P0-K8S-DEPLOYMENT.md](ljwx-boot/k8s/P0-K8S-DEPLOYMENT.md)** - 完整K8s部署指南（40KB） +- **[P0-DEV-QUICK-START.md](P0-DEV-QUICK-START.md)** - 本地开发环境快速启动 +- **[P0-3实施报告](ljwx-boot/docs/architecture/P0-3_REDIS_DEDUP_IMPLEMENTATION.md)** - Redis去重详细设计 +- **[P0-4实施报告](ljwx-boot/docs/architecture/P0-4_CACHE_INVALIDATION_IMPLEMENTATION.md)** - 缓存失效详细设计 + +--- + +## 架构对比 + +### Docker Compose环境 vs K8s环境 + +| 维度 | Docker Compose | Kubernetes | +|------|---------------|------------| +| **用途** | 本地开发、快速验证 | 生产环境、高可用 | +| **服务发现** | host.docker.internal | Service DNS | +| **高可用** | 单实例 | 多副本（2+）| +| **自动扩缩容** | ❌ | ✅ HPA | +| **滚动更新** | ❌ | ✅ RollingUpdate | +| **监控集成** | 手动配置 | ServiceMonitor自动发现 | +| **配置管理** | .env文件 | ConfigMap + Secret | +| **日志收集** | docker logs | EFK/Loki | + +--- + +**最后更新**: 2025-12-15 +**文档版本**: v1.0 + +**下一步**: 运行 `./ljwx-boot/k8s/deploy-p0.sh` 开始部署！ diff --git a/P0-DEV-QUICK-START.md b/P0-DEV-QUICK-START.md new file mode 100644 index 00000000..e3e6e940 --- /dev/null +++ b/P0-DEV-QUICK-START.md @@ -0,0 +1,404 @@ +# P0 监控体系开发环境启动指南 + +## 环境架构 + +**混合模式**: 基础设施在Docker中运行，业务应用在本地运行 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 开发环境架构 │ +├─────────────────────────────────────────────────────────────┤ +│ │ +│ 本地运行 (方便调试和快速迭代): │ +│ ├─ ljwx-boot (Spring Boot) :9998 │ +│ └─ ljwx-admin (Vue3) :3333 或 :5173 │ +│ │ +│ Docker运行 (稳定基础设施): │ +│ ├─ MySQL :3306 │ +│ ├─ Redis :6379 │ +│ ├─ Redis Exporter :9121 │ +│ ├─ Prometheus :9090 │ +│ └─ Grafana :3000 │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## 优势 + +- ✅ **快速开发**: 代码修改后无需重新构建Docker镜像 +- ✅ **方便调试**: 可以直接在IDE中断点调试 +- ✅ **快速启动**: Docker只需启动基础设施，启动时间< 1分钟 +- ✅ **稳定环境**: 数据库和监控在Docker中保持稳定配置 +- ✅ **资源友好**: 不需要构建大型应用镜像 + +## 快速启动 + +### 1. 启动Docker基础设施（首次或重启后） + +```bash +# 在项目根目录 +cd /Users/brunogao/work/codes/ljwx/ljwx-health + +# 启动所有基础设施服务 +docker-compose -f docker-compose-p0-dev.yml up -d + +# 查看服务状态 +docker-compose -f docker-compose-p0-dev.yml ps + +# 查看日志（可选） +docker-compose -f docker-compose-p0-dev.yml logs -f +``` + +**预期输出**: +``` +✔ Container ljwx-mysql Started +✔ Container ljwx-redis Started +✔ Container redis-exporter Started +✔ Container prometheus Started +✔ Container grafana Started +``` + +### 2. 启动后端应用 ljwx-boot + +```bash +# 进入后端目录 +cd ljwx-boot + +# 启动后端服务（会自动编译并运行） +./run-local.sh + +# 检查启动状态 +./run-local.sh status + +# 查看日志 +tail -f logs/ljwx-boot.log +``` + +**验证启动成功**: +```bash +curl http://localhost:9998/actuator/health +# 应该返回: {"status":"UP"} +``` + +### 3. 启动前端应用 ljwx-admin + +```bash +# 进入前端目录 +cd ljwx-admin + +# 首次启动需要安装依赖 +pnpm install + +# 启动开发服务器 +pnpm dev +``` + +**访问地址**: http://localhost:3333 或 http://localhost:5173 + +### 4. 验证完整环境 + +```bash +# 在项目根目录运行验证脚本 +./scripts/verify-p0-dev.sh +``` + +验证脚本会检查： +- Docker基础设施服务状态 +- 本地ljwx-boot运行状态 +- Redis连接和去重Key +- Prometheus指标采集 +- Grafana运行状态 +- 可选：上传测试数据 + +## 服务访问地址 + +### 业务应用 (本地运行) + +| 服务 | 地址 | 说明 | +|------|------|------| +| ljwx-boot API | http://localhost:9998 | 后端API服务 | +| API文档 | http://localhost:9998/doc.html | Swagger/Knife4j文档 | +| 健康检查 | http://localhost:9998/actuator/health | Spring Boot健康检查 | +| Prometheus Metrics | http://localhost:9998/actuator/prometheus | 暴露的指标 | +| ljwx-admin | http://localhost:3333 | 前端管理界面 | + +### 监控系统 (Docker) + +| 服务 | 地址 | 凭据 | +|------|------|------| +| Grafana | http://localhost:3000 | admin/admin123 | +| Prometheus | http://localhost:9090 | - | +| Targets | http://localhost:9090/targets | 查看采集目标 | +| Alerts | http://localhost:9090/alerts | 查看告警规则 | + +### 基础设施 (Docker) + +| 服务 | 地址 | 凭据 | +|------|------|------| +| MySQL | localhost:3306 | root/123456, database: test | +| Redis | localhost:6379 | password: 123456, db: 1 | +| Redis Exporter | http://localhost:9121/metrics | - | + +**注意**: 如果端口被占用，Docker会自动分配其他端口（如3307、9091等），请使用`docker ps`查看实际端口。 + +## 配置Grafana Dashboard + +### 首次配置 + +1. 访问 Grafana: http://localhost:3000 +2. 登录: admin / admin123 +3. 导入Dashboard: + - 点击左侧菜单 "Dashboards" → "Import" + - 点击 "Upload JSON file" + - 选择文件: `ljwx-boot/monitoring/grafana-dashboard-p0.json` + - 选择数据源: Prometheus + - 点击 "Import" + +### Dashboard功能 + +导入后可以看到14个监控面板： + +**系统概览**: +- 请求总量、成功率、错误率 +- 平均响应时间、P95/P99响应时间 +- QPS（每秒请求数） + +**缓存失效监控** (P0-4): +- 缓存失效总数、成功/失败数 +- 失效延迟统计 +- 失效类型分布 + +**Redis性能** (P0-3): +- Redis连接数、命中率 +- 内存使用情况 +- Key数量趋势 + +**告警状态**: +- 活跃告警数量 +- 告警历史 + +## 性能压测 + +### 运行压测脚本 + +```bash +# 基础压测（100并发，60秒） +./ljwx-boot/scripts/performance-test.sh + +# 自定义参数 +./ljwx-boot/scripts/performance-test.sh --users=200 --duration=120 +``` + +### SLA目标 + +| 指标 | 目标值 | 说明 | +|------|-------|------| +| P50响应时间 | < 50ms | 50%请求响应时间 | +| P95响应时间 | < 100ms | 95%请求响应时间 | +| P99响应时间 | < 200ms | 99%请求响应时间 | +| QPS | ≥ 100 | 每秒请求数 | +| 成功率 | ≥ 99% | 请求成功率 | + +## 常用操作命令 + +### Docker服务管理 + +```bash +# 启动所有服务 +docker-compose -f docker-compose-p0-dev.yml up -d + +# 停止所有服务 +docker-compose -f docker-compose-p0-dev.yml down + +# 重启某个服务 +docker-compose -f docker-compose-p0-dev.yml restart prometheus + +# 查看服务日志 +docker-compose -f docker-compose-p0-dev.yml logs -f grafana + +# 查看服务状态 +docker-compose -f docker-compose-p0-dev.yml ps + +# 进入容器 +docker exec -it ljwx-mysql bash +docker exec -it ljwx-redis redis-cli -a 123456 +``` + +### ljwx-boot管理 + +```bash +cd ljwx-boot + +# 启动 +./run-local.sh + +# 停止 +./run-local.sh stop + +# 重启 +./run-local.sh restart + +# 查看状态 +./run-local.sh status + +# 查看日志 +tail -f logs/ljwx-boot.log +``` + +### Redis监控 + +```bash +# 查看Redis去重Key +docker exec ljwx-redis redis-cli -a 123456 KEYS "health:dedup:*" + +# 查看Key数量 +docker exec ljwx-redis redis-cli -a 123456 DBSIZE + +# 查看Key详情 +docker exec ljwx-redis redis-cli -a 123456 TTL "health:dedup:某个Key" + +# 监控Redis实时命令 +docker exec ljwx-redis redis-cli -a 123456 MONITOR + +# 使用监控脚本 +REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD=123456 \ + ./ljwx-boot/scripts/monitor-redis-keys.sh +``` + +## 故障排查 + +### ljwx-boot启动失败 + +**问题**: `./run-local.sh` 启动失败 + +**排查步骤**: +1. 检查MySQL是否启动: `docker ps | grep ljwx-mysql` +2. 检查Redis是否启动: `docker ps | grep ljwx-redis` +3. 检查端口是否被占用: `lsof -i :9998` +4. 查看启动日志: `cat logs/ljwx-boot.log` + +**常见原因**: +- MySQL/Redis未启动或健康检查未通过 +- 9998端口被占用 +- 配置文件错误（检查`application-local.yml`） + +### Prometheus无法采集ljwx-boot指标 + +**问题**: Prometheus Targets显示ljwx-boot为DOWN + +**排查步骤**: +1. 确认ljwx-boot在本地运行: `curl http://localhost:9998/actuator/health` +2. 确认metrics端点可访问: `curl http://localhost:9998/actuator/prometheus` +3. 检查Prometheus配置: `docker exec prometheus cat /etc/prometheus/prometheus.yml` +4. 查看Prometheus日志: `docker logs prometheus` + +**解决方案**: +- 确保ljwx-boot的`application-local.yml`中开启了metrics暴露: + ```yaml + management: + endpoints: + web: + exposure: + include: "*" + metrics: + export: + prometheus: + enabled: true + ``` + +### Grafana Dashboard无数据 + +**问题**: Dashboard显示"No data" + +**排查步骤**: +1. 检查Prometheus是否有数据: 访问 http://localhost:9090/graph +2. 执行查询测试: `health_cache_invalidation_total` +3. 检查数据源配置: Grafana → Configuration → Data Sources → Prometheus +4. 确认URL为: `http://prometheus:9090` + +### Redis去重不生效 + +**问题**: 重复数据未被拦截 + +**检查配置**: +```bash +# 在ljwx-boot的application-local.yml中确认: +health: + dedup: + use-redis: true # 必须为true + ttl-hours: 24 # TTL时间 + use-content-hash: true # 使用内容哈希 +``` + +**验证**: +```bash +# 查看Redis中的去重Key +docker exec ljwx-redis redis-cli -a 123456 KEYS "health:dedup:*" + +# 应该看到类似: health:dedup:1000:TEST-100:xxxxx 的Key +``` + +## 环境变量配置 + +如需修改端口或其他配置，编辑 `.env` 文件（复制自`.env.p0`）: + +```bash +# 复制环境变量模板 +cp .env.p0 .env + +# 编辑配置 +vim .env +``` + +主要配置项: +```properties +# 基础设施端口 +MYSQL_PORT=3306 +REDIS_PORT=6379 +REDIS_EXPORTER_PORT=9121 + +# 监控系统端口 +PROMETHEUS_PORT=9090 +GRAFANA_PORT=3000 + +# MySQL配置 +MYSQL_ROOT_PASSWORD=123456 +MYSQL_DATABASE=test + +# Redis配置 +REDIS_PASSWORD=123456 +REDIS_DATABASE=1 +REDIS_MAXMEMORY=2gb + +# P0-3 Redis去重配置 +HEALTH_DEDUP_USE_REDIS=true +HEALTH_DEDUP_TTL_HOURS=24 + +# P0-4 缓存失效配置 +HEALTH_CACHE_AUTO_INVALIDATION=true +HEALTH_CACHE_INVALIDATION_DELAY_MS=1000 +``` + +修改后需要重启Docker服务: +```bash +docker-compose -f docker-compose-p0-dev.yml down +docker-compose -f docker-compose-p0-dev.yml up -d +``` + +## 下一步 + +1. ✅ 基础设施已启动 +2. ✅ 业务应用已运行 +3. ✅ 监控系统已配置 +4. 📝 开始开发和调试 +5. 🧪 运行性能压测验证SLA +6. 📊 在Grafana中观察实时指标 + +## 相关文档 + +- [完整docker-compose部署方案](./docker-compose-p0.yml) - 生产环境用 +- [开发环境docker-compose](./docker-compose-p0-dev.yml) - 当前使用 +- [P0-3 Redis去重设计](./ljwx-boot/docs/P0-3-Redis去重.md) +- [P0-4 缓存失效监控](./ljwx-boot/docs/P0-4-缓存失效监控.md) +- [Grafana Dashboard JSON](./ljwx-boot/monitoring/grafana-dashboard-p0.json) +- [Prometheus告警规则](./ljwx-boot/monitoring/prometheus-alerts-p0.yml) diff --git a/docker-compose-p0-dev.yml b/docker-compose-p0-dev.yml new file mode 100644 index 00000000..3fa72ada --- /dev/null +++ b/docker-compose-p0-dev.yml @@ -0,0 +1,189 @@ +# ============================================================================ +# P0 开发环境 Docker Compose 配置 +# 用途: 启动基础设施和监控组件，业务应用（ljwx-boot, ljwx-admin）在本地运行 +# 优势: 代码修改无需重新构建镜像，启动快速，方便调试 +# ============================================================================ +version: '3.8' + +networks: + ljwx-network: + driver: bridge + +volumes: + mysql_data: + redis_data: + prometheus_data: + grafana_data: + +services: + # ============================================================================ + # 基础设施服务 + # ============================================================================ + + # MySQL 数据库 + ljwx-mysql: + image: mysql:8.0 + container_name: ljwx-mysql + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-123456} + - MYSQL_DATABASE=${MYSQL_DATABASE:-test} + - TZ=${TZ:-Asia/Shanghai} + volumes: + - mysql_data:/var/lib/mysql + - ./sql:/docker-entrypoint-initdb.d:ro + ports: + - "${MYSQL_PORT:-3306}:3306" + networks: + - ljwx-network + restart: unless-stopped + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p${MYSQL_ROOT_PASSWORD:-123456}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + command: --default-authentication-plugin=mysql_native_password --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + + # Redis 缓存 (用于去重和缓存) + ljwx-redis: + image: redis:7-alpine + container_name: ljwx-redis + command: > + redis-server + --requirepass ${REDIS_PASSWORD:-123456} + --maxmemory ${REDIS_MAXMEMORY:-2gb} + --maxmemory-policy allkeys-lru + --appendonly yes + volumes: + - redis_data:/data + ports: + - "${REDIS_PORT:-6379}:6379" + networks: + - ljwx-network + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD:-123456}", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + # Redis Exporter (导出Redis指标到Prometheus) + redis-exporter: + image: oliver006/redis_exporter:latest + container_name: redis-exporter + command: + - '--redis.addr=ljwx-redis:6379' + - '--redis.password=${REDIS_PASSWORD:-123456}' + ports: + - "${REDIS_EXPORTER_PORT:-9121}:9121" + networks: + - ljwx-network + depends_on: + ljwx-redis: + condition: service_healthy + restart: unless-stopped + + # ============================================================================ + # 监控系统服务 + # ============================================================================ + + # Prometheus 指标收集和存储 + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./ljwx-boot/monitoring/prometheus-p0-dev.yml:/etc/prometheus/prometheus.yml:ro + - ./ljwx-boot/monitoring/prometheus-alerts-p0.yml:/etc/prometheus/alerts/p0-alerts.yml:ro + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-lifecycle' + - '--storage.tsdb.retention.time=30d' + ports: + - "${PROMETHEUS_PORT:-9090}:9090" + networks: + - ljwx-network + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + redis-exporter: + condition: service_started + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 10s + retries: 3 + + # Grafana 可视化面板 + grafana: + image: grafana/grafana:latest + container_name: grafana + volumes: + - ./ljwx-boot/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./ljwx-boot/monitoring:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin123} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://localhost:${GRAFANA_PORT:-3000} + - GF_INSTALL_PLUGINS=grafana-piechart-panel + - TZ=${TZ:-Asia/Shanghai} + ports: + - "${GRAFANA_PORT:-3000}:3000" + networks: + - ljwx-network + depends_on: + prometheus: + condition: service_healthy + restart: unless-stopped + +# ============================================================================ +# 使用说明 +# ============================================================================ +# +# 1. 启动基础设施和监控: +# docker-compose -f docker-compose-p0-dev.yml up -d +# +# 2. 启动本地业务应用: +# # 启动后端 (在 ljwx-boot 目录) +# cd ljwx-boot && ./run-local.sh +# +# # 启动前端 (在 ljwx-admin 目录) +# cd ljwx-admin && pnpm dev +# +# 3. 查看服务状态: +# docker-compose -f docker-compose-p0-dev.yml ps +# +# 4. 查看服务日志: +# docker-compose -f docker-compose-p0-dev.yml logs -f [service-name] +# +# 5. 停止所有服务: +# docker-compose -f docker-compose-p0-dev.yml down +# +# ============================================================================ +# 服务访问地址 (开发环境) +# ============================================================================ +# +# 业务应用 (本地运行): +# - ljwx-boot 后端API: http://localhost:9998 +# - ljwx-boot API文档: http://localhost:9998/doc.html +# - ljwx-boot 健康检查: http://localhost:9998/actuator/health +# - ljwx-boot Metrics: http://localhost:9998/actuator/prometheus +# - ljwx-admin 前端: http://localhost:3333 (或 5173) +# +# 监控系统 (Docker): +# - Grafana Dashboard: http://localhost:3000 (admin/admin123) +# - Prometheus: http://localhost:9090 +# - Prometheus Targets: http://localhost:9090/targets +# - Prometheus Alerts: http://localhost:9090/alerts +# +# 基础设施 (Docker): +# - MySQL: localhost:3306 (root/123456, database: test) +# - Redis: localhost:6379 (password: 123456, db: 1) +# - Redis Exporter: http://localhost:9121/metrics +# diff --git a/docker-compose-p0.yml b/docker-compose-p0.yml new file mode 100644 index 00000000..34a6ee99 --- /dev/null +++ b/docker-compose-p0.yml @@ -0,0 +1,272 @@ +# ============================================================================ +# P0 监控完整环境 Docker Compose 配置 +# 包含业务应用(ljwx-boot, ljwx-admin)和完整监控栈(Prometheus, Grafana) +# ============================================================================ +version: '3.8' + +networks: + ljwx-network: + driver: bridge + +volumes: + mysql_data: + redis_data: + prometheus_data: + grafana_data: + +services: + # ============================================================================ + # 基础设施服务 + # ============================================================================ + + # MySQL 数据库 + ljwx-mysql: + image: mysql:8.0 + container_name: ljwx-mysql + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${MYSQL_DATABASE} + - TZ=${TZ} + volumes: + - mysql_data:/var/lib/mysql + - ./sql:/docker-entrypoint-initdb.d:ro + ports: + - "${MYSQL_PORT}:3306" + networks: + - ljwx-network + restart: unless-stopped + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p${MYSQL_ROOT_PASSWORD}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + command: --default-authentication-plugin=mysql_native_password --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + + # Redis 缓存 (用于去重和缓存) + ljwx-redis: + image: redis:7-alpine + container_name: ljwx-redis + command: > + redis-server + --requirepass ${REDIS_PASSWORD} + --maxmemory ${REDIS_MAXMEMORY} + --maxmemory-policy allkeys-lru + --appendonly yes + volumes: + - redis_data:/data + ports: + - "${REDIS_PORT}:6379" + networks: + - ljwx-network + restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + # Redis Exporter (导出Redis指标到Prometheus) + redis-exporter: + image: oliver006/redis_exporter:latest + container_name: redis-exporter + command: + - '--redis.addr=ljwx-redis:6379' + - '--redis.password=${REDIS_PASSWORD}' + ports: + - "${REDIS_EXPORTER_PORT}:9121" + networks: + - ljwx-network + depends_on: + ljwx-redis: + condition: service_healthy + restart: unless-stopped + + # ============================================================================ + # 业务应用服务 + # ============================================================================ + + # ljwx-boot 后端服务 + ljwx-boot: + build: + context: ./ljwx-boot + dockerfile: ljwx-boot-admin/Dockerfile + image: ljwx/boot:p0-latest + container_name: ljwx-boot + environment: + # Spring Boot 配置 + - SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE} + - TZ=${TZ} + + # MySQL 配置 + - MYSQL_HOST=ljwx-mysql + - MYSQL_PORT=3306 + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + + # Redis 配置 + - REDIS_HOST=ljwx-redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_DATABASE=${REDIS_DATABASE} + + # P0-3 Redis去重配置 + - HEALTH_DEDUP_USE_REDIS=${HEALTH_DEDUP_USE_REDIS} + - HEALTH_DEDUP_TTL_HOURS=${HEALTH_DEDUP_TTL_HOURS} + - HEALTH_DEDUP_USE_CONTENT_HASH=${HEALTH_DEDUP_USE_CONTENT_HASH} + + # P0-4 缓存失效配置 + - HEALTH_CACHE_AUTO_INVALIDATION=${HEALTH_CACHE_AUTO_INVALIDATION} + - HEALTH_CACHE_INVALIDATION_DELAY_MS=${HEALTH_CACHE_INVALIDATION_DELAY_MS} + + # Prometheus Metrics 配置 + - MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE=${MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE} + - MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED=${MANAGEMENT_METRICS_EXPORT_PROMETHEUS_ENABLED} + ports: + - "${LJWX_BOOT_PORT}:9998" + depends_on: + ljwx-mysql: + condition: service_healthy + ljwx-redis: + condition: service_healthy + networks: + - ljwx-network + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:9998/actuator/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s + + # ljwx-admin 前端管理界面 + ljwx-admin: + build: + context: ./ljwx-admin + dockerfile: Dockerfile + args: + - VITE_API_BASE_URL=http://ljwx-boot:9998 + image: ljwx/admin:p0-latest + container_name: ljwx-admin + environment: + - NODE_ENV=${NODE_ENV} + - VITE_USER_NODE_ENV=${VITE_USER_NODE_ENV} + - TZ=${TZ} + ports: + - "${LJWX_ADMIN_PORT}:5173" + depends_on: + ljwx-boot: + condition: service_healthy + networks: + - ljwx-network + restart: unless-stopped + + # ============================================================================ + # 监控系统服务 + # ============================================================================ + + # Prometheus 指标收集和存储 + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./ljwx-boot/monitoring/prometheus-p0.yml:/etc/prometheus/prometheus.yml:ro + - ./ljwx-boot/monitoring/prometheus-alerts-p0.yml:/etc/prometheus/alerts/p0-alerts.yml:ro + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-lifecycle' + - '--storage.tsdb.retention.time=30d' + ports: + - "${PROMETHEUS_PORT}:9090" + networks: + - ljwx-network + depends_on: + ljwx-boot: + condition: service_healthy + redis-exporter: + condition: service_started + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 10s + retries: 3 + + # Grafana 可视化面板 + grafana: + image: grafana/grafana:latest + container_name: grafana + volumes: + - ./ljwx-boot/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./ljwx-boot/monitoring:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://localhost:${GRAFANA_PORT} + - GF_INSTALL_PLUGINS=grafana-piechart-panel + - TZ=${TZ} + ports: + - "${GRAFANA_PORT}:3000" + networks: + - ljwx-network + depends_on: + prometheus: + condition: service_healthy + restart: unless-stopped + +# ============================================================================ +# 使用说明 +# ============================================================================ +# +# 1. 配置环境变量: +# cp .env.p0 .env +# # 根据需要修改 .env 中的端口配置 +# +# 2. 构建并启动所有服务: +# docker-compose -f docker-compose-p0.yml up -d --build +# +# 3. 查看服务状态: +# docker-compose -f docker-compose-p0.yml ps +# +# 4. 查看服务日志: +# docker-compose -f docker-compose-p0.yml logs -f [service-name] +# +# 5. 快速验证: +# # 等待所有服务启动完成 (约2-3分钟) +# docker-compose -f docker-compose-p0.yml exec ljwx-boot wget -q -O- http://localhost:9998/actuator/health +# +# 6. 停止所有服务: +# docker-compose -f docker-compose-p0.yml down +# +# 7. 停止并删除数据卷 (⚠️ 会删除所有数据): +# docker-compose -f docker-compose-p0.yml down -v +# +# ============================================================================ +# 服务访问地址 (默认端口) +# ============================================================================ +# +# 业务应用: +# - ljwx-boot 后端API: http://localhost:9998 +# - ljwx-boot API文档: http://localhost:9998/doc.html +# - ljwx-boot 健康检查: http://localhost:9998/actuator/health +# - ljwx-boot Metrics: http://localhost:9998/actuator/prometheus +# - ljwx-admin 前端: http://localhost:5173 +# +# 监控系统: +# - Grafana Dashboard: http://localhost:3000 (admin/admin123) +# - Prometheus: http://localhost:9090 +# - Prometheus Targets: http://localhost:9090/targets +# - Prometheus Alerts: http://localhost:9090/alerts +# +# 基础设施: +# - MySQL: localhost:3306 (root/123456, database: test) +# - Redis: localhost:6379 (password: 123456, db: 1) +# - Redis Exporter: http://localhost:9121/metrics +# diff --git a/ljwx-boot/k8s/P0-K8S-DEPLOYMENT.md b/ljwx-boot/k8s/P0-K8S-DEPLOYMENT.md new file mode 100644 index 00000000..e8b1d038 --- /dev/null +++ b/ljwx-boot/k8s/P0-K8S-DEPLOYMENT.md @@ -0,0 +1,556 @@ +# P0监控 - Kubernetes部署指南 + +**部署目标**: 将ljwx-health业务系统部署到K8s的`ljwx-health` namespace，集成现有的K8s monitoring组件 + +**适用环境**: Kubernetes集群（已安装Prometheus Operator + Grafana） + +**版本**: v1.0 - 2025-12-15 + +--- + +## 目录 + +- [1. 前提条件](#1-前提条件) +- [2. 架构说明](#2-架构说明) +- [3. 部署步骤](#3-部署步骤) +- [4. 验证部署](#4-验证部署) +- [5. Grafana Dashboard配置](#5-grafana-dashboard配置) +- [6. 故障排查](#6-故障排查) +- [7. 运维操作](#7-运维操作) + +--- + +## 1. 前提条件 + +### 1.1 K8s集群要求 + +- **Kubernetes版本**: >= 1.21 +- **Prometheus Operator**: 已安装（用于ServiceMonitor和PrometheusRule） +- **Grafana**: 已安装（用于Dashboard可视化） + +**验证方法**: +```bash +# 检查Prometheus Operator CRDs +kubectl get crd | grep monitoring.coreos.com + +# 应该看到: +# prometheuses.monitoring.coreos.com +# servicemonitors.monitoring.coreos.com +# prometheusrules.monitoring.coreos.com +``` + +### 1.2 命名空间准备 + +```bash +# 创建ljwx-health namespace（如果不存在） +kubectl create namespace ljwx-health + +# 验证namespace +kubectl get namespace ljwx-health +``` + +### 1.3 镜像准备 + +确保ljwx-boot镜像已推送到Harbor仓库: +```bash +# 镜像地址示例 +harbor.omniverseai.net/ljwx-health/ljwx-boot:latest +``` + +### 1.4 依赖服务 + +确保以下服务在`ljwx-health` namespace中可用: +- **MySQL**: `mysql-service.ljwx-health.svc.cluster.local:3306` +- **Redis**: `redis-service.ljwx-health.svc.cluster.local:6379` + +**验证方法**: +```bash +kubectl get svc -n ljwx-health | grep -E "mysql|redis" +``` + +--- + +## 2. 架构说明 + +### 2.1 整体架构 + +``` +┌────────────────────────────────────────────────────────────────┐ +│ K8s ljwx-health Namespace │ +├────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ ljwx-boot │─────────│ MySQL │ │ +│ │ (2 Pods) │ │ Service │ │ +│ └──────┬───────┘ └──────────────┘ │ +│ │ │ +│ │ ┌──────────────┐ │ +│ └─────────────────│ Redis │ │ +│ │ Service │ │ +│ └──────────────┘ │ +│ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Monitoring Resources (ServiceMonitor + PrometheusRule)│ │ +│ └────────────────────────────────────────────────────────┘ │ +│ │ │ +└───────────────────────────┼────────────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────────┐ +│ K8s Monitoring Namespace │ +├────────────────────────────────────────────────────────────────┤ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ Prometheus │◀────────│ Grafana │ │ +│ │ (Operator) │ │ (Dashboard) │ │ +│ └──────────────┘ └──────────────┘ │ +└────────────────────────────────────────────────────────────────┘ +``` + +### 2.2 监控组件说明 + +| 组件 | 类型 | 用途 | +|------|------|------| +| **ServiceMonitor** | Prometheus CRD | 自动发现ljwx-boot Pod并采集metrics | +| **PrometheusRule** | Prometheus CRD | 定义P0告警规则（18条规则，6个分组）| +| **ConfigMap** | K8s ConfigMap | 存储P0配置（去重、缓存失效）| +| **Service** | K8s Service | 暴露ljwx-boot服务供Prometheus访问 | + +### 2.3 P0功能支持 + +- ✅ **P0-3 Redis去重**: 使用Redis SETNX实现分布式去重 +- ✅ **P0-4 缓存失效**: 事件驱动的自动缓存失效机制 +- ✅ **完整监控**: 14个Grafana Dashboard面板 +- ✅ **智能告警**: 18条告警规则覆盖所有关键场景 + +--- + +## 3. 部署步骤 + +### 3.1 部署基础资源 + +```bash +# 进入K8s配置目录 +cd /Users/brunogao/work/codes/ljwx/ljwx-health/ljwx-boot/k8s + +# Step 1: 创建Secret（数据库和Redis密码） +kubectl apply -f secret.yml.template -n ljwx-health +# 注意: 先编辑secret.yml.template，修改为实际密码，然后重命名为secret.yml + +# Step 2: 创建ConfigMap（应用配置 + P0配置） +kubectl apply -f configmap.yml -n ljwx-health + +# Step 3: 创建Service（暴露ljwx-boot服务） +kubectl apply -f service.yml -n ljwx-health +``` + +**验证基础资源**: +```bash +kubectl get secret,configmap,svc -n ljwx-health -l app=ljwx-boot +``` + +### 3.2 部署应用 + +```bash +# Step 4: 创建Deployment（部署ljwx-boot应用） +kubectl apply -f deployment.yml -n ljwx-health + +# Step 5: 检查Pod状态 +kubectl get pods -n ljwx-health -l app=ljwx-boot -w +# 等待Pod状态变为Running (约2-3分钟) +``` + +**验证应用健康**: +```bash +# 查看Pod日志 +kubectl logs -f -n ljwx-health -l app=ljwx-boot --tail=100 + +# 检查健康端点 +kubectl port-forward -n ljwx-health svc/ljwx-boot 9998:9998 & +curl http://localhost:9998/actuator/health +# 预期返回: {"status":"UP"} +``` + +### 3.3 部署监控组件 + +```bash +# Step 6: 创建ServiceMonitor（Prometheus自动发现） +kubectl apply -f servicemonitor.yml -n ljwx-health + +# Step 7: 创建PrometheusRule（告警规则） +kubectl apply -f prometheusrule.yml -n ljwx-health + +# Step 8: 创建Grafana Dashboard ConfigMap（可选） +kubectl apply -f grafana-dashboard-configmap.yml -n ljwx-health +``` + +**验证监控资源**: +```bash +kubectl get servicemonitor,prometheusrule -n ljwx-health +``` + +### 3.4 配置Ingress（可选） + +```bash +# Step 9: 配置Ingress暴露服务 +kubectl apply -f ingress.yml -n ljwx-health + +# 查看Ingress地址 +kubectl get ingress -n ljwx-health +``` + +--- + +## 4. 验证部署 + +### 4.1 验证应用运行状态 + +```bash +# 检查所有资源 +kubectl get all -n ljwx-health -l app=ljwx-boot + +# 预期输出: +# NAME READY STATUS RESTARTS AGE +# pod/ljwx-boot-xxxxxxxxx-xxxxx 1/1 Running 0 5m +# pod/ljwx-boot-xxxxxxxxx-yyyyy 1/1 Running 0 5m +# +# NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +# service/ljwx-boot ClusterIP 10.96.xxx.xxx 9998/TCP 10m +# +# NAME READY UP-TO-DATE AVAILABLE AGE +# deployment.apps/ljwx-boot 2/2 2 2 10m +``` + +### 4.2 验证Prometheus采集 + +```bash +# 方法1: 通过Prometheus UI验证 +kubectl port-forward -n monitoring svc/prometheus 9090:9090 + +# 访问 http://localhost:9090/targets +# 查找 ljwx-boot target，状态应为 UP + +# 方法2: 查询P0 Metrics +# 在Prometheus UI执行查询: +health_cache_invalidation_enabled +health_dedup_mode +redis_db_keys +``` + +### 4.3 验证告警规则加载 + +```bash +# 访问 http://localhost:9090/alerts +# 应该看到以下告警规则组: +# - ljwx-boot.availability (2条规则) +# - ljwx-boot.p0-3-redis-dedup (3条规则) +# - ljwx-boot.p0-4-cache-invalidation (3条规则) +# - ljwx-boot.jvm-resources (2条规则) +# - ljwx-boot.database (2条规则) +# - ljwx-boot.performance (2条规则) +# - ljwx-boot.pod-resources (3条规则) +``` + +### 4.4 验证P0功能 + +```bash +# 检查P0 Metrics暴露 +kubectl exec -n ljwx-health -it $(kubectl get pod -n ljwx-health -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') -- \ + curl -s http://localhost:9998/actuator/prometheus | grep -E "health_cache|health_dedup" + +# 预期输出类似: +# health_cache_invalidation_enabled{...} 1.0 +# health_dedup_mode{...} 1.0 +# health_cache_invalidation_total{...} 0.0 +``` + +--- + +## 5. Grafana Dashboard配置 + +### 5.1 方式1: 通过UI手动导入（推荐） + +```bash +# Step 1: 端口转发到Grafana +kubectl port-forward -n monitoring svc/grafana 3000:3000 + +# Step 2: 访问 http://localhost:3000 +# 登录: admin / + +# Step 3: 导入Dashboard +# - 点击左侧菜单 "Dashboards" → "+ Import" +# - 点击 "Upload JSON file" +# - 选择文件: ljwx-boot/monitoring/grafana-dashboard-p0.json +# - 选择数据源: Prometheus (应该是默认数据源) +# - 点击 "Import" + +# Step 4: 查看Dashboard +# - 导入成功后会自动跳转到Dashboard页面 +# - 应该看到14个面板显示数据 +``` + +### 5.2 方式2: 使用kubectl apply导入 + +```bash +# 如果Grafana配置了Dashboard sidecar auto-discovery +kubectl label configmap ljwx-boot-p0-dashboard -n ljwx-health grafana_dashboard=1 + +# Grafana sidecar会自动检测并导入Dashboard +# 等待30秒后刷新Grafana UI +``` + +### 5.3 方式3: 使用Grafana API自动导入 + +```bash +# 获取Grafana admin token +GRAFANA_TOKEN=$(kubectl get secret -n monitoring grafana-admin-credentials -o jsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d) + +# 导入Dashboard +kubectl port-forward -n monitoring svc/grafana 3000:3000 & + +curl -X POST http://admin:${GRAFANA_TOKEN}@localhost:3000/api/dashboards/db \ + -H "Content-Type: application/json" \ + -d @ljwx-boot/monitoring/grafana-dashboard-p0.json +``` + +--- + +## 6. 故障排查 + +### 6.1 Pod无法启动 + +**症状**: Pod状态为 `CrashLoopBackOff` 或 `Error` + +**排查步骤**: +```bash +# 1. 查看Pod事件 +kubectl describe pod -n ljwx-health -l app=ljwx-boot + +# 2. 查看容器日志 +kubectl logs -n ljwx-health -l app=ljwx-boot --tail=200 + +# 3. 常见问题检查 +# - MySQL连接失败: 检查mysql-service是否可达 +kubectl exec -n ljwx-health -it $(kubectl get pod -n ljwx-health -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') -- \ + nc -zv mysql-service.ljwx-health.svc.cluster.local 3306 + +# - Redis连接失败: 检查redis-service是否可达 +kubectl exec -n ljwx-health -it $(kubectl get pod -n ljwx-health -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') -- \ + nc -zv redis-service.ljwx-health.svc.cluster.local 6379 + +# - 配置错误: 检查ConfigMap +kubectl get configmap ljwx-boot-config -n ljwx-health -o yaml +``` + +### 6.2 Prometheus无法采集Metrics + +**症状**: Prometheus UI中ljwx-boot target显示为 DOWN + +**排查步骤**: +```bash +# 1. 检查ServiceMonitor配置 +kubectl describe servicemonitor ljwx-boot-p0-monitor -n ljwx-health + +# 2. 检查Prometheus是否有权限访问ljwx-health namespace +kubectl get rolebinding -n ljwx-health | grep prometheus + +# 3. 验证metrics端点可访问 +kubectl port-forward -n ljwx-health svc/ljwx-boot 9998:9998 +curl http://localhost:9998/actuator/prometheus + +# 4. 检查Prometheus日志 +kubectl logs -n monitoring -l app=prometheus --tail=100 | grep ljwx-boot +``` + +### 6.3 告警规则未生效 + +**症状**: Prometheus Alerts页面看不到ljwx-boot的告警规则 + +**排查步骤**: +```bash +# 1. 检查PrometheusRule资源 +kubectl get prometheusrule ljwx-boot-p0-alerts -n ljwx-health -o yaml + +# 2. 检查PrometheusRule是否被Prometheus Operator识别 +kubectl describe prometheusrule ljwx-boot-p0-alerts -n ljwx-health + +# 3. 查看Prometheus Operator日志 +kubectl logs -n monitoring -l app=prometheus-operator --tail=100 + +# 4. 验证规则格式 +# 访问 http://localhost:9090/config +# 查找 ljwx-boot.availability 等规则组 +``` + +### 6.4 Grafana Dashboard无数据 + +**症状**: Dashboard显示 "No Data" + +**排查步骤**: +```bash +# 1. 确认Prometheus在采集数据 +# 访问 Prometheus UI: http://localhost:9090/graph +# 执行查询: health_cache_invalidation_total + +# 2. 检查Grafana数据源配置 +# Grafana UI: Configuration → Data Sources → Prometheus +# URL应该是: http://prometheus.monitoring.svc.cluster.local:9090 + +# 3. 检查Dashboard查询表达式 +# 打开Dashboard → 点击Panel → Edit +# 验证查询表达式是否正确 + +# 4. 确认P0代码已激活 +kubectl exec -n ljwx-health -it $(kubectl get pod -n ljwx-health -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') -- \ + curl -s http://localhost:9998/actuator/prometheus | grep health_cache +``` + +--- + +## 7. 运维操作 + +### 7.1 更新应用配置 + +```bash +# 1. 编辑ConfigMap +kubectl edit configmap ljwx-boot-config -n ljwx-health + +# 2. 重启Pod使配置生效（滚动更新） +kubectl rollout restart deployment ljwx-boot -n ljwx-health + +# 3. 验证更新 +kubectl rollout status deployment ljwx-boot -n ljwx-health +``` + +### 7.2 扩缩容 + +```bash +# 扩容到3个副本 +kubectl scale deployment ljwx-boot -n ljwx-health --replicas=3 + +# 验证 +kubectl get pods -n ljwx-health -l app=ljwx-boot +``` + +### 7.3 查看日志 + +```bash +# 查看所有Pod日志 +kubectl logs -f -n ljwx-health -l app=ljwx-boot --all-containers=true + +# 查看特定Pod日志 +kubectl logs -f -n ljwx-health ljwx-boot-xxxxxxxxx-xxxxx + +# 查看上一次启动的日志（Pod重启后） +kubectl logs -p -n ljwx-health ljwx-boot-xxxxxxxxx-xxxxx +``` + +### 7.4 性能压测 + +```bash +# 方法1: 在Pod内运行性能测试 +kubectl exec -n ljwx-health -it $(kubectl get pod -n ljwx-health -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') -- \ + /bin/bash -c "cd /app && ./scripts/performance-test.sh --users=100 --duration=60" + +# 方法2: 从集群外访问（需要配置Ingress或NodePort） +# 假设Ingress地址为: http://ljwx-boot.example.com +./ljwx-boot/scripts/performance-test.sh --host=ljwx-boot.example.com --users=1000 --duration=300 +``` + +### 7.5 备份和恢复 + +```bash +# 备份所有K8s资源 +kubectl get all,servicemonitor,prometheusrule,configmap,secret -n ljwx-health -o yaml > ljwx-health-backup.yaml + +# 恢复资源 +kubectl apply -f ljwx-health-backup.yaml +``` + +### 7.6 删除部署 + +```bash +# 删除监控组件（保留应用） +kubectl delete servicemonitor,prometheusrule -n ljwx-health -l app=ljwx-boot + +# 删除整个应用 +kubectl delete deployment,service,configmap,secret -n ljwx-health -l app=ljwx-boot + +# 删除namespace（慎重！） +# kubectl delete namespace ljwx-health +``` + +--- + +## 8. 监控指标说明 + +### 8.1 P0-3 Redis去重指标 + +| 指标名称 | 类型 | 说明 | +|---------|------|------| +| `health_dedup_mode` | Gauge | 去重模式（1=Redis, 0=内存）| +| `redis_db_keys{db="database1"}` | Gauge | Redis去重Key总数 | +| `redis_memory_used_bytes` | Gauge | Redis内存使用量 | + +### 8.2 P0-4 缓存失效指标 + +| 指标名称 | 类型 | 说明 | +|---------|------|------| +| `health_cache_invalidation_enabled` | Gauge | 缓存失效功能状态（1=启用, 0=禁用）| +| `health_cache_invalidation_total{result,type}` | Counter | 缓存失效次数 | +| `health_cache_invalidation_duration_seconds` | Histogram | 缓存失效耗时 | + +### 8.3 应用性能指标 + +| 指标名称 | 类型 | 说明 | +|---------|------|------| +| `http_server_requests_seconds` | Histogram | HTTP请求响应时间 | +| `http_server_requests_active_seconds` | Summary | 活跃请求数 | +| `jvm_memory_used_bytes{area="heap"}` | Gauge | JVM堆内存使用 | +| `hikaricp_connections_active` | Gauge | 活跃数据库连接数 | + +--- + +## 9. 告警规则说明 + +### 9.1 告警级别 + +- **critical**: 严重告警，需要立即处理 +- **warning**: 警告告警，需要关注 + +### 9.2 告警规则列表 + +| 告警名称 | 级别 | 触发条件 | 持续时间 | +|---------|------|---------|---------| +| LjwxBootDown | critical | 服务不可用 | 1分钟 | +| LjwxBootHighErrorRate | warning | 5xx错误率>5% | 5分钟 | +| RedisConnectionFailed | critical | Redis降级到内存模式 | 2分钟 | +| RedisMemoryHigh | warning | Redis内存>80% | 10分钟 | +| CacheInvalidationHighFailureRate | warning | 缓存失效失败率>5% | 10分钟 | +| JvmMemoryHigh | warning | JVM堆内存>85% | 10分钟 | +| DatabaseConnectionPoolExhausted | critical | 连接池>90% | 5分钟 | +| ApiResponseTimeSlow | warning | P95响应时间>1s | 10分钟 | +| PodCpuHigh | warning | CPU使用率>80% | 10分钟 | +| PodRestartFrequent | critical | 1小时重启>3次 | 立即 | + +--- + +## 10. 相关文档 + +- [P0-DEV-QUICK-START.md](../../P0-DEV-QUICK-START.md) - 本地开发环境快速启动指南 +- [P0_EXECUTION_PLAN.md](../docs/architecture/P0_EXECUTION_PLAN.md) - P0执行计划 +- [P0-3_REDIS_DEDUP_IMPLEMENTATION.md](../docs/architecture/P0-3_REDIS_DEDUP_IMPLEMENTATION.md) - P0-3实施报告 +- [P0-4_CACHE_INVALIDATION_IMPLEMENTATION.md](../docs/architecture/P0-4_CACHE_INVALIDATION_IMPLEMENTATION.md) - P0-4实施报告 + +--- + +## 11. 联系支持 + +遇到问题？联系团队: +- **DevOps团队**: devops@ljwx.com +- **运维支持**: ops@ljwx.com +- **技术负责人**: bruno.gao@ljwx.com + +--- + +**最后更新**: 2025-12-15 +**文档版本**: v1.0 diff --git a/ljwx-boot/k8s/configmap.yml b/ljwx-boot/k8s/configmap.yml index a239d0d2..7ad72052 100644 --- a/ljwx-boot/k8s/configmap.yml +++ b/ljwx-boot/k8s/configmap.yml @@ -53,17 +53,40 @@ data: endpoints: web: exposure: - include: health,info,metrics,prometheus + include: "*" # 暴露所有端点（生产环境可限制） base-path: /actuator endpoint: health: - show-details: when-authorized + show-details: always # 生产环境改为when-authorized + show-components: always probes: enabled: true + metrics: + enabled: true + prometheus: + enabled: true metrics: export: prometheus: enabled: true + step: 30s # 30秒数据间隔 + tags: + application: ljwx-boot + environment: kubernetes # K8s环境标识 + + # P0-3 健康数据去重配置 + health: + dedup: + use-redis: true # 使用Redis去重（推荐） + ttl-hours: 24 # Redis Key TTL 24小时 + use-content-hash: true # 启用内容哈希去重 + memory-cleanup-interval-ms: 3600000 # 内存模式清理间隔（1小时） + memory-warning-threshold: 100000 # 内存Key数量告警阈值 + + # P0-4 缓存自动失效配置 + cache: + auto-invalidation: true # 启用自动缓存失效 + invalidation-delay-ms: 1000 # 失效延迟1秒，避免并发问题 # 日志配置 logging: diff --git a/ljwx-boot/k8s/deploy-p0.sh b/ljwx-boot/k8s/deploy-p0.sh new file mode 100755 index 00000000..37809923 --- /dev/null +++ b/ljwx-boot/k8s/deploy-p0.sh @@ -0,0 +1,253 @@ +#!/bin/bash +############################################################################### +# P0监控 - K8s快速部署脚本 +# 用途: 一键部署ljwx-boot到K8s并配置P0监控 +# 用法: ./deploy-p0.sh [--namespace=ljwx-health] [--dry-run] +############################################################################### + +set -e + +# ========== 配置参数 ========== +NAMESPACE="${NAMESPACE:-ljwx-health}" +DRY_RUN=false +SKIP_MONITORING=false + +# 颜色输出 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# ========== 参数解析 ========== +for arg in "$@"; do + case $arg in + --namespace=*) + NAMESPACE="${arg#*=}" + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --skip-monitoring) + SKIP_MONITORING=true + shift + ;; + --help|-h) + echo "用法: $0 [选项]" + echo "" + echo "选项:" + echo " --namespace= 目标namespace (默认: ljwx-health)" + echo " --dry-run 只显示将要执行的命令，不实际执行" + echo " --skip-monitoring 跳过监控组件部署" + echo " --help 显示帮助信息" + echo "" + echo "示例:" + echo " $0" + echo " $0 --namespace=production" + echo " $0 --dry-run" + exit 0 + ;; + esac +done + +# ========== 辅助函数 ========== +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +run_cmd() { + local cmd="$1" + local desc="$2" + + if [ "$DRY_RUN" = true ]; then + echo -e "${BLUE}[DRY-RUN]${NC} $desc" + echo " $ $cmd" + else + log_info "$desc" + eval "$cmd" + if [ $? -eq 0 ]; then + log_success "完成: $desc" + else + log_error "失败: $desc" + exit 1 + fi + fi +} + +check_prerequisite() { + local cmd="$1" + local name="$2" + + if ! command -v "$cmd" &> /dev/null; then + log_error "$name 未安装，请先安装 $cmd" + exit 1 + fi +} + +# ========== 前置检查 ========== +echo "==========================================" +echo "P0监控 - K8s快速部署" +echo "==========================================" +echo "Namespace: $NAMESPACE" +echo "Dry Run: $DRY_RUN" +echo "==========================================" +echo "" + +log_info "执行前置检查..." + +# 检查依赖工具 +check_prerequisite "kubectl" "kubectl" + +# 检查K8s连接 +if ! kubectl cluster-info &> /dev/null; then + log_error "无法连接到K8s集群，请检查kubeconfig配置" + exit 1 +fi +log_success "K8s集群连接正常" + +# 检查namespace是否存在 +if ! kubectl get namespace "$NAMESPACE" &> /dev/null; then + log_warning "Namespace '$NAMESPACE' 不存在，将自动创建" + run_cmd "kubectl create namespace $NAMESPACE" "创建namespace: $NAMESPACE" +else + log_success "Namespace '$NAMESPACE' 已存在" +fi + +# 检查Prometheus Operator CRDs +if ! kubectl get crd servicemonitors.monitoring.coreos.com &> /dev/null; then + log_error "Prometheus Operator CRDs未安装，请先安装Prometheus Operator" + exit 1 +fi +log_success "Prometheus Operator CRDs已安装" + +echo "" + +# ========== 部署基础资源 ========== +log_info "部署基础资源..." + +# Secret +if [ -f "secret.yml" ]; then + run_cmd "kubectl apply -f secret.yml -n $NAMESPACE" "创建Secret" +else + log_warning "secret.yml 不存在，请根据 secret.yml.template 创建" + if [ -f "secret.yml.template" ]; then + log_info "提示: cp secret.yml.template secret.yml && vim secret.yml" + fi +fi + +# ConfigMap +run_cmd "kubectl apply -f configmap.yml -n $NAMESPACE" "创建ConfigMap" + +# Service +run_cmd "kubectl apply -f service.yml -n $NAMESPACE" "创建Service" + +echo "" + +# ========== 部署应用 ========== +log_info "部署应用..." + +run_cmd "kubectl apply -f deployment.yml -n $NAMESPACE" "创建Deployment" + +echo "" +log_info "等待Pod启动..." +if [ "$DRY_RUN" = false ]; then + kubectl wait --for=condition=ready pod -l app=ljwx-boot -n "$NAMESPACE" --timeout=300s || { + log_error "Pod启动超时，请检查日志: kubectl logs -n $NAMESPACE -l app=ljwx-boot" + exit 1 + } +fi +log_success "Pod已启动" + +echo "" + +# ========== 部署监控组件 ========== +if [ "$SKIP_MONITORING" = false ]; then + log_info "部署监控组件..." + + run_cmd "kubectl apply -f servicemonitor.yml -n $NAMESPACE" "创建ServiceMonitor" + run_cmd "kubectl apply -f prometheusrule.yml -n $NAMESPACE" "创建PrometheusRule" + run_cmd "kubectl apply -f grafana-dashboard-configmap.yml -n $NAMESPACE" "创建Grafana Dashboard ConfigMap" + + echo "" +else + log_warning "跳过监控组件部署 (--skip-monitoring)" +fi + +# ========== 验证部署 ========== +log_info "验证部署..." + +if [ "$DRY_RUN" = false ]; then + echo "" + echo "==========================================" + echo "部署资源清单" + echo "==========================================" + kubectl get all,servicemonitor,prometheusrule -n "$NAMESPACE" -l app=ljwx-boot + echo "" +fi + +# ========== 健康检查 ========== +if [ "$DRY_RUN" = false ]; then + log_info "执行健康检查..." + + POD_NAME=$(kubectl get pod -n "$NAMESPACE" -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}') + + if kubectl exec -n "$NAMESPACE" "$POD_NAME" -- curl -sf http://localhost:9998/actuator/health > /dev/null 2>&1; then + log_success "应用健康检查通过" + else + log_warning "应用健康检查失败，请检查日志" + fi + + echo "" +fi + +# ========== 完成总结 ========== +echo "==========================================" +echo "部署完成" +echo "==========================================" +echo "" + +if [ "$DRY_RUN" = false ]; then + log_info "下一步操作:" + echo "" + echo "1. 查看Pod状态:" + echo " kubectl get pods -n $NAMESPACE -l app=ljwx-boot" + echo "" + echo "2. 查看日志:" + echo " kubectl logs -f -n $NAMESPACE -l app=ljwx-boot" + echo "" + echo "3. 验证Prometheus采集:" + echo " kubectl port-forward -n monitoring svc/prometheus 9090:9090" + echo " 访问: http://localhost:9090/targets" + echo " 查找: ljwx-boot target (状态应为 UP)" + echo "" + echo "4. 导入Grafana Dashboard:" + echo " kubectl port-forward -n monitoring svc/grafana 3000:3000" + echo " 访问: http://localhost:3000" + echo " 导入: ljwx-boot/monitoring/grafana-dashboard-p0.json" + echo "" + echo "5. 验证P0 Metrics:" + echo " POD=\$(kubectl get pod -n $NAMESPACE -l app=ljwx-boot -o jsonpath='{.items[0].metadata.name}')" + echo " kubectl exec -n $NAMESPACE \$POD -- curl -s http://localhost:9998/actuator/prometheus | grep health_cache" + echo "" + + log_success "P0监控部署完成！" +else + log_info "这是DRY-RUN模式，未实际执行任何命令" + log_info "移除 --dry-run 参数以实际部署" +fi + +echo "" diff --git a/ljwx-boot/k8s/grafana-dashboard-configmap.yml b/ljwx-boot/k8s/grafana-dashboard-configmap.yml new file mode 100644 index 00000000..3a743405 --- /dev/null +++ b/ljwx-boot/k8s/grafana-dashboard-configmap.yml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ljwx-boot-p0-dashboard + namespace: ljwx-health + labels: + app: ljwx-boot + grafana_dashboard: "1" # Grafana sidecar会自动发现此标签 +data: + # Dashboard配置说明 + dashboard-config.json: | + { + "apiVersion": 1, + "providers": [ + { + "name": "ljwx-boot-p0", + "orgId": 1, + "folder": "LJWX Health", + "type": "file", + "disableDeletion": false, + "updateIntervalSeconds": 30, + "allowUiUpdates": true, + "options": { + "path": "/var/lib/grafana/dashboards/ljwx-boot" + } + } + ] + } + +--- +# 注意: Dashboard JSON文件太大，建议使用以下两种方式之一导入: +# +# 方式1: 通过Grafana UI手动导入 (推荐) +# kubectl port-forward -n monitoring svc/grafana 3000:3000 +# 访问 http://localhost:3000 +# 登录后: Dashboards -> Import -> Upload JSON file +# 选择文件: ljwx-boot/monitoring/grafana-dashboard-p0.json +# +# 方式2: 使用Grafana API自动导入 +# GRAFANA_URL="http://grafana.monitoring.svc.cluster.local:3000" +# GRAFANA_TOKEN="your-admin-token" +# kubectl create secret generic grafana-dashboard-p0 \ +# --from-file=dashboard.json=ljwx-boot/monitoring/grafana-dashboard-p0.json \ +# -n ljwx-health +# +# 方式3: 使用Grafana Dashboard CRD (如果安装了grafana-operator) +# apiVersion: grafana.integreatly.org/v1beta1 +# kind: GrafanaDashboard +# metadata: +# name: ljwx-boot-p0 +# spec: +# json: diff --git a/ljwx-boot/k8s/prometheusrule.yml b/ljwx-boot/k8s/prometheusrule.yml new file mode 100644 index 00000000..4d50e9bd --- /dev/null +++ b/ljwx-boot/k8s/prometheusrule.yml @@ -0,0 +1,256 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: ljwx-boot-p0-alerts + namespace: ljwx-health + labels: + app: ljwx-boot + release: prometheus + monitoring: p0 +spec: + groups: + # ========== 服务可用性告警 ========== + - name: ljwx-boot.availability + interval: 30s + rules: + - alert: LjwxBootDown + expr: up{job="ljwx-boot"} == 0 + for: 1m + labels: + severity: critical + component: ljwx-boot + annotations: + summary: "LJWX Boot服务不可用" + description: "LJWX Boot服务 {{ $labels.pod }} 已停止响应超过1分钟" + runbook_url: "https://wiki.ljwx.com/runbooks/ljwx-boot-down" + + - alert: LjwxBootHighErrorRate + expr: | + rate(http_server_requests_seconds_count{status=~"5..",job="ljwx-boot"}[5m]) + / + rate(http_server_requests_seconds_count{job="ljwx-boot"}[5m]) > 0.05 + for: 5m + labels: + severity: warning + component: ljwx-boot + annotations: + summary: "LJWX Boot错误率过高" + description: "Pod {{ $labels.pod }} 5xx错误率超过5%已持续5分钟" + + # ========== P0-3 Redis去重告警 ========== + - name: ljwx-boot.p0-3-redis-dedup + interval: 30s + rules: + - alert: RedisConnectionFailed + expr: health_dedup_mode{job="ljwx-boot"} == 0 + for: 2m + labels: + severity: critical + component: redis + feature: p0-3 + annotations: + summary: "Redis连接失败，已降级到内存模式" + description: "Pod {{ $labels.pod }} Redis连接失败，去重功能已切换到内存模式，可能影响去重准确性" + action: "检查Redis服务状态和网络连接" + + - alert: RedisMemoryHigh + expr: | + redis_memory_used_bytes{job="ljwx-boot"} + / + redis_memory_max_bytes > 0.8 + for: 10m + labels: + severity: warning + component: redis + feature: p0-3 + annotations: + summary: "Redis内存使用率超过80%" + description: "Redis实例 {{ $labels.instance }} 内存使用率 {{ $value | humanizePercentage }}，去重Key可能被驱逐" + action: "考虑增加Redis内存或调整TTL策略" + + - alert: RedisKeyExplosion + expr: | + rate(redis_db_keys{db="database1",job="ljwx-boot"}[15m]) > 1000 + for: 5m + labels: + severity: warning + component: redis + feature: p0-3 + annotations: + summary: "Redis去重Key增长异常" + description: "Redis Key增长速度超过1000/分钟，可能存在数据异常或攻击" + action: "检查数据上传日志和用户行为" + + # ========== P0-4 缓存失效告警 ========== + - name: ljwx-boot.p0-4-cache-invalidation + interval: 30s + rules: + - alert: CacheInvalidationDisabled + expr: health_cache_invalidation_enabled{job="ljwx-boot"} == 0 + for: 1m + labels: + severity: warning + component: cache + feature: p0-4 + annotations: + summary: "缓存自动失效功能已禁用" + description: "Pod {{ $labels.pod }} 缓存自动失效功能处于禁用状态，可能导致数据不一致" + action: "检查配置 health.cache.auto-invalidation" + + - alert: CacheInvalidationHighFailureRate + expr: | + rate(health_cache_invalidation_total{result="failure",job="ljwx-boot"}[5m]) + / + rate(health_cache_invalidation_total{job="ljwx-boot"}[5m]) > 0.05 + for: 10m + labels: + severity: warning + component: cache + feature: p0-4 + annotations: + summary: "缓存失效失败率过高" + description: "Pod {{ $labels.pod }} 缓存失效失败率 {{ $value | humanizePercentage }} 超过5%" + action: "检查Redis连接和缓存服务日志" + + - alert: CacheInvalidationSlowResponse + expr: | + histogram_quantile(0.95, + rate(health_cache_invalidation_duration_seconds_bucket{job="ljwx-boot"}[5m]) + ) > 0.5 + for: 10m + labels: + severity: warning + component: cache + feature: p0-4 + annotations: + summary: "缓存失效响应时间过长" + description: "Pod {{ $labels.pod }} P95缓存失效耗时超过500ms" + action: "检查Redis性能和网络延迟" + + # ========== JVM资源告警 ========== + - name: ljwx-boot.jvm-resources + interval: 30s + rules: + - alert: JvmMemoryHigh + expr: | + jvm_memory_used_bytes{area="heap",job="ljwx-boot"} + / + jvm_memory_max_bytes{area="heap"} > 0.85 + for: 10m + labels: + severity: warning + component: jvm + annotations: + summary: "JVM堆内存使用率过高" + description: "Pod {{ $labels.pod }} 堆内存使用率 {{ $value | humanizePercentage }} 超过85%" + action: "检查内存泄漏或考虑增加JVM内存" + + - alert: JvmGcTimeHigh + expr: | + rate(jvm_gc_pause_seconds_sum{job="ljwx-boot"}[5m]) + / + rate(jvm_gc_pause_seconds_count{job="ljwx-boot"}[5m]) > 0.1 + for: 10m + labels: + severity: warning + component: jvm + annotations: + summary: "JVM GC时间占比过高" + description: "Pod {{ $labels.pod }} 平均GC停顿时间 {{ $value | humanizeDuration }} 超过100ms" + + # ========== 数据库连接池告警 ========== + - name: ljwx-boot.database + interval: 30s + rules: + - alert: DatabaseConnectionPoolExhausted + expr: | + hikaricp_connections_active{job="ljwx-boot"} + / + hikaricp_connections{job="ljwx-boot"} > 0.9 + for: 5m + labels: + severity: critical + component: database + annotations: + summary: "数据库连接池即将耗尽" + description: "Pod {{ $labels.pod }} 活跃连接占比 {{ $value | humanizePercentage }} 超过90%" + action: "检查慢查询或考虑增加连接池大小" + + - alert: DatabaseConnectionPoolLow + expr: hikaricp_connections_idle{job="ljwx-boot"} == 0 + for: 5m + labels: + severity: warning + component: database + annotations: + summary: "数据库连接池无空闲连接" + description: "Pod {{ $labels.pod }} 连接池无空闲连接已持续5分钟" + + # ========== 应用性能告警 ========== + - name: ljwx-boot.performance + interval: 30s + rules: + - alert: ApiResponseTimeSlow + expr: | + histogram_quantile(0.95, + rate(http_server_requests_seconds_bucket{job="ljwx-boot",uri!~"/actuator.*"}[5m]) + ) > 1.0 + for: 10m + labels: + severity: warning + component: api + annotations: + summary: "API响应时间过慢" + description: "Pod {{ $labels.pod }} P95响应时间超过1秒" + action: "检查慢接口和数据库性能" + + - alert: HighConcurrentRequests + expr: | + http_server_requests_active_seconds_count{job="ljwx-boot"} > 100 + for: 5m + labels: + severity: warning + component: api + annotations: + summary: "并发请求数过高" + description: "Pod {{ $labels.pod }} 并发请求数 {{ $value }} 超过100" + + # ========== Pod资源告警 ========== + - name: ljwx-boot.pod-resources + interval: 30s + rules: + - alert: PodCpuHigh + expr: | + rate(container_cpu_usage_seconds_total{pod=~"ljwx-boot.*",namespace="ljwx-health"}[5m]) > 0.8 + for: 10m + labels: + severity: warning + component: pod + annotations: + summary: "Pod CPU使用率过高" + description: "Pod {{ $labels.pod }} CPU使用率超过80%已持续10分钟" + + - alert: PodMemoryHigh + expr: | + container_memory_usage_bytes{pod=~"ljwx-boot.*",namespace="ljwx-health"} + / + container_spec_memory_limit_bytes > 0.85 + for: 10m + labels: + severity: warning + component: pod + annotations: + summary: "Pod内存使用率过高" + description: "Pod {{ $labels.pod }} 内存使用率 {{ $value | humanizePercentage }} 超过85%" + + - alert: PodRestartFrequent + expr: | + increase(kube_pod_container_status_restarts_total{pod=~"ljwx-boot.*",namespace="ljwx-health"}[1h]) > 3 + for: 0m + labels: + severity: critical + component: pod + annotations: + summary: "Pod重启频繁" + description: "Pod {{ $labels.pod }} 在过去1小时内重启超过3次" + action: "检查Pod日志和健康检查配置" diff --git a/ljwx-boot/k8s/servicemonitor.yml b/ljwx-boot/k8s/servicemonitor.yml new file mode 100644 index 00000000..3dba1451 --- /dev/null +++ b/ljwx-boot/k8s/servicemonitor.yml @@ -0,0 +1,40 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: ljwx-boot-p0-monitor + namespace: ljwx-health + labels: + app: ljwx-boot + release: prometheus # 匹配Prometheus Operator的selector + monitoring: p0 +spec: + selector: + matchLabels: + app: ljwx-boot + + # 选择要监控的Service + namespaceSelector: + matchNames: + - ljwx-health + + # 监控端点配置 + endpoints: + - port: http # 匹配service.yml中的端口名称 + path: /actuator/prometheus + interval: 30s # 采集间隔 + scrapeTimeout: 10s # 采集超时 + + # 标签重写 + relabelings: + - sourceLabels: [__meta_kubernetes_pod_name] + targetLabel: pod + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - sourceLabels: [__meta_kubernetes_namespace] + targetLabel: namespace + + # Metrics重写 + metricRelabelings: + - sourceLabels: [__name__] + regex: '(health_cache_invalidation_.*|health_dedup_.*|redis_.*|jvm_.*|http_server_.*|hikaricp_.*)' + action: keep # 只保留P0和关键指标 diff --git a/ljwx-boot/ljwx-boot-admin/src/main/resources/application.yml b/ljwx-boot/ljwx-boot-admin/src/main/resources/application.yml index 2ef1678a..b9d49f4e 100644 --- a/ljwx-boot/ljwx-boot-admin/src/main/resources/application.yml +++ b/ljwx-boot/ljwx-boot-admin/src/main/resources/application.yml @@ -135,3 +135,10 @@ health: memory-cleanup-interval-ms: 3600000 # 内存Set警告阈值 memory-warning-threshold: 100000 + + # P0-4 缓存自动失效配置 + cache: + # 是否启用自动缓存失效 + auto-invalidation: true + # 缓存失效延迟（毫秒），避免并发问题 + invalidation-delay-ms: 1000 diff --git a/ljwx-boot/monitoring/grafana-dashboard-p0.json b/ljwx-boot/monitoring/grafana-dashboard-p0.json index 60c39cb8..0ed7128b 100644 --- a/ljwx-boot/monitoring/grafana-dashboard-p0.json +++ b/ljwx-boot/monitoring/grafana-dashboard-p0.json @@ -1,447 +1,654 @@ { - "dashboard": { - "title": "P0 Redis去重 + 缓存失效监控", - "tags": ["p0", "redis", "cache", "health"], - "timezone": "browser", - "schemaVersion": 38, - "version": 1, - "refresh": "30s", - "time": { - "from": "now-6h", - "to": "now" - }, - "panels": [ - { - "id": 1, - "title": "缓存失效 - 总览", - "type": "stat", - "gridPos": {"h": 8, "w": 6, "x": 0, "y": 0}, - "targets": [ - { - "expr": "sum(health_cache_invalidation_total)", - "legendFormat": "总事件数", - "refId": "A" - } - ], - "options": { - "colorMode": "background", - "graphMode": "area", - "textMode": "value_and_name" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - {"value": null, "color": "green"}, - {"value": 10000, "color": "yellow"}, - {"value": 100000, "color": "red"} - ] - } - } - } + "title": "P0 Redis去重 + 缓存失效监控", + "tags": [ + "p0", + "redis", + "cache", + "health" + ], + "timezone": "browser", + "schemaVersion": 38, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "title": "缓存失效 - 总览", + "type": "stat", + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 }, - { - "id": 2, - "title": "缓存失效 - 成功率", - "type": "gauge", - "gridPos": {"h": 8, "w": 6, "x": 6, "y": 0}, - "targets": [ - { - "expr": "sum(rate(health_cache_invalidation_total{result=\"success\"}[5m])) / sum(rate(health_cache_invalidation_total[5m])) * 100", - "legendFormat": "成功率", - "refId": "A" - } - ], - "options": { - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"] - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - {"value": null, "color": "red"}, - {"value": 95, "color": "yellow"}, - {"value": 99, "color": "green"} - ] - } - } + "targets": [ + { + "expr": "sum(health_cache_invalidation_total)", + "legendFormat": "总事件数", + "refId": "A" } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "textMode": "value_and_name" }, - { - "id": 3, - "title": "缓存失效 - 成功/失败次数", - "type": "stat", - "gridPos": {"h": 8, "w": 6, "x": 12, "y": 0}, - "targets": [ - { - "expr": "sum(health_cache_invalidation_total{result=\"success\"})", - "legendFormat": "成功", - "refId": "A" - }, - { - "expr": "sum(health_cache_invalidation_total{result=\"failure\"})", - "legendFormat": "失败", - "refId": "B" + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 10000, + "color": "yellow" + }, + { + "value": 100000, + "color": "red" + } + ] } - ], - "options": { - "colorMode": "value", - "textMode": "value_and_name" - } - }, - { - "id": 4, - "title": "缓存失效 - 状态", - "type": "stat", - "gridPos": {"h": 8, "w": 6, "x": 18, "y": 0}, - "targets": [ - { - "expr": "health_cache_invalidation_enabled", - "legendFormat": "自动失效状态", - "refId": "A" - } - ], - "options": { - "colorMode": "background", - "textMode": "value" - }, - "fieldConfig": { - "overrides": [ - { - "matcher": {"id": "byValue", "options": 1}, - "properties": [ - {"id": "mappings", "value": [{"type": "value", "value": "1", "text": "✅ 已启用"}]}, - {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}} - ] - }, - { - "matcher": {"id": "byValue", "options": 0}, - "properties": [ - {"id": "mappings", "value": [{"type": "value", "value": "0", "text": "❌ 已禁用"}]}, - {"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}} - ] - } - ] - } - }, - { - "id": 5, - "title": "缓存失效 - P50/P95/P99耗时", - "type": "timeseries", - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(health_cache_invalidation_duration_seconds_bucket[5m])) * 1000", - "legendFormat": "P50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, rate(health_cache_invalidation_duration_seconds_bucket[5m])) * 1000", - "legendFormat": "P95", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, rate(health_cache_invalidation_duration_seconds_bucket[5m])) * 1000", - "legendFormat": "P99", - "refId": "C" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ms", - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - } - } - }, - "options": { - "tooltip": {"mode": "multi"}, - "legend": {"displayMode": "list", "placement": "bottom"} - } - }, - { - "id": 6, - "title": "缓存失效 - 事件类型分布", - "type": "piechart", - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, - "targets": [ - { - "expr": "sum by (type) (health_cache_invalidation_total)", - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "options": { - "legend": {"displayMode": "table", "placement": "right", "values": ["value", "percent"]}, - "pieType": "donut" - } - }, - { - "id": 7, - "title": "Redis去重 - Key总数", - "type": "stat", - "gridPos": {"h": 8, "w": 6, "x": 0, "y": 16}, - "targets": [ - { - "expr": "redis_db_keys{db=\"database1\"}", - "legendFormat": "Key总数", - "refId": "A" - } - ], - "options": { - "colorMode": "background", - "graphMode": "area", - "textMode": "value_and_name" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - {"value": null, "color": "green"}, - {"value": 500000, "color": "yellow"}, - {"value": 1000000, "color": "red"} - ] - } - } - } - }, - { - "id": 8, - "title": "Redis去重 - Key增长速率", - "type": "timeseries", - "gridPos": {"h": 8, "w": 6, "x": 6, "y": 16}, - "targets": [ - { - "expr": "rate(redis_db_keys{db=\"database1\"}[5m]) * 60", - "legendFormat": "Key/分钟", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "keys", - "custom": { - "lineWidth": 2, - "fillOpacity": 20 - } - } - } - }, - { - "id": 9, - "title": "Redis去重 - 内存使用", - "type": "gauge", - "gridPos": {"h": 8, "w": 6, "x": 12, "y": 16}, - "targets": [ - { - "expr": "redis_memory_used_bytes / 1024 / 1024 / 1024", - "legendFormat": "Redis内存", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "decgbytes", - "min": 0, - "max": 4, - "thresholds": { - "mode": "absolute", - "steps": [ - {"value": null, "color": "green"}, - {"value": 2, "color": "yellow"}, - {"value": 3, "color": "red"} - ] - } - } - } - }, - { - "id": 10, - "title": "Redis去重 - 去重模式", - "type": "stat", - "gridPos": {"h": 8, "w": 6, "x": 18, "y": 16}, - "targets": [ - { - "expr": "health_dedup_mode", - "legendFormat": "去重模式", - "refId": "A" - } - ], - "fieldConfig": { - "overrides": [ - { - "matcher": {"id": "byRegexp", "options": ".*redis.*"}, - "properties": [ - {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}, - {"id": "mappings", "value": [{"type": "value", "value": "1", "text": "✅ Redis模式"}]} - ] - }, - { - "matcher": {"id": "byRegexp", "options": ".*memory.*"}, - "properties": [ - {"id": "color", "value": {"mode": "fixed", "fixedColor": "yellow"}}, - {"id": "mappings", "value": [{"type": "value", "value": "0", "text": "⚠️ 内存模式"}]} - ] - } - ] - } - }, - { - "id": 11, - "title": "缓存失效 - 按类型统计（时序）", - "type": "timeseries", - "gridPos": {"h": 8, "w": 24, "x": 0, "y": 24}, - "targets": [ - { - "expr": "sum by (type) (rate(health_cache_invalidation_total[5m])) * 60", - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "ops", - "custom": { - "lineWidth": 2, - "fillOpacity": 10, - "stacking": {"mode": "normal"} - } - } - }, - "options": { - "tooltip": {"mode": "multi"}, - "legend": {"displayMode": "table", "placement": "right", "values": ["value", "max", "min"]} - } - }, - { - "id": 12, - "title": "JVM内存使用", - "type": "timeseries", - "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, - "targets": [ - { - "expr": "jvm_memory_used_bytes{area=\"heap\"} / 1024 / 1024 / 1024", - "legendFormat": "堆内存", - "refId": "A" - }, - { - "expr": "jvm_memory_max_bytes{area=\"heap\"} / 1024 / 1024 / 1024", - "legendFormat": "堆最大", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "unit": "decgbytes", - "custom": { - "lineWidth": 2, - "fillOpacity": 20 - } - } - } - }, - { - "id": 13, - "title": "数据库连接池", - "type": "timeseries", - "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, - "targets": [ - { - "expr": "hikaricp_connections_active", - "legendFormat": "活跃连接", - "refId": "A" - }, - { - "expr": "hikaricp_connections_idle", - "legendFormat": "空闲连接", - "refId": "B" - }, - { - "expr": "hikaricp_connections", - "legendFormat": "总连接数", - "refId": "C" - } - ], - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "lineWidth": 2, - "fillOpacity": 10 - } - } - } - }, - { - "id": 14, - "title": "告警事件 - 最近失败的缓存失效", - "type": "logs", - "gridPos": {"h": 8, "w": 24, "x": 0, "y": 40}, - "targets": [ - { - "expr": "{job=\"ljwx-boot\"} |= \"❌ 缓存失效失败\"", - "refId": "A" - } - ], - "options": { - "showTime": true, - "wrapLogMessage": true, - "sortOrder": "Descending" } } - ], - "templating": { - "list": [ - { - "name": "datasource", - "type": "datasource", - "query": "prometheus", - "current": { - "text": "Prometheus", - "value": "Prometheus" - } - }, - { - "name": "environment", - "type": "query", - "query": "label_values(health_cache_invalidation_total, environment)", - "current": { - "text": "production", - "value": "production" - }, - "multi": false - } - ] }, - "annotations": { - "list": [ + { + "id": 2, + "title": "缓存失效 - 成功率", + "type": "gauge", + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ { - "name": "部署事件", - "datasource": "Prometheus", - "enable": true, - "expr": "changes(process_start_time_seconds[5m]) > 0", - "iconColor": "blue", - "tagKeys": "deploy" + "expr": "sum(rate(health_cache_invalidation_total{result=\"success\"}[5m])) / sum(rate(health_cache_invalidation_total[5m])) * 100", + "legendFormat": "成功率", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ] + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "red" + }, + { + "value": 95, + "color": "yellow" + }, + { + "value": 99, + "color": "green" + } + ] + } + } + } + }, + { + "id": 3, + "title": "缓存失效 - 成功/失败次数", + "type": "stat", + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "expr": "sum(health_cache_invalidation_total{result=\"success\"})", + "legendFormat": "成功", + "refId": "A" }, { - "name": "告警事件", - "datasource": "Prometheus", - "enable": true, - "expr": "ALERTS{alertstate=\"firing\"}", - "iconColor": "red", - "tagKeys": "alert" + "expr": "sum(health_cache_invalidation_total{result=\"failure\"})", + "legendFormat": "失败", + "refId": "B" } - ] + ], + "options": { + "colorMode": "value", + "textMode": "value_and_name" + } + }, + { + "id": 4, + "title": "缓存失效 - 状态", + "type": "stat", + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "expr": "health_cache_invalidation_enabled", + "legendFormat": "自动失效状态", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "textMode": "value" + }, + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": 1 + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "value": "1", + "text": "✅ 已启用" + } + ] + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": 0 + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "value": "0", + "text": "❌ 已禁用" + } + ] + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + } + }, + { + "id": 5, + "title": "缓存失效 - P50/P95/P99耗时", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(health_cache_invalidation_duration_seconds_bucket[5m])) * 1000", + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(health_cache_invalidation_duration_seconds_bucket[5m])) * 1000", + "legendFormat": "P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, rate(health_cache_invalidation_duration_seconds_bucket[5m])) * 1000", + "legendFormat": "P99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + } + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "缓存失效 - 事件类型分布", + "type": "piechart", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "expr": "sum by (type) (health_cache_invalidation_total)", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut" + } + }, + { + "id": 7, + "title": "Redis去重 - Key总数", + "type": "stat", + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 16 + }, + "targets": [ + { + "expr": "redis_db_keys{db=\"database1\"}", + "legendFormat": "Key总数", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "textMode": "value_and_name" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 500000, + "color": "yellow" + }, + { + "value": 1000000, + "color": "red" + } + ] + } + } + } + }, + { + "id": 8, + "title": "Redis去重 - Key增长速率", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 16 + }, + "targets": [ + { + "expr": "rate(redis_db_keys{db=\"database1\"}[5m]) * 60", + "legendFormat": "Key/分钟", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "keys", + "custom": { + "lineWidth": 2, + "fillOpacity": 20 + } + } + } + }, + { + "id": 9, + "title": "Redis去重 - 内存使用", + "type": "gauge", + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 16 + }, + "targets": [ + { + "expr": "redis_memory_used_bytes / 1024 / 1024 / 1024", + "legendFormat": "Redis内存", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "decgbytes", + "min": 0, + "max": 4, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 2, + "color": "yellow" + }, + { + "value": 3, + "color": "red" + } + ] + } + } + } + }, + { + "id": 10, + "title": "Redis去重 - 去重模式", + "type": "stat", + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 16 + }, + "targets": [ + { + "expr": "health_dedup_mode", + "legendFormat": "去重模式", + "refId": "A" + } + ], + "fieldConfig": { + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*redis.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "value": "1", + "text": "✅ Redis模式" + } + ] + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*memory.*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "value": "0", + "text": "⚠️ 内存模式" + } + ] + } + ] + } + ] + } + }, + { + "id": 11, + "title": "缓存失效 - 按类型统计（时序）", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "targets": [ + { + "expr": "sum by (type) (rate(health_cache_invalidation_total[5m])) * 60", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "stacking": { + "mode": "normal" + } + } + } + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "values": [ + "value", + "max", + "min" + ] + } + } + }, + { + "id": 12, + "title": "JVM内存使用", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "targets": [ + { + "expr": "jvm_memory_used_bytes{area=\"heap\"} / 1024 / 1024 / 1024", + "legendFormat": "堆内存", + "refId": "A" + }, + { + "expr": "jvm_memory_max_bytes{area=\"heap\"} / 1024 / 1024 / 1024", + "legendFormat": "堆最大", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "decgbytes", + "custom": { + "lineWidth": 2, + "fillOpacity": 20 + } + } + } + }, + { + "id": 13, + "title": "数据库连接池", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "targets": [ + { + "expr": "hikaricp_connections_active", + "legendFormat": "活跃连接", + "refId": "A" + }, + { + "expr": "hikaricp_connections_idle", + "legendFormat": "空闲连接", + "refId": "B" + }, + { + "expr": "hikaricp_connections", + "legendFormat": "总连接数", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + } + } + }, + { + "id": 14, + "title": "告警事件 - 最近失败的缓存失效", + "type": "logs", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "targets": [ + { + "expr": "{job=\"ljwx-boot\"} |= \"❌ 缓存失效失败\"", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } } + ], + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": { + "text": "Prometheus", + "value": "Prometheus" + } + }, + { + "name": "environment", + "type": "query", + "query": "label_values(health_cache_invalidation_total, environment)", + "current": { + "text": "production", + "value": "production" + }, + "multi": false + } + ] + }, + "annotations": { + "list": [ + { + "name": "部署事件", + "datasource": "Prometheus", + "enable": true, + "expr": "changes(process_start_time_seconds[5m]) > 0", + "iconColor": "blue", + "tagKeys": "deploy" + }, + { + "name": "告警事件", + "datasource": "Prometheus", + "enable": true, + "expr": "ALERTS{alertstate=\"firing\"}", + "iconColor": "red", + "tagKeys": "alert" + } + ] } -} +} \ No newline at end of file diff --git a/ljwx-boot/monitoring/prometheus-p0-dev.yml b/ljwx-boot/monitoring/prometheus-p0-dev.yml new file mode 100644 index 00000000..b656c16f --- /dev/null +++ b/ljwx-boot/monitoring/prometheus-p0-dev.yml @@ -0,0 +1,41 @@ +global: + scrape_interval: 30s + evaluation_interval: 30s + external_labels: + cluster: 'ljwx-health' + environment: 'p0-development' + +# 加载P0告警规则 +rule_files: + - "/etc/prometheus/alerts/p0-alerts.yml" + +# 数据采集配置 +scrape_configs: + # Prometheus自监控 + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # ljwx-boot Spring Boot应用 (本地运行，通过host.docker.internal访问) + - job_name: 'ljwx-boot' + metrics_path: '/actuator/prometheus' + static_configs: + - targets: ['host.docker.internal:9998'] + labels: + application: 'ljwx-boot' + environment: 'p0-development' + component: 'backend' + mode: 'local' + scrape_interval: 30s + scrape_timeout: 10s + + # Redis Exporter - Redis指标 (Docker容器) + - job_name: 'redis' + static_configs: + - targets: ['redis-exporter:9121'] + labels: + service: 'redis' + component: 'cache' + db: 'database1' + mode: 'docker' + scrape_interval: 30s diff --git a/ljwx-boot/monitoring/prometheus-p0.yml b/ljwx-boot/monitoring/prometheus-p0.yml index c5d7546e..1e2f001f 100644 --- a/ljwx-boot/monitoring/prometheus-p0.yml +++ b/ljwx-boot/monitoring/prometheus-p0.yml @@ -16,11 +16,11 @@ scrape_configs: static_configs: - targets: ['localhost:9090'] - # ljwx-boot Spring Boot应用 (running locally) + # ljwx-boot Spring Boot应用 (Docker网络) - job_name: 'ljwx-boot' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['host.docker.internal:9998'] + - targets: ['ljwx-boot:9998'] labels: application: 'ljwx-boot' environment: 'p0-monitoring' diff --git a/scripts/verify-p0-dev.sh b/scripts/verify-p0-dev.sh new file mode 100755 index 00000000..7b05629b --- /dev/null +++ b/scripts/verify-p0-dev.sh @@ -0,0 +1,220 @@ +#!/bin/bash +############################################################################### +# P0开发环境验证脚本 - 混合模式 +# 用途: 验证Docker基础设施 + 本地业务应用 +# 环境: 基础设施在Docker，ljwx-boot和ljwx-admin在本地运行 +############################################################################### + +set -e + +echo "==========================================" +echo "P0监控体系开发环境验证（混合模式）" +echo "==========================================" +echo "" + +# 检查Docker服务是否运行 +echo "1️⃣ 检查Docker基础设施服务..." +DOCKER_SERVICES="ljwx-mysql ljwx-redis redis-exporter prometheus grafana" +ALL_DOCKER_OK=true + +for service in $DOCKER_SERVICES; do + if docker ps --format '{{.Names}}' | grep -q "^${service}$"; then + STATUS=$(docker inspect -f '{{.State.Status}}' $service 2>/dev/null) + HEALTH=$(docker inspect -f '{{.State.Health.Status}}' $service 2>/dev/null || echo "N/A") + if [ "$HEALTH" != "N/A" ]; then + echo " ✅ $service: $STATUS ($HEALTH)" + else + echo " ✅ $service: $STATUS" + fi + else + echo " ❌ $service 未运行" + ALL_DOCKER_OK=false + fi +done + +if [ "$ALL_DOCKER_OK" = false ]; then + echo "" + echo "❌ Docker基础设施服务未完全启动" + echo " 请先运行: docker-compose -f docker-compose-p0-dev.yml up -d" + exit 1 +fi + +# 检查本地ljwx-boot是否运行 +echo "" +echo "2️⃣ 检查本地ljwx-boot服务..." +if curl -s http://localhost:9998/actuator/health > /dev/null 2>&1; then + BOOT_HEALTH=$(curl -s http://localhost:9998/actuator/health | grep -o '"status":"[^"]*"' | cut -d'"' -f4) + if [ "$BOOT_HEALTH" = "UP" ]; then + echo " ✅ ljwx-boot 运行中 (状态: UP)" + else + echo " ⚠️ ljwx-boot 运行中但状态异常: $BOOT_HEALTH" + fi +else + echo " ⚠️ ljwx-boot 未运行" + echo " 请在 ljwx-boot 目录运行: ./run-local.sh" + echo "" + echo " 继续验证Docker基础设施..." +fi + +# 检查Redis连接 +echo "" +echo "3️⃣ 检查Redis连接..." +REDIS_PING=$(docker exec ljwx-redis redis-cli -a 123456 --no-auth-warning ping 2>/dev/null) +if [ "$REDIS_PING" = "PONG" ]; then + echo " ✅ Redis连接正常" + + # 检查Redis去重Key（如果ljwx-boot在运行） + if curl -s http://localhost:9998/actuator/health > /dev/null 2>&1; then + REDIS_KEY_COUNT=$(docker exec ljwx-redis redis-cli -a 123456 --no-auth-warning \ + KEYS "health:dedup:*" 2>/dev/null | wc -l) + echo " Redis去重Key总数: $REDIS_KEY_COUNT" + + if [ "$REDIS_KEY_COUNT" -gt 0 ]; then + echo " 前3个Key示例:" + docker exec ljwx-redis redis-cli -a 123456 --no-auth-warning \ + KEYS "health:dedup:*" 2>/dev/null | head -3 | sed 's/^/ /' + fi + fi +else + echo " ❌ Redis连接失败" +fi + +# 检查Prometheus +echo "" +echo "4️⃣ 检查Prometheus..." +PROM_PORT=$(docker port prometheus | grep 9090 | cut -d':' -f2) +if curl -s http://localhost:$PROM_PORT/-/healthy > /dev/null 2>&1; then + echo " ✅ Prometheus运行正常" + + # 检查targets + TARGETS_UP=$(curl -s http://localhost:$PROM_PORT/api/v1/targets 2>/dev/null | grep -o '"health":"up"' | wc -l) + echo " 活跃Targets数量: $TARGETS_UP" + + # 检查ljwx-boot target状态 + if curl -s http://localhost:9998/actuator/health > /dev/null 2>&1; then + echo " 💡 Prometheus正在采集本地ljwx-boot指标 (host.docker.internal:9998)" + else + echo " ⚠️ ljwx-boot未运行，Prometheus无法采集指标" + fi +else + echo " ❌ Prometheus连接失败" +fi + +# 检查Grafana +echo "" +echo "5️⃣ 检查Grafana..." +GRAFANA_PORT=$(docker port grafana | grep 3000 | cut -d':' -f2) +if curl -s http://localhost:$GRAFANA_PORT/api/health > /dev/null 2>&1; then + echo " ✅ Grafana运行正常" +else + echo " ❌ Grafana连接失败" +fi + +# 可选: 上传测试数据（如果ljwx-boot在运行） +echo "" +if curl -s http://localhost:9998/actuator/health > /dev/null 2>&1; then + read -p "是否上传测试健康数据？(y/n): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "6️⃣ 上传测试健康数据..." + SUCCESS_COUNT=0 + for i in {1..10}; do + RESPONSE=$(curl -s -X POST http://localhost:9998/api/health/data/upload \ + -H "Content-Type: application/json" \ + -d "{ + \"userId\": $((1000 + i)), + \"customerId\": 100, + \"deviceSn\": \"TEST-$((100 + i))\", + \"heartRate\": $((60 + RANDOM % 40)), + \"bloodOxygen\": $((95 + RANDOM % 5)), + \"temperature\": 36.$((RANDOM % 10)), + \"pressureHigh\": 120, + \"pressureLow\": 80, + \"createTime\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\" + }") + + if echo "$RESPONSE" | grep -q '"success":true'; then + SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) + echo " ✅ 第${i}条数据上传成功" + else + echo " ⚠️ 第${i}条数据上传失败" + fi + sleep 0.5 + done + + echo "" + echo " 上传完成: $SUCCESS_COUNT/10 成功" + + # 等待处理 + echo " 等待5秒处理数据..." + sleep 5 + + # 检查Redis Key + REDIS_KEY_COUNT=$(docker exec ljwx-redis redis-cli -a 123456 --no-auth-warning \ + KEYS "health:dedup:*" 2>/dev/null | wc -l) + echo " 更新后Redis去重Key总数: $REDIS_KEY_COUNT" + fi +fi + +# 服务访问信息 +echo "" +echo "==========================================" +echo "📋 服务访问地址" +echo "==========================================" +echo "" +echo "业务应用 (本地运行):" +if curl -s http://localhost:9998/actuator/health > /dev/null 2>&1; then + echo " ✅ ljwx-boot API: http://localhost:9998" + echo " 📚 API文档: http://localhost:9998/doc.html" + echo " 💓 健康检查: http://localhost:9998/actuator/health" + echo " 📈 Metrics: http://localhost:9998/actuator/prometheus" +else + echo " ⚠️ ljwx-boot: 未运行" + echo " 启动命令: cd ljwx-boot && ./run-local.sh" +fi + +if curl -s http://localhost:3333 > /dev/null 2>&1; then + echo " ✅ ljwx-admin: http://localhost:3333" +elif curl -s http://localhost:5173 > /dev/null 2>&1; then + echo " ✅ ljwx-admin: http://localhost:5173" +else + echo " ⚠️ ljwx-admin: 未运行" + echo " 启动命令: cd ljwx-admin && pnpm dev" +fi + +echo "" +echo "监控系统 (Docker):" +GRAFANA_PORT=$(docker port grafana | grep 3000 | cut -d':' -f2) +PROM_PORT=$(docker port prometheus | grep 9090 | cut -d':' -f2) +echo " 📊 Grafana: http://localhost:$GRAFANA_PORT (admin/admin123)" +echo " 📈 Prometheus: http://localhost:$PROM_PORT" +echo " 🎯 Targets: http://localhost:$PROM_PORT/targets" +echo " 🔔 Alerts: http://localhost:$PROM_PORT/alerts" + +echo "" +echo "基础设施 (Docker):" +MYSQL_PORT=$(docker port ljwx-mysql | grep 3306 | cut -d':' -f2) +REDIS_PORT=$(docker port ljwx-redis | grep 6379 | cut -d':' -f2) +echo " 🗄️ MySQL: localhost:$MYSQL_PORT (root/123456, db: test)" +echo " 💾 Redis: localhost:$REDIS_PORT (password: 123456, db: 1)" +echo " 📊 Redis Exporter: http://localhost:9121/metrics" + +echo "" +echo "==========================================" +echo "✅ 验证完成！" +echo "==========================================" +echo "" +echo "下一步:" +echo " 1. 如果ljwx-boot未运行，请启动:" +echo " cd ljwx-boot && ./run-local.sh" +echo "" +echo " 2. 如果ljwx-admin未运行，请启动:" +echo " cd ljwx-admin && pnpm dev" +echo "" +echo " 3. 导入Grafana Dashboard:" +echo " 访问 http://localhost:$GRAFANA_PORT" +echo " Dashboards → Import → 上传 ljwx-boot/monitoring/grafana-dashboard-p0.json" +echo "" +echo " 4. 运行性能压测:" +echo " ./scripts/performance-test.sh" +echo ""