363 lines
11 KiB
Bash
Executable File
363 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
set -e
|
||
|
||
echo "======================================"
|
||
echo "可观测性 P0 任务完整验收"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
# 颜色定义
|
||
GREEN='\033[0;32m'
|
||
RED='\033[0;31m'
|
||
YELLOW='\033[1;33m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# 计数器
|
||
TOTAL_CHECKS=0
|
||
PASSED_CHECKS=0
|
||
FAILED_CHECKS=0
|
||
|
||
check_result() {
|
||
TOTAL_CHECKS=$((TOTAL_CHECKS + 1))
|
||
if [ $1 -eq 0 ]; then
|
||
echo -e " ${GREEN}✅ $2${NC}"
|
||
PASSED_CHECKS=$((PASSED_CHECKS + 1))
|
||
else
|
||
echo -e " ${RED}❌ $2${NC}"
|
||
FAILED_CHECKS=$((FAILED_CHECKS + 1))
|
||
fi
|
||
}
|
||
|
||
echo "======================================"
|
||
echo "T1: Prometheus Scrape Configs 验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "1. 检查 Prometheus ConfigMap..."
|
||
kubectl get configmap prometheus-config -n monitoring > /dev/null 2>&1
|
||
check_result $? "prometheus-config ConfigMap 存在"
|
||
|
||
kubectl get configmap prometheus-config -n monitoring -o yaml | grep -q "postgres-exporter"
|
||
check_result $? "配置包含 postgres-exporter job"
|
||
|
||
kubectl get configmap prometheus-config -n monitoring -o yaml | grep -q "redis-exporter"
|
||
check_result $? "配置包含 redis-exporter job"
|
||
|
||
kubectl get configmap prometheus-config -n monitoring -o yaml | grep -q "business-services"
|
||
check_result $? "配置包含 business-services job"
|
||
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "T2: Exporter 部署验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "2. 检查 Exporter Pods..."
|
||
kubectl get pods -n data | grep -q "postgres.*metrics.*Running"
|
||
check_result $? "Postgres Exporter Pod 运行中"
|
||
|
||
kubectl get pods -n data | grep -q "redis.*metrics.*Running"
|
||
check_result $? "Redis Exporter Pod 运行中"
|
||
|
||
echo ""
|
||
echo "3. 检查 Exporter Services..."
|
||
kubectl get svc postgres-postgresql-metrics -n data > /dev/null 2>&1
|
||
check_result $? "Postgres Exporter Service 存在"
|
||
|
||
kubectl get svc redis-metrics -n data > /dev/null 2>&1
|
||
check_result $? "Redis Exporter Service 存在"
|
||
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "T3: Grafana Dashboard 验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "4. 检查 Grafana Pod..."
|
||
kubectl get pods -n monitoring -l app=grafana | grep -q "Running"
|
||
check_result $? "Grafana Pod 运行中"
|
||
|
||
echo ""
|
||
echo "5. 启动端口转发（如需）..."
|
||
# 检查端口是否已被占用
|
||
if lsof -Pi :3000 -sTCP:LISTEN -t >/dev/null 2>&1; then
|
||
echo " ℹ️ 端口 3000 已被使用，跳过端口转发"
|
||
else
|
||
kubectl port-forward -n monitoring svc/grafana 3000:3000 > /tmp/grafana-pf-verify.log 2>&1 &
|
||
GRAFANA_PF_PID=$!
|
||
sleep 5
|
||
echo " ✅ Grafana 端口转发已启动 (PID: $GRAFANA_PF_PID)"
|
||
fi
|
||
|
||
echo ""
|
||
echo "6. 验证 Grafana 可访问..."
|
||
GRAFANA_STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000/api/health)
|
||
if [ "$GRAFANA_STATUS" = "200" ]; then
|
||
check_result 0 "Grafana API 可访问 (HTTP 200)"
|
||
else
|
||
check_result 1 "Grafana API 不可访问 (HTTP $GRAFANA_STATUS)"
|
||
fi
|
||
|
||
echo ""
|
||
echo "7. 验证 Dashboards..."
|
||
DASHBOARDS=$(curl -s -u admin:admin123 http://localhost:3000/api/search?type=dash-db | jq -r '.[].title' 2>/dev/null)
|
||
|
||
if echo "$DASHBOARDS" | grep -q "PostgreSQL"; then
|
||
check_result 0 "PostgreSQL Dashboard 已导入"
|
||
else
|
||
check_result 1 "PostgreSQL Dashboard 未找到"
|
||
fi
|
||
|
||
if echo "$DASHBOARDS" | grep -q "Redis"; then
|
||
check_result 0 "Redis Dashboard 已导入"
|
||
else
|
||
check_result 1 "Redis Dashboard 未找到"
|
||
fi
|
||
|
||
if echo "$DASHBOARDS" | grep -q "Kubernetes"; then
|
||
check_result 0 "Kubernetes Dashboard 已导入"
|
||
else
|
||
check_result 1 "Kubernetes Dashboard 未找到"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "T4: Loki ↔ Tempo 关联验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "8. 检查 Loki 数据源..."
|
||
LOKI_DS=$(curl -s -u admin:admin123 http://localhost:3000/api/datasources/name/Loki 2>/dev/null)
|
||
|
||
if echo "$LOKI_DS" | jq -e '.jsonData.derivedFields' > /dev/null 2>&1; then
|
||
DERIVED_FIELDS_COUNT=$(echo "$LOKI_DS" | jq '.jsonData.derivedFields | length')
|
||
if [ "$DERIVED_FIELDS_COUNT" -ge 1 ]; then
|
||
check_result 0 "Loki Derived Fields 已配置 ($DERIVED_FIELDS_COUNT 个)"
|
||
else
|
||
check_result 1 "Loki Derived Fields 未配置"
|
||
fi
|
||
else
|
||
check_result 1 "Loki Derived Fields 配置未找到"
|
||
fi
|
||
|
||
echo ""
|
||
echo "9. 检查 Tempo 数据源..."
|
||
TEMPO_DS=$(curl -s -u admin:admin123 http://localhost:3000/api/datasources/name/Tempo 2>/dev/null)
|
||
|
||
if echo "$TEMPO_DS" | jq -e '.jsonData.tracesToLogs' > /dev/null 2>&1; then
|
||
check_result 0 "Tempo tracesToLogs 已配置"
|
||
else
|
||
check_result 1 "Tempo tracesToLogs 未配置"
|
||
fi
|
||
|
||
if echo "$TEMPO_DS" | jq -e '.jsonData.tracesToMetrics' > /dev/null 2>&1; then
|
||
check_result 0 "Tempo tracesToMetrics 已配置"
|
||
else
|
||
check_result 1 "Tempo tracesToMetrics 未配置"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "T5 & T6: 业务应用集成文档验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "10. 检查集成文档..."
|
||
if [ -f "docs/observability/business-instrumentation-spring-boot.md" ]; then
|
||
check_result 0 "Spring Boot 集成文档存在"
|
||
else
|
||
check_result 1 "Spring Boot 集成文档未找到"
|
||
fi
|
||
|
||
if [ -f "docs/observability/business-instrumentation-python.md" ]; then
|
||
check_result 0 "Python 集成文档存在"
|
||
else
|
||
check_result 1 "Python 集成文档未找到"
|
||
fi
|
||
|
||
if [ -f "docs/observability/pg-stat-statements-setup.md" ]; then
|
||
check_result 0 "pg_stat_statements 设置文档存在"
|
||
else
|
||
check_result 1 "pg_stat_statements 设置文档未找到"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "Prometheus Targets & Metrics 验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "11. 启动 Prometheus 端口转发..."
|
||
if lsof -Pi :9090 -sTCP:LISTEN -t >/dev/null 2>&1; then
|
||
echo " ℹ️ 端口 9090 已被使用，跳过端口转发"
|
||
else
|
||
kubectl port-forward -n monitoring svc/prometheus 9090:9090 > /tmp/prometheus-pf-verify.log 2>&1 &
|
||
PROMETHEUS_PF_PID=$!
|
||
sleep 5
|
||
echo " ✅ Prometheus 端口转发已启动 (PID: $PROMETHEUS_PF_PID)"
|
||
fi
|
||
|
||
echo ""
|
||
echo "12. 验证 Prometheus Targets..."
|
||
|
||
# Postgres Exporter
|
||
PG_TARGET=$(curl -s http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | select(.labels.job=="postgres-exporter") | .health' | head -1)
|
||
if [ "$PG_TARGET" = "up" ]; then
|
||
check_result 0 "Postgres Exporter Target UP"
|
||
else
|
||
check_result 1 "Postgres Exporter Target 不正常 ($PG_TARGET)"
|
||
fi
|
||
|
||
# Redis Exporter
|
||
REDIS_TARGET=$(curl -s http://localhost:9090/api/v1/targets | jq -r '.data.activeTargets[] | select(.labels.job=="redis-exporter") | .health' | head -1)
|
||
if [ "$REDIS_TARGET" = "up" ]; then
|
||
check_result 0 "Redis Exporter Target UP"
|
||
else
|
||
check_result 1 "Redis Exporter Target 不正常 ($REDIS_TARGET)"
|
||
fi
|
||
|
||
echo ""
|
||
echo "13. 验证 Prometheus Metrics..."
|
||
|
||
# pg_up
|
||
PG_UP=$(curl -s 'http://localhost:9090/api/v1/query?query=pg_up' | jq -r '.data.result | length')
|
||
if [ "$PG_UP" -gt 0 ]; then
|
||
check_result 0 "pg_up 指标存在 ($PG_UP 个实例)"
|
||
else
|
||
check_result 1 "pg_up 指标未找到"
|
||
fi
|
||
|
||
# redis_up
|
||
REDIS_UP=$(curl -s 'http://localhost:9090/api/v1/query?query=redis_up' | jq -r '.data.result | length')
|
||
if [ "$REDIS_UP" -gt 0 ]; then
|
||
check_result 0 "redis_up 指标存在 ($REDIS_UP 个实例)"
|
||
else
|
||
check_result 1 "redis_up 指标未找到"
|
||
fi
|
||
|
||
# kube_pod_info
|
||
K8S_PODS=$(curl -s 'http://localhost:9090/api/v1/query?query=kube_pod_info' | jq -r '.data.result | length')
|
||
if [ "$K8S_PODS" -gt 0 ]; then
|
||
check_result 0 "kube_pod_info 指标存在 ($K8S_PODS 个 Pod)"
|
||
else
|
||
check_result 1 "kube_pod_info 指标未找到"
|
||
fi
|
||
|
||
echo ""
|
||
echo "14. 验证关键 Postgres 指标..."
|
||
|
||
PG_CONNECTIONS=$(curl -s 'http://localhost:9090/api/v1/query?query=pg_stat_database_numbackends' | jq -r '.data.result | length')
|
||
if [ "$PG_CONNECTIONS" -gt 0 ]; then
|
||
check_result 0 "pg_stat_database_numbackends (连接数) 存在"
|
||
else
|
||
check_result 1 "pg_stat_database_numbackends 未找到"
|
||
fi
|
||
|
||
PG_TRANSACTIONS=$(curl -s 'http://localhost:9090/api/v1/query?query=pg_stat_database_xact_commit' | jq -r '.data.result | length')
|
||
if [ "$PG_TRANSACTIONS" -gt 0 ]; then
|
||
check_result 0 "pg_stat_database_xact_commit (事务数) 存在"
|
||
else
|
||
check_result 1 "pg_stat_database_xact_commit 未找到"
|
||
fi
|
||
|
||
echo ""
|
||
echo "15. 验证关键 Redis 指标..."
|
||
|
||
REDIS_MEMORY=$(curl -s 'http://localhost:9090/api/v1/query?query=redis_memory_used_bytes' | jq -r '.data.result | length')
|
||
if [ "$REDIS_MEMORY" -gt 0 ]; then
|
||
check_result 0 "redis_memory_used_bytes (内存使用) 存在"
|
||
else
|
||
check_result 1 "redis_memory_used_bytes 未找到"
|
||
fi
|
||
|
||
REDIS_CLIENTS=$(curl -s 'http://localhost:9090/api/v1/query?query=redis_connected_clients' | jq -r '.data.result | length')
|
||
if [ "$REDIS_CLIENTS" -gt 0 ]; then
|
||
check_result 0 "redis_connected_clients (连接数) 存在"
|
||
else
|
||
check_result 1 "redis_connected_clients 未找到"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "告警规则验证"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
echo "16. 检查告警规则 ConfigMap..."
|
||
kubectl get configmap prometheus-rules -n monitoring > /dev/null 2>&1
|
||
check_result $? "prometheus-rules ConfigMap 存在"
|
||
|
||
RULES_FILES=$(kubectl get configmap prometheus-rules -n monitoring -o jsonpath='{.data}' | grep -o '\.yml' | wc -l | tr -d ' ')
|
||
echo " 发现 $RULES_FILES 个规则文件"
|
||
|
||
echo ""
|
||
echo "17. 检查 Prometheus 规则加载状态..."
|
||
RULES_COUNT=$(curl -s http://localhost:9090/api/v1/rules | jq -r '.data.groups | length' 2>/dev/null || echo "0")
|
||
|
||
if [ "$RULES_COUNT" -gt 0 ]; then
|
||
check_result 0 "Prometheus 已加载 $RULES_COUNT 个规则组"
|
||
else
|
||
echo -e " ${YELLOW}⚠️ Prometheus 未加载告警规则（需运行 enable-prometheus-alerts.sh）${NC}"
|
||
TOTAL_CHECKS=$((TOTAL_CHECKS + 1))
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# 清理端口转发
|
||
if [ -n "$GRAFANA_PF_PID" ]; then
|
||
kill $GRAFANA_PF_PID 2>/dev/null || true
|
||
fi
|
||
if [ -n "$PROMETHEUS_PF_PID" ]; then
|
||
kill $PROMETHEUS_PF_PID 2>/dev/null || true
|
||
fi
|
||
|
||
echo "======================================"
|
||
echo "验收结果汇总"
|
||
echo "======================================"
|
||
echo ""
|
||
echo "总检查项: $TOTAL_CHECKS"
|
||
echo -e "${GREEN}通过: $PASSED_CHECKS${NC}"
|
||
echo -e "${RED}失败: $FAILED_CHECKS${NC}"
|
||
echo ""
|
||
|
||
PASS_RATE=$((PASSED_CHECKS * 100 / TOTAL_CHECKS))
|
||
echo "通过率: $PASS_RATE%"
|
||
echo ""
|
||
|
||
if [ $FAILED_CHECKS -eq 0 ]; then
|
||
echo -e "${GREEN}======================================"
|
||
echo "✅ 所有检查通过！P0 任务完成！"
|
||
echo "======================================${NC}"
|
||
echo ""
|
||
echo "🎉 可观测性监控系统已就绪！"
|
||
echo ""
|
||
echo "快速访问:"
|
||
echo "- Grafana: http://grafana.omniverseai.net:31080 (admin/admin123)"
|
||
echo "- Prometheus: http://localhost:9090"
|
||
echo ""
|
||
echo "Dashboards:"
|
||
echo "- PostgreSQL: http://grafana.omniverseai.net:31080/d/000000039/postgresql-database"
|
||
echo "- Redis: http://grafana.omniverseai.net:31080/d/xDLNRKUWz/4b82829"
|
||
echo "- Kubernetes: http://grafana.omniverseai.net:31080/d/k8s_views_pods/kubernetes-views-pods"
|
||
echo ""
|
||
exit 0
|
||
else
|
||
echo -e "${YELLOW}======================================"
|
||
echo "⚠️ 部分检查未通过"
|
||
echo "======================================${NC}"
|
||
echo ""
|
||
echo "建议操作:"
|
||
echo "1. 检查失败项的详细日志"
|
||
echo "2. 参考 OBSERVABILITY-P0-COMPLETE.md 进行排查"
|
||
echo "3. 如需启用告警规则，运行: ./enable-prometheus-alerts.sh"
|
||
echo ""
|
||
exit 1
|
||
fi
|