392 lines
12 KiB
Bash
Executable File
392 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
# 可观测性增强组件验收脚本
|
||
# 验证 Postgres/Redis Exporter + Prometheus + Grafana + Loki ↔ Tempo 集成
|
||
|
||
set -euo pipefail
|
||
|
||
# 颜色输出
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# 配置
|
||
MONITORING_NS="monitoring"
|
||
INFRA_NS="infra"
|
||
|
||
# 计数器
|
||
PASSED=0
|
||
FAILED=0
|
||
WARNINGS=0
|
||
|
||
# 辅助函数
|
||
print_header() {
|
||
echo -e "\n${BLUE}========================================${NC}"
|
||
echo -e "${BLUE}$1${NC}"
|
||
echo -e "${BLUE}========================================${NC}\n"
|
||
}
|
||
|
||
print_test() {
|
||
echo -e "${YELLOW}[TEST]${NC} $1"
|
||
}
|
||
|
||
print_pass() {
|
||
echo -e "${GREEN}[PASS]${NC} $1"
|
||
((PASSED++))
|
||
}
|
||
|
||
print_fail() {
|
||
echo -e "${RED}[FAIL]${NC} $1"
|
||
((FAILED++))
|
||
}
|
||
|
||
print_warn() {
|
||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||
((WARNINGS++))
|
||
}
|
||
|
||
# 检查命令是否存在
|
||
check_command() {
|
||
if ! command -v "$1" &> /dev/null; then
|
||
print_fail "命令 $1 未安装"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
# 等待 Pod 就绪
|
||
wait_for_pod() {
|
||
local namespace=$1
|
||
local label=$2
|
||
local timeout=${3:-60}
|
||
|
||
print_test "等待 $label Pod 就绪..."
|
||
|
||
if kubectl wait --for=condition=ready pod -l "$label" -n "$namespace" --timeout="${timeout}s" &> /dev/null; then
|
||
print_pass "$label Pod 已就绪"
|
||
return 0
|
||
else
|
||
print_fail "$label Pod 未就绪"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# 检查 Service 是否存在
|
||
check_service() {
|
||
local namespace=$1
|
||
local service=$2
|
||
|
||
print_test "检查 Service $namespace/$service"
|
||
|
||
if kubectl get svc "$service" -n "$namespace" &> /dev/null; then
|
||
print_pass "Service $namespace/$service 存在"
|
||
return 0
|
||
else
|
||
print_fail "Service $namespace/$service 不存在"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# 检查 Prometheus Target
|
||
check_prometheus_target() {
|
||
local job=$1
|
||
|
||
print_test "检查 Prometheus Target: $job"
|
||
|
||
# 端口转发到 Prometheus
|
||
kubectl port-forward -n "$MONITORING_NS" svc/prometheus 9090:9090 &> /dev/null &
|
||
local PF_PID=$!
|
||
sleep 2
|
||
|
||
# 查询 targets
|
||
local result=$(curl -s http://localhost:9090/api/v1/targets | \
|
||
grep -o "\"job\":\"$job\"" | wc -l || echo "0")
|
||
|
||
kill $PF_PID 2>/dev/null || true
|
||
sleep 1
|
||
|
||
if [ "$result" -gt 0 ]; then
|
||
print_pass "Prometheus Target $job 存在"
|
||
return 0
|
||
else
|
||
print_warn "Prometheus Target $job 未找到（可能 exporter 未部署）"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# 检查 Metrics Endpoint
|
||
check_metrics_endpoint() {
|
||
local namespace=$1
|
||
local service=$2
|
||
local port=$3
|
||
local path=$4
|
||
|
||
print_test "检查 Metrics Endpoint: $namespace/$service:$port$path"
|
||
|
||
# 端口转发
|
||
kubectl port-forward -n "$namespace" "svc/$service" "$port:$port" &> /dev/null &
|
||
local PF_PID=$!
|
||
sleep 2
|
||
|
||
# 检查 endpoint
|
||
local status=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port$path" || echo "000")
|
||
|
||
kill $PF_PID 2>/dev/null || true
|
||
sleep 1
|
||
|
||
if [ "$status" = "200" ]; then
|
||
print_pass "Metrics Endpoint $namespace/$service 可访问"
|
||
return 0
|
||
else
|
||
print_fail "Metrics Endpoint $namespace/$service 返回 $status"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# 检查 Grafana Datasource
|
||
check_grafana_datasource() {
|
||
local datasource=$1
|
||
|
||
print_test "检查 Grafana Datasource: $datasource"
|
||
|
||
# 端口转发到 Grafana
|
||
kubectl port-forward -n "$MONITORING_NS" svc/grafana 3000:3000 &> /dev/null &
|
||
local PF_PID=$!
|
||
sleep 3
|
||
|
||
# 检查 datasource（使用默认密码）
|
||
local result=$(curl -s -u admin:admin \
|
||
"http://localhost:3000/api/datasources/name/$datasource" 2>/dev/null | \
|
||
grep -o "\"name\":\"$datasource\"" || echo "")
|
||
|
||
kill $PF_PID 2>/dev/null || true
|
||
sleep 1
|
||
|
||
if [ -n "$result" ]; then
|
||
print_pass "Grafana Datasource $datasource 已配置"
|
||
return 0
|
||
else
|
||
print_fail "Grafana Datasource $datasource 未配置"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# 主验收流程
|
||
main() {
|
||
print_header "可观测性增强组件验收"
|
||
|
||
# 0. 检查必要命令
|
||
print_header "1. 检查必要工具"
|
||
check_command kubectl
|
||
check_command curl
|
||
|
||
# 1. 检查核心组件
|
||
print_header "2. 检查核心可观测性组件"
|
||
|
||
wait_for_pod "$MONITORING_NS" "app=prometheus" || true
|
||
wait_for_pod "$MONITORING_NS" "app=grafana" || true
|
||
|
||
# 检查 Loki 和 Tempo（可能在不同 namespace）
|
||
if kubectl get ns logging &> /dev/null; then
|
||
wait_for_pod "logging" "app=loki" || true
|
||
fi
|
||
|
||
if kubectl get ns tracing &> /dev/null; then
|
||
wait_for_pod "tracing" "app.kubernetes.io/name=tempo" || true
|
||
fi
|
||
|
||
# 2. 检查 Exporter 部署
|
||
print_header "3. 检查 Exporter 部署状态"
|
||
|
||
# Postgres Exporter
|
||
if kubectl get deployment postgres-exporter -n "$MONITORING_NS" &> /dev/null; then
|
||
wait_for_pod "$MONITORING_NS" "app=postgres-exporter" || true
|
||
check_service "$MONITORING_NS" "postgres-exporter" || true
|
||
check_metrics_endpoint "$MONITORING_NS" "postgres-exporter" 9187 "/metrics" || true
|
||
else
|
||
print_warn "postgres-exporter 未部署（运行 kubectl apply -f infra/observability/postgres-exporter.yaml）"
|
||
fi
|
||
|
||
# Redis Exporter
|
||
if kubectl get deployment redis-exporter -n "$MONITORING_NS" &> /dev/null; then
|
||
wait_for_pod "$MONITORING_NS" "app=redis-exporter" || true
|
||
check_service "$MONITORING_NS" "redis-exporter" || true
|
||
check_metrics_endpoint "$MONITORING_NS" "redis-exporter" 9121 "/metrics" || true
|
||
else
|
||
print_warn "redis-exporter 未部署（运行 kubectl apply -f infra/observability/redis-exporter.yaml）"
|
||
fi
|
||
|
||
# 3. 检查 Prometheus Targets
|
||
print_header "4. 检查 Prometheus Scrape Targets"
|
||
|
||
check_prometheus_target "postgres-exporter" || true
|
||
check_prometheus_target "redis-exporter" || true
|
||
check_prometheus_target "node-exporter" || true
|
||
check_prometheus_target "kube-state-metrics" || true
|
||
|
||
# 4. 检查 Prometheus 告警规则
|
||
print_header "5. 检查 Prometheus 告警规则"
|
||
|
||
print_test "检查 Prometheus ConfigMap 中的告警规则配置"
|
||
|
||
if kubectl get configmap prometheus-config -n "$MONITORING_NS" -o yaml | grep -q "rule_files"; then
|
||
print_pass "Prometheus 配置包含告警规则"
|
||
else
|
||
print_warn "Prometheus 配置未包含告警规则（需要更新 ConfigMap）"
|
||
fi
|
||
|
||
# 检查 PrometheusRule CRD（如果使用 Prometheus Operator）
|
||
if kubectl get crd prometheusrules.monitoring.coreos.com &> /dev/null; then
|
||
local rule_count=$(kubectl get prometheusrule -n "$MONITORING_NS" 2>/dev/null | wc -l)
|
||
if [ "$rule_count" -gt 1 ]; then
|
||
print_pass "发现 $((rule_count - 1)) 个 PrometheusRule 资源"
|
||
else
|
||
print_warn "未发现 PrometheusRule 资源"
|
||
fi
|
||
fi
|
||
|
||
# 5. 检查 Grafana 数据源
|
||
print_header "6. 检查 Grafana 数据源配置"
|
||
|
||
check_grafana_datasource "Prometheus" || true
|
||
check_grafana_datasource "Loki" || true
|
||
check_grafana_datasource "Tempo" || true
|
||
|
||
# 6. 验证 Loki ↔ Tempo 关联
|
||
print_header "7. 验证 Loki ↔ Tempo 双向跳转配置"
|
||
|
||
print_test "检查 Grafana Loki datasource 的 derivedFields 配置"
|
||
|
||
if kubectl get configmap grafana-datasources -n "$MONITORING_NS" -o yaml | grep -q "derivedFields"; then
|
||
print_pass "Loki datasource 配置了 derivedFields（Log → Trace）"
|
||
else
|
||
print_warn "Loki datasource 未配置 derivedFields"
|
||
fi
|
||
|
||
print_test "检查 Grafana Tempo datasource 的 tracesToLogs 配置"
|
||
|
||
if kubectl get configmap grafana-datasources -n "$MONITORING_NS" -o yaml | grep -q "tracesToLogs"; then
|
||
print_pass "Tempo datasource 配置了 tracesToLogs（Trace → Log）"
|
||
else
|
||
print_warn "Tempo datasource 未配置 tracesToLogs"
|
||
fi
|
||
|
||
# 7. 检查 pg_stat_statements（如果 Postgres 存在）
|
||
print_header "8. 检查 PostgreSQL pg_stat_statements"
|
||
|
||
if kubectl get statefulset -n "$INFRA_NS" 2>/dev/null | grep -q postgres; then
|
||
print_test "检查 PostgreSQL 中是否启用 pg_stat_statements"
|
||
|
||
# 尝试在 postgres pod 中执行查询
|
||
local pg_check=$(kubectl exec -n "$INFRA_NS" postgres-0 -- \
|
||
psql -U postgres -tAc "SELECT count(*) FROM pg_stat_statements LIMIT 1" 2>/dev/null || echo "0")
|
||
|
||
if [ "$pg_check" -ge 0 ]; then
|
||
print_pass "pg_stat_statements 已启用"
|
||
else
|
||
print_warn "pg_stat_statements 未启用（参考 docs/observability/pg-stat-statements-setup.md）"
|
||
fi
|
||
else
|
||
print_warn "未发现 PostgreSQL 部署，跳过 pg_stat_statements 检查"
|
||
fi
|
||
|
||
# 8. 总结
|
||
print_header "验收总结"
|
||
|
||
echo -e "${GREEN}通过测试: $PASSED${NC}"
|
||
echo -e "${YELLOW}警告: $WARNINGS${NC}"
|
||
echo -e "${RED}失败测试: $FAILED${NC}"
|
||
|
||
echo ""
|
||
echo "================================"
|
||
echo "后续步骤："
|
||
echo "================================"
|
||
echo ""
|
||
|
||
if [ $WARNINGS -gt 0 ] || [ $FAILED -gt 0 ]; then
|
||
echo "⚠️ 检测到警告或失败，请按以下步骤修复："
|
||
echo ""
|
||
|
||
if kubectl get deployment postgres-exporter -n "$MONITORING_NS" &> /dev/null; then
|
||
:
|
||
else
|
||
echo "1. 部署 postgres-exporter:"
|
||
echo " kubectl apply -f infra/observability/postgres-secret-example.yaml"
|
||
echo " # 修改 Secret 中的连接串"
|
||
echo " kubectl apply -f infra/observability/postgres-exporter.yaml"
|
||
echo ""
|
||
fi
|
||
|
||
if kubectl get deployment redis-exporter -n "$MONITORING_NS" &> /dev/null; then
|
||
:
|
||
else
|
||
echo "2. 部署 redis-exporter:"
|
||
echo " kubectl apply -f infra/observability/redis-secret-example.yaml"
|
||
echo " # 修改 Secret 中的连接串"
|
||
echo " kubectl apply -f infra/observability/redis-exporter.yaml"
|
||
echo ""
|
||
fi
|
||
|
||
echo "3. 更新 Prometheus 配置:"
|
||
echo " kubectl apply -f monitoring/prometheus/prometheus-configmap-enhanced.yaml"
|
||
echo " kubectl rollout restart deployment/prometheus -n monitoring"
|
||
echo ""
|
||
|
||
echo "4. 部署告警规则:"
|
||
echo " kubectl create configmap prometheus-rules \\"
|
||
echo " --from-file=monitoring/prometheus/rules/ \\"
|
||
echo " -n monitoring --dry-run=client -o yaml | kubectl apply -f -"
|
||
echo ""
|
||
|
||
echo "5. 更新 Grafana 数据源:"
|
||
echo " kubectl apply -f monitoring/grafana/datasources-loki-tempo-enhanced.yaml"
|
||
echo " kubectl rollout restart deployment/grafana -n monitoring"
|
||
echo ""
|
||
|
||
echo "6. 导入 Grafana Dashboards:"
|
||
echo " cd monitoring/grafana/dashboards"
|
||
echo " ./import-dashboards.sh"
|
||
echo ""
|
||
else
|
||
echo "✅ 所有组件验收通过！"
|
||
echo ""
|
||
echo "可以开始使用："
|
||
echo ""
|
||
echo "1. 访问 Grafana:"
|
||
echo " kubectl port-forward -n monitoring svc/grafana 3000:3000"
|
||
echo " 浏览器打开: http://localhost:3000"
|
||
echo ""
|
||
echo "2. 查看 Prometheus Targets:"
|
||
echo " kubectl port-forward -n monitoring svc/prometheus 9090:9090"
|
||
echo " 浏览器打开: http://localhost:9090/targets"
|
||
echo ""
|
||
echo "3. 验证 Loki → Tempo 跳转:"
|
||
echo " - 在 Grafana Explore 中选择 Loki"
|
||
echo " - 查询包含 trace_id 的日志"
|
||
echo " - 点击日志中的 'View Trace' 链接"
|
||
echo ""
|
||
echo "4. 验证 Tempo → Loki 跳转:"
|
||
echo " - 在 Grafana Explore 中选择 Tempo"
|
||
echo " - 查询一个 trace"
|
||
echo " - 点击 span 上的 'Logs for this span' 按钮"
|
||
echo ""
|
||
fi
|
||
|
||
echo ""
|
||
echo "📚 参考文档:"
|
||
echo " - 业务应用集成: docs/observability/business-instrumentation-*.md"
|
||
echo " - pg_stat_statements: docs/observability/pg-stat-statements-setup.md"
|
||
echo " - Dashboard 导入: monitoring/grafana/dashboards/README.md"
|
||
echo ""
|
||
|
||
# 返回状态码
|
||
if [ $FAILED -gt 0 ]; then
|
||
exit 1
|
||
else
|
||
exit 0
|
||
fi
|
||
}
|
||
|
||
# 执行主函数
|
||
main "$@"
|