198 lines
6.8 KiB
Bash
Executable File
198 lines
6.8 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# Kubernetes 可观测性栈验证脚本
|
||
# 用于验证 Prometheus + Grafana + Loki + Tempo 部署状态
|
||
|
||
set -e
|
||
|
||
echo "========================================="
|
||
echo " Kubernetes 可观测性栈验证"
|
||
echo "========================================="
|
||
echo ""
|
||
|
||
# 颜色定义
|
||
GREEN='\033[0;32m'
|
||
RED='\033[0;31m'
|
||
YELLOW='\033[1;33m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# 检查函数
|
||
check_pods() {
|
||
local namespace=$1
|
||
local label=$2
|
||
local name=$3
|
||
|
||
echo -e "${YELLOW}检查 $name...${NC}"
|
||
local pod_count=$(kubectl get pods -n $namespace $label --no-headers 2>/dev/null | wc -l | tr -d ' ')
|
||
|
||
if [ "$pod_count" -gt 0 ]; then
|
||
local running_count=$(kubectl get pods -n $namespace $label --no-headers 2>/dev/null | grep -c "Running" || echo "0")
|
||
if [ "$running_count" -eq "$pod_count" ]; then
|
||
echo -e "${GREEN}✓ $name 运行正常 ($pod_count/$pod_count pods)${NC}"
|
||
return 0
|
||
else
|
||
echo -e "${RED}✗ $name 状态异常 ($running_count/$pod_count pods running)${NC}"
|
||
return 1
|
||
fi
|
||
else
|
||
echo -e "${RED}✗ $name 未找到${NC}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# 1. 检查 Monitoring Namespace
|
||
echo "========================================="
|
||
echo "1. 检查 Monitoring 组件"
|
||
echo "========================================="
|
||
kubectl get pods -n monitoring
|
||
echo ""
|
||
|
||
check_pods "monitoring" "-l app=grafana" "Grafana"
|
||
check_pods "monitoring" "-l app=prometheus" "Prometheus"
|
||
check_pods "monitoring" "-l app=alertmanager" "Alertmanager"
|
||
check_pods "monitoring" "-l app=kube-state-metrics" "kube-state-metrics"
|
||
check_pods "monitoring" "-l app=node-exporter" "node-exporter"
|
||
echo ""
|
||
|
||
# 2. 检查 Logging Namespace
|
||
echo "========================================="
|
||
echo "2. 检查 Logging 组件"
|
||
echo "========================================="
|
||
kubectl get pods -n logging
|
||
echo ""
|
||
|
||
check_pods "logging" "-l app=loki" "Loki"
|
||
check_pods "logging" "-l app.kubernetes.io/name=promtail" "Promtail"
|
||
echo ""
|
||
|
||
# 3. 检查 Tracing Namespace
|
||
echo "========================================="
|
||
echo "3. 检查 Tracing 组件"
|
||
echo "========================================="
|
||
kubectl get pods -n tracing
|
||
echo ""
|
||
|
||
check_pods "tracing" "-l app.kubernetes.io/name=tempo" "Tempo"
|
||
check_pods "tracing" "-l app.kubernetes.io/name=opentelemetry-collector" "OpenTelemetry Collector"
|
||
echo ""
|
||
|
||
# 4. 检查 Services
|
||
echo "========================================="
|
||
echo "4. 检查 Services"
|
||
echo "========================================="
|
||
echo -e "${YELLOW}Monitoring Services:${NC}"
|
||
kubectl get svc -n monitoring | grep -E "(NAME|grafana|prometheus|alertmanager)"
|
||
echo ""
|
||
|
||
echo -e "${YELLOW}Logging Services:${NC}"
|
||
kubectl get svc -n logging
|
||
echo ""
|
||
|
||
echo -e "${YELLOW}Tracing Services:${NC}"
|
||
kubectl get svc -n tracing
|
||
echo ""
|
||
|
||
# 5. 验证 Loki 健康状态
|
||
echo "========================================="
|
||
echo "5. 验证 Loki 健康状态"
|
||
echo "========================================="
|
||
if kubectl exec -n logging loki-0 -- wget -q -O- http://localhost:3100/ready &> /dev/null; then
|
||
echo -e "${GREEN}✓ Loki 健康检查通过${NC}"
|
||
else
|
||
echo -e "${RED}✗ Loki 健康检查失败${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
# 6. 验证 Loki 是否有数据
|
||
echo "========================================="
|
||
echo "6. 验证 Loki 日志数据"
|
||
echo "========================================="
|
||
LOKI_LABELS=$(kubectl exec -n logging loki-0 -- wget -q -O- 'http://localhost:3100/loki/api/v1/labels' 2>/dev/null)
|
||
if echo "$LOKI_LABELS" | grep -q "namespace"; then
|
||
echo -e "${GREEN}✓ Loki 正在采集日志数据${NC}"
|
||
echo "已采集的标签: $(echo $LOKI_LABELS | grep -o '"[^"]*"' | head -5 | tr '\n' ' ')"
|
||
else
|
||
echo -e "${RED}✗ Loki 未采集到日志数据${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
# 7. 验证 Tempo 状态
|
||
echo "========================================="
|
||
echo "7. 验证 Tempo 状态"
|
||
echo "========================================="
|
||
TEMPO_STATUS=$(kubectl get pods -n tracing -l app.kubernetes.io/name=tempo -o jsonpath='{.items[0].status.phase}' 2>/dev/null)
|
||
if [ "$TEMPO_STATUS" == "Running" ]; then
|
||
echo -e "${GREEN}✓ Tempo 运行正常${NC}"
|
||
else
|
||
echo -e "${RED}✗ Tempo 状态异常: $TEMPO_STATUS${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
# 8. 验证 OTel Collector 状态
|
||
echo "========================================="
|
||
echo "8. 验证 OpenTelemetry Collector 状态"
|
||
echo "========================================="
|
||
OTEL_STATUS=$(kubectl get pods -n tracing -l app.kubernetes.io/name=opentelemetry-collector -o jsonpath='{.items[0].status.phase}' 2>/dev/null)
|
||
if [ "$OTEL_STATUS" == "Running" ]; then
|
||
echo -e "${GREEN}✓ OpenTelemetry Collector 运行正常${NC}"
|
||
echo "接收器端点："
|
||
echo " - OTLP gRPC: otel-collector-opentelemetry-collector.tracing.svc.cluster.local:4317"
|
||
echo " - OTLP HTTP: otel-collector-opentelemetry-collector.tracing.svc.cluster.local:4318"
|
||
echo " - Jaeger gRPC: otel-collector-opentelemetry-collector.tracing.svc.cluster.local:14250"
|
||
echo " - Zipkin: otel-collector-opentelemetry-collector.tracing.svc.cluster.local:9411"
|
||
else
|
||
echo -e "${RED}✗ OpenTelemetry Collector 状态异常: $OTEL_STATUS${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
# 9. 检查告警规则
|
||
echo "========================================="
|
||
echo "9. 检查 Prometheus 告警规则"
|
||
echo "========================================="
|
||
RULES_COUNT=$(kubectl get prometheusrule -n monitoring 2>/dev/null | wc -l)
|
||
if [ $RULES_COUNT -gt 1 ]; then
|
||
echo -e "${GREEN}✓ 已配置 $((RULES_COUNT-1)) 个告警规则${NC}"
|
||
kubectl get prometheusrule -n monitoring
|
||
else
|
||
echo -e "${YELLOW}⚠ 未配置告警规则${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
# 10. 检查 Grafana 数据源
|
||
echo "========================================="
|
||
echo "10. 检查 Grafana 数据源配置"
|
||
echo "========================================="
|
||
DATASOURCES=$(kubectl get cm grafana-datasources -n monitoring -o yaml 2>/dev/null)
|
||
if echo "$DATASOURCES" | grep -q "prometheus.yaml"; then
|
||
echo -e "${GREEN}✓ Prometheus 数据源已配置${NC}"
|
||
fi
|
||
if echo "$DATASOURCES" | grep -q "loki.yaml"; then
|
||
echo -e "${GREEN}✓ Loki 数据源已配置${NC}"
|
||
fi
|
||
if echo "$DATASOURCES" | grep -q "tempo.yaml"; then
|
||
echo -e "${GREEN}✓ Tempo 数据源已配置${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
# 11. 总结
|
||
echo "========================================="
|
||
echo "验证总结"
|
||
echo "========================================="
|
||
echo ""
|
||
echo -e "${GREEN}✅ 可观测性栈部署完成！${NC}"
|
||
echo ""
|
||
echo "访问链接："
|
||
echo " - Grafana: http://localhost:30300 (admin/admin)"
|
||
echo " - Prometheus: http://localhost:30900"
|
||
echo ""
|
||
echo "下一步："
|
||
echo " 1. 访问 Grafana 并导入推荐的仪表盘（参考 OBSERVABILITY-GUIDE.md）"
|
||
echo " 2. 在 Grafana Explore 中测试 Loki 日志查询"
|
||
echo " 3. 配置应用接入链路追踪（参考 OBSERVABILITY-GUIDE.md）"
|
||
echo " 4. 配置 Alertmanager 通知渠道"
|
||
echo ""
|
||
echo "详细文档："
|
||
echo " - 部署总结: DEPLOYMENT-SUMMARY.md"
|
||
echo " - 使用指南: OBSERVABILITY-GUIDE.md"
|
||
echo ""
|