302 lines
9.0 KiB
Bash
Executable File
302 lines
9.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# Kubernetes 集群快速修复脚本
|
|
# 用途: 修复镜像拉取失败和优化集群性能
|
|
# 生成时间: 2026-01-04
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# 颜色定义
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# 日志函数
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
# 检查必要工具
|
|
check_prerequisites() {
|
|
log_info "检查必要工具..."
|
|
|
|
if ! command -v kubectl &> /dev/null; then
|
|
log_error "kubectl 未安装，请先安装 kubectl"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "docker 未安装，请先安装 Docker"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "./cache-image.sh" ]; then
|
|
log_error "cache-image.sh 未找到，请确保在正确的目录下运行此脚本"
|
|
exit 1
|
|
fi
|
|
|
|
log_success "工具检查通过"
|
|
}
|
|
|
|
# 显示菜单
|
|
show_menu() {
|
|
echo ""
|
|
echo "======================================"
|
|
echo " Kubernetes 集群快速修复菜单"
|
|
echo "======================================"
|
|
echo "1. 🔥 紧急修复: 修复 Grafana 组件镜像拉取失败 (loki, tempo)"
|
|
echo "2. 📦 批量缓存: 缓存所有 P1 高优先级镜像 (40+ 个)"
|
|
echo "3. 🧹 清理: 删除 dev 命名空间失败的 Deployment"
|
|
echo "4. 📊 诊断: 查看所有失败和异常的 Pod"
|
|
echo "5. 🔍 检查: 查看镜像拉取错误事件"
|
|
echo "6. ⚡ 全自动: 执行所有修复步骤 (1 + 2)"
|
|
echo "7. 📈 部署: 安装 Metrics Server"
|
|
echo "0. 退出"
|
|
echo "======================================"
|
|
}
|
|
|
|
# P0: 修复 Grafana 组件
|
|
fix_grafana_images() {
|
|
log_info "开始修复 Grafana 组件镜像拉取问题..."
|
|
|
|
# 缓存 Loki 镜像
|
|
log_info "缓存 Loki 镜像..."
|
|
./cache-image.sh grafana/loki:latest
|
|
./cache-image.sh grafana/loki:2.6.1
|
|
|
|
# 缓存 Tempo 镜像
|
|
log_info "缓存 Tempo 镜像..."
|
|
./cache-image.sh grafana/tempo:latest
|
|
./cache-image.sh grafana/tempo:2.9.0
|
|
|
|
# 缓存 Promtail 镜像
|
|
log_info "缓存 Promtail 镜像..."
|
|
./cache-image.sh grafana/promtail:3.5.1
|
|
|
|
# 重启 Pod
|
|
log_info "重启 Loki 和 Tempo Pod..."
|
|
kubectl delete pod -n infra loki-0 2>/dev/null || log_warning "infra/loki-0 不存在或已删除"
|
|
kubectl delete pod -n infra tempo-0 2>/dev/null || log_warning "infra/tempo-0 不存在或已删除"
|
|
kubectl delete pod -n logging loki-0 2>/dev/null || log_warning "logging/loki-0 不存在或已删除"
|
|
kubectl delete pod -n tracing tempo-0 2>/dev/null || log_warning "tracing/tempo-0 不存在或已删除"
|
|
|
|
log_success "Grafana 组件镜像修复完成！"
|
|
log_info "等待 30 秒让 Pod 重新启动..."
|
|
sleep 30
|
|
|
|
# 检查状态
|
|
log_info "检查 Pod 状态..."
|
|
kubectl get pods -n infra -l app=loki 2>/dev/null || true
|
|
kubectl get pods -n infra -l app=tempo 2>/dev/null || true
|
|
kubectl get pods -n logging -l app=loki 2>/dev/null || true
|
|
kubectl get pods -n tracing -l app.kubernetes.io/name=tempo 2>/dev/null || true
|
|
}
|
|
|
|
# P1: 批量缓存镜像
|
|
cache_p1_images() {
|
|
log_info "开始批量缓存 P1 高优先级镜像..."
|
|
|
|
if [ ! -f "images-to-cache-p1.txt" ]; then
|
|
log_error "镜像列表文件 images-to-cache-p1.txt 未找到"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "镜像列表文件: images-to-cache-p1.txt"
|
|
log_warning "这将花费 5-10 分钟，请耐心等待..."
|
|
|
|
./cache-image.sh -f images-to-cache-p1.txt
|
|
|
|
log_success "P1 镜像批量缓存完成！"
|
|
}
|
|
|
|
# 清理 dev 命名空间
|
|
clean_dev_namespace() {
|
|
log_info "清理 dev 命名空间失败的 Deployment..."
|
|
|
|
log_warning "将要删除以下 Deployment:"
|
|
echo " - dev/backend"
|
|
echo " - dev/frontend"
|
|
echo " - dev/quality-dashboard"
|
|
|
|
read -p "确认删除? (y/N): " -n 1 -r
|
|
echo
|
|
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
|
kubectl delete deployment -n dev backend 2>/dev/null || log_warning "backend 不存在"
|
|
kubectl delete deployment -n dev frontend 2>/dev/null || log_warning "frontend 不存在"
|
|
kubectl delete deployment -n dev quality-dashboard 2>/dev/null || log_warning "quality-dashboard 不存在"
|
|
log_success "dev 命名空间清理完成"
|
|
else
|
|
log_info "已取消删除操作"
|
|
fi
|
|
}
|
|
|
|
# 诊断失败的 Pod
|
|
diagnose_failed_pods() {
|
|
log_info "诊断所有失败和异常的 Pod..."
|
|
|
|
echo ""
|
|
echo "=== ImagePullBackOff / ErrImagePull ==="
|
|
kubectl get pods --all-namespaces --field-selector=status.phase=Pending -o wide 2>/dev/null || log_warning "没有 Pending 状态的 Pod"
|
|
|
|
echo ""
|
|
echo "=== CrashLoopBackOff ==="
|
|
kubectl get pods --all-namespaces -o json 2>/dev/null | \
|
|
jq -r '.items[] | select(.status.containerStatuses[]? | .state.waiting?.reason == "CrashLoopBackOff") | "\(.metadata.namespace)\t\(.metadata.name)\t\(.status.containerStatuses[0].restartCount)"' || \
|
|
log_warning "jq 未安装或没有 CrashLoopBackOff Pod"
|
|
|
|
echo ""
|
|
echo "=== 高重启次数的 Pod (>100) ==="
|
|
kubectl get pods --all-namespaces -o json 2>/dev/null | \
|
|
jq -r '.items[] | select(.status.containerStatuses[]? | .restartCount > 100) | "\(.metadata.namespace)\t\(.metadata.name)\t\(.status.containerStatuses[0].restartCount)"' || \
|
|
log_warning "jq 未安装或没有高重启次数的 Pod"
|
|
}
|
|
|
|
# 检查镜像拉取错误
|
|
check_image_pull_errors() {
|
|
log_info "检查最近的镜像拉取错误事件..."
|
|
|
|
kubectl get events --all-namespaces --field-selector type=Warning 2>/dev/null | grep -i "image\|pull" | head -20 || \
|
|
log_success "没有发现镜像拉取错误事件"
|
|
}
|
|
|
|
# 全自动修复
|
|
auto_fix_all() {
|
|
log_info "开始执行全自动修复流程..."
|
|
|
|
# 1. 修复 Grafana 组件
|
|
fix_grafana_images
|
|
|
|
echo ""
|
|
|
|
# 2. 批量缓存镜像
|
|
cache_p1_images
|
|
|
|
echo ""
|
|
log_success "全自动修复完成！"
|
|
|
|
# 最终诊断
|
|
echo ""
|
|
log_info "执行最终诊断..."
|
|
diagnose_failed_pods
|
|
}
|
|
|
|
# 部署 Metrics Server
|
|
deploy_metrics_server() {
|
|
log_info "部署 Metrics Server..."
|
|
|
|
log_info "下载 Metrics Server manifests..."
|
|
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
|
|
|
|
log_info "配置 OrbStack 兼容性 (--kubelet-insecure-tls)..."
|
|
kubectl patch deployment metrics-server -n kube-system --type='json' \
|
|
-p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]' 2>/dev/null || \
|
|
log_warning "Patch 失败，可能 Metrics Server 已配置"
|
|
|
|
log_info "等待 Metrics Server 启动..."
|
|
sleep 20
|
|
|
|
log_info "验证 Metrics Server..."
|
|
kubectl top nodes || log_warning "Metrics Server 可能还未就绪，请稍后再试 kubectl top nodes"
|
|
|
|
log_success "Metrics Server 部署完成"
|
|
}
|
|
|
|
# 主函数
|
|
main() {
|
|
echo "Kubernetes 集群快速修复工具"
|
|
echo "================================"
|
|
|
|
check_prerequisites
|
|
|
|
if [ $# -eq 0 ]; then
|
|
# 交互式菜单
|
|
while true; do
|
|
show_menu
|
|
read -p "请选择操作 (0-7): " choice
|
|
|
|
case $choice in
|
|
1)
|
|
fix_grafana_images
|
|
;;
|
|
2)
|
|
cache_p1_images
|
|
;;
|
|
3)
|
|
clean_dev_namespace
|
|
;;
|
|
4)
|
|
diagnose_failed_pods
|
|
;;
|
|
5)
|
|
check_image_pull_errors
|
|
;;
|
|
6)
|
|
auto_fix_all
|
|
;;
|
|
7)
|
|
deploy_metrics_server
|
|
;;
|
|
0)
|
|
log_info "退出程序"
|
|
exit 0
|
|
;;
|
|
*)
|
|
log_error "无效选择，请重新输入"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
read -p "按回车键继续..."
|
|
done
|
|
else
|
|
# 命令行参数模式
|
|
case "$1" in
|
|
fix-grafana|1)
|
|
fix_grafana_images
|
|
;;
|
|
cache-p1|2)
|
|
cache_p1_images
|
|
;;
|
|
clean-dev|3)
|
|
clean_dev_namespace
|
|
;;
|
|
diagnose|4)
|
|
diagnose_failed_pods
|
|
;;
|
|
check-errors|5)
|
|
check_image_pull_errors
|
|
;;
|
|
auto|6)
|
|
auto_fix_all
|
|
;;
|
|
metrics|7)
|
|
deploy_metrics_server
|
|
;;
|
|
*)
|
|
echo "用法: $0 [fix-grafana|cache-p1|clean-dev|diagnose|check-errors|auto|metrics]"
|
|
echo " 或直接运行 $0 进入交互式菜单"
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
# 运行主函数
|
|
main "$@"
|