347 lines
11 KiB
Bash
Executable File
347 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# Kubernetes 集群快速修复脚本 v2
|
|
# 用途: 清理重复部署、优化集群性能
|
|
# 更新时间: 2026-01-04
|
|
|
|
set -e
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# 颜色定义
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# 日志函数
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
# 检查必要工具
|
|
check_prerequisites() {
|
|
log_info "检查必要工具..."
|
|
|
|
if ! command -v kubectl &> /dev/null; then
|
|
log_error "kubectl 未安装，请先安装 kubectl"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "docker 未安装，请先安装 Docker"
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -f "./cache-image.sh" ]; then
|
|
log_error "cache-image.sh 未找到，请确保在正确的目录下运行此脚本"
|
|
exit 1
|
|
fi
|
|
|
|
log_success "工具检查通过"
|
|
}
|
|
|
|
# 显示菜单
|
|
show_menu() {
|
|
echo ""
|
|
echo "======================================"
|
|
echo " Kubernetes 集群快速修复菜单 v2"
|
|
echo "======================================"
|
|
echo "1. 🧹 删除重复部署: 清理 infra namespace 中的 loki/tempo（推荐）"
|
|
echo "2. 📦 批量缓存: 缓存所有 P1 高优先级镜像 (40+ 个)"
|
|
echo "3. 🗑️ 清理 dev: 删除 dev 命名空间失败的 Deployment"
|
|
echo "4. 📊 诊断: 查看所有失败和异常的 Pod"
|
|
echo "5. 🔍 检查: 查看镜像拉取错误事件"
|
|
echo "6. ⚡ 全自动: 执行所有修复步骤 (1 + 2 + 3)"
|
|
echo "7. 📈 部署: 安装 Metrics Server"
|
|
echo "0. 退出"
|
|
echo "======================================"
|
|
}
|
|
|
|
# P0: 删除重复的 loki/tempo 部署
|
|
delete_duplicate_deployments() {
|
|
log_info "开始删除 infra namespace 中的重复部署..."
|
|
|
|
echo ""
|
|
log_info "当前情况:"
|
|
echo " - ✅ logging namespace: loki-0 (正常运行)"
|
|
echo " - ✅ tracing namespace: tempo-0 (正常运行)"
|
|
echo " - ❌ infra namespace: loki-0, tempo-0 (重复部署，ImagePullBackOff)"
|
|
echo ""
|
|
|
|
log_warning "将要删除以下资源:"
|
|
echo " - statefulset.apps/loki (infra)"
|
|
echo " - statefulset.apps/tempo (infra)"
|
|
echo " - service/loki (infra)"
|
|
echo " - service/tempo (infra)"
|
|
echo ""
|
|
|
|
read -p "确认删除? (y/N): " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
log_info "已取消删除操作"
|
|
return
|
|
fi
|
|
|
|
# 删除 StatefulSet
|
|
log_info "删除 StatefulSet..."
|
|
kubectl delete statefulset -n infra loki 2>/dev/null && log_success "已删除 statefulset/loki" || log_warning "loki StatefulSet 不存在或已删除"
|
|
kubectl delete statefulset -n infra tempo 2>/dev/null && log_success "已删除 statefulset/tempo" || log_warning "tempo StatefulSet 不存在或已删除"
|
|
|
|
# 删除 Service
|
|
log_info "删除 Service..."
|
|
kubectl delete service -n infra loki 2>/dev/null && log_success "已删除 service/loki" || log_warning "loki Service 不存在或已删除"
|
|
kubectl delete service -n infra tempo 2>/dev/null && log_success "已删除 service/tempo" || log_warning "tempo Service 不存在或已删除"
|
|
|
|
echo ""
|
|
log_success "重复部署清理完成！"
|
|
|
|
# 验证正确的实例
|
|
echo ""
|
|
log_info "验证正确的实例运行状态..."
|
|
echo ""
|
|
echo "=== logging namespace ==="
|
|
kubectl get pods -n logging -l app=loki 2>/dev/null || log_warning "logging/loki 不存在"
|
|
echo ""
|
|
echo "=== tracing namespace ==="
|
|
kubectl get pods -n tracing -l app.kubernetes.io/name=tempo 2>/dev/null || log_warning "tracing/tempo 不存在"
|
|
}
|
|
|
|
# P1: 批量缓存镜像
|
|
cache_p1_images() {
|
|
log_info "开始批量缓存 P1 高优先级镜像..."
|
|
|
|
if [ ! -f "images-to-cache-p1.txt" ]; then
|
|
log_error "镜像列表文件 images-to-cache-p1.txt 未找到"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "镜像列表文件: images-to-cache-p1.txt"
|
|
log_warning "这将花费 5-10 分钟，请耐心等待..."
|
|
|
|
./cache-image.sh -f images-to-cache-p1.txt
|
|
|
|
log_success "P1 镜像批量缓存完成！"
|
|
}
|
|
|
|
# 清理 dev 命名空间
|
|
clean_dev_namespace() {
|
|
log_info "清理 dev 命名空间失败的 Deployment..."
|
|
|
|
log_warning "将要删除以下 Deployment:"
|
|
echo " - dev/backend (镜像不存在)"
|
|
echo " - dev/frontend (镜像不存在)"
|
|
echo " - dev/quality-dashboard (镜像不存在)"
|
|
echo ""
|
|
log_info "提示: 这些镜像需要先通过 CI/CD 构建并推送到 Harbor"
|
|
echo ""
|
|
|
|
read -p "确认删除? (y/N): " -n 1 -r
|
|
echo
|
|
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
|
kubectl delete deployment -n dev backend 2>/dev/null && log_success "已删除 dev/backend" || log_warning "backend 不存在"
|
|
kubectl delete deployment -n dev frontend 2>/dev/null && log_success "已删除 dev/frontend" || log_warning "frontend 不存在"
|
|
kubectl delete deployment -n dev quality-dashboard 2>/dev/null && log_success "已删除 dev/quality-dashboard" || log_warning "quality-dashboard 不存在"
|
|
log_success "dev 命名空间清理完成"
|
|
else
|
|
log_info "已取消删除操作"
|
|
fi
|
|
}
|
|
|
|
# 诊断失败的 Pod
|
|
diagnose_failed_pods() {
|
|
log_info "诊断所有失败和异常的 Pod..."
|
|
|
|
echo ""
|
|
echo "=== ImagePullBackOff / ErrImagePull ==="
|
|
kubectl get pods --all-namespaces --field-selector=status.phase=Pending -o wide 2>/dev/null || log_warning "没有 Pending 状态的 Pod"
|
|
|
|
echo ""
|
|
echo "=== CrashLoopBackOff ==="
|
|
kubectl get pods --all-namespaces -o json 2>/dev/null | \
|
|
jq -r '.items[] | select(.status.containerStatuses[]? | .state.waiting?.reason == "CrashLoopBackOff") | "\(.metadata.namespace)\t\(.metadata.name)\t\(.status.containerStatuses[0].restartCount)"' 2>/dev/null || \
|
|
log_warning "jq 未安装或没有 CrashLoopBackOff Pod"
|
|
|
|
echo ""
|
|
echo "=== 高重启次数的 Pod (>100) ==="
|
|
kubectl get pods --all-namespaces -o json 2>/dev/null | \
|
|
jq -r '.items[] | select(.status.containerStatuses[]? | .restartCount > 100) | "\(.metadata.namespace)\t\(.metadata.name)\t\(.status.containerStatuses[0].restartCount)"' 2>/dev/null || \
|
|
log_warning "jq 未安装或没有高重启次数的 Pod"
|
|
}
|
|
|
|
# 检查镜像拉取错误
|
|
check_image_pull_errors() {
|
|
log_info "检查最近的镜像拉取错误事件..."
|
|
|
|
kubectl get events --all-namespaces --field-selector type=Warning 2>/dev/null | grep -i "image\|pull" | head -20 || \
|
|
log_success "没有发现镜像拉取错误事件"
|
|
}
|
|
|
|
# 全自动修复
|
|
auto_fix_all() {
|
|
log_info "开始执行全自动修复流程..."
|
|
log_warning "将执行以下操作:"
|
|
echo " 1. 删除重复的 loki/tempo 部署"
|
|
echo " 2. 批量缓存 P1 高优先级镜像"
|
|
echo " 3. 清理 dev 命名空间失败的 Deployment"
|
|
echo ""
|
|
|
|
read -p "确认执行全自动修复? (y/N): " -n 1 -r
|
|
echo
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
log_info "已取消全自动修复"
|
|
return
|
|
fi
|
|
|
|
# 1. 删除重复部署（自动确认）
|
|
log_info "步骤 1/3: 删除重复的 loki/tempo 部署..."
|
|
kubectl delete statefulset -n infra loki tempo 2>/dev/null || log_warning "StatefulSet 已删除或不存在"
|
|
kubectl delete service -n infra loki tempo 2>/dev/null || log_warning "Service 已删除或不存在"
|
|
log_success "重复部署清理完成"
|
|
|
|
echo ""
|
|
|
|
# 2. 批量缓存镜像
|
|
log_info "步骤 2/3: 批量缓存 P1 镜像..."
|
|
if [ -f "images-to-cache-p1.txt" ]; then
|
|
./cache-image.sh -f images-to-cache-p1.txt
|
|
log_success "镜像缓存完成"
|
|
else
|
|
log_warning "镜像列表文件不存在，跳过此步骤"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# 3. 清理 dev 命名空间（自动确认）
|
|
log_info "步骤 3/3: 清理 dev 命名空间..."
|
|
kubectl delete deployment -n dev backend frontend quality-dashboard 2>/dev/null || log_warning "dev Deployment 已删除或不存在"
|
|
log_success "dev 命名空间清理完成"
|
|
|
|
echo ""
|
|
log_success "全自动修复完成！"
|
|
|
|
# 最终诊断
|
|
echo ""
|
|
log_info "执行最终诊断..."
|
|
diagnose_failed_pods
|
|
}
|
|
|
|
# 部署 Metrics Server
|
|
deploy_metrics_server() {
|
|
log_info "部署 Metrics Server..."
|
|
|
|
log_info "下载 Metrics Server manifests..."
|
|
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
|
|
|
|
log_info "配置 OrbStack 兼容性 (--kubelet-insecure-tls)..."
|
|
kubectl patch deployment metrics-server -n kube-system --type='json' \
|
|
-p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]' 2>/dev/null || \
|
|
log_warning "Patch 失败，可能 Metrics Server 已配置"
|
|
|
|
log_info "等待 Metrics Server 启动..."
|
|
sleep 20
|
|
|
|
log_info "验证 Metrics Server..."
|
|
kubectl top nodes || log_warning "Metrics Server 可能还未就绪，请稍后再试 kubectl top nodes"
|
|
|
|
log_success "Metrics Server 部署完成"
|
|
}
|
|
|
|
# 主函数
|
|
main() {
|
|
echo "Kubernetes 集群快速修复工具 v2"
|
|
echo "================================"
|
|
echo "更新: 修正了重复部署问题"
|
|
echo ""
|
|
|
|
check_prerequisites
|
|
|
|
if [ $# -eq 0 ]; then
|
|
# 交互式菜单
|
|
while true; do
|
|
show_menu
|
|
read -p "请选择操作 (0-7): " choice
|
|
|
|
case $choice in
|
|
1)
|
|
delete_duplicate_deployments
|
|
;;
|
|
2)
|
|
cache_p1_images
|
|
;;
|
|
3)
|
|
clean_dev_namespace
|
|
;;
|
|
4)
|
|
diagnose_failed_pods
|
|
;;
|
|
5)
|
|
check_image_pull_errors
|
|
;;
|
|
6)
|
|
auto_fix_all
|
|
;;
|
|
7)
|
|
deploy_metrics_server
|
|
;;
|
|
0)
|
|
log_info "退出程序"
|
|
exit 0
|
|
;;
|
|
*)
|
|
log_error "无效选择，请重新输入"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
read -p "按回车键继续..."
|
|
done
|
|
else
|
|
# 命令行参数模式
|
|
case "$1" in
|
|
delete-duplicates|1)
|
|
delete_duplicate_deployments
|
|
;;
|
|
cache-p1|2)
|
|
cache_p1_images
|
|
;;
|
|
clean-dev|3)
|
|
clean_dev_namespace
|
|
;;
|
|
diagnose|4)
|
|
diagnose_failed_pods
|
|
;;
|
|
check-errors|5)
|
|
check_image_pull_errors
|
|
;;
|
|
auto|6)
|
|
auto_fix_all
|
|
;;
|
|
metrics|7)
|
|
deploy_metrics_server
|
|
;;
|
|
*)
|
|
echo "用法: $0 [delete-duplicates|cache-p1|clean-dev|diagnose|check-errors|auto|metrics]"
|
|
echo " 或直接运行 $0 进入交互式菜单"
|
|
exit 1
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
# 运行主函数
|
|
main "$@"
|