149 lines
4.4 KiB
Bash
Executable File
149 lines
4.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# Kubernetes 镜像拉取问题后台监控和自动修复
|
|
# 用途：持续监控集群中的 ImagePull 错误并自动修复
|
|
|
|
# Note: 不使用 set -e，因为这是一个持续运行的监控脚本
|
|
# 允许单个命令失败而不退出整个脚本
|
|
|
|
CACHE_SCRIPT="/Users/brunogao/work/codes/tools/macos/k8s/cache-image.sh"
|
|
CHECK_INTERVAL="${CHECK_INTERVAL:-30}"
|
|
LOG_FILE="/tmp/image-watch.log"
|
|
FIX_LOG="/tmp/image-fix.log"
|
|
|
|
# 颜色定义
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
log() {
|
|
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
|
echo -e "$msg" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_info() { log "${BLUE}[INFO]${NC} $1"; }
|
|
log_success() { log "${GREEN}[✓]${NC} $1"; }
|
|
log_error() { log "${RED}[✗]${NC} $1"; }
|
|
log_warning() { log "${YELLOW}[!]${NC} $1"; }
|
|
|
|
# 检查脚本是否存在
|
|
if [[ ! -x "$CACHE_SCRIPT" ]]; then
|
|
log_error "缓存脚本不存在或不可执行: $CACHE_SCRIPT"
|
|
exit 1
|
|
fi
|
|
|
|
# 信号处理
|
|
cleanup() {
|
|
log_info "收到停止信号，退出监控..."
|
|
exit 0
|
|
}
|
|
|
|
trap cleanup SIGINT SIGTERM
|
|
|
|
# 主监控循环
|
|
log_info "=== 启动 Kubernetes 镜像拉取监控 ==="
|
|
log_info "检查间隔: ${CHECK_INTERVAL}s"
|
|
log_info "日志文件: $LOG_FILE"
|
|
log_info "修复日志: $FIX_LOG"
|
|
log_info "按 Ctrl+C 停止"
|
|
echo ""
|
|
|
|
# 统计
|
|
TOTAL_CHECKS=0
|
|
TOTAL_FIXES=0
|
|
TOTAL_FAILURES=0
|
|
|
|
while true; do
|
|
((TOTAL_CHECKS++))
|
|
|
|
# 查找所有 ImagePull 失败的 Pods
|
|
FAILED_PODS=$(kubectl get pods --all-namespaces -o json 2>/dev/null | \
|
|
jq -r '.items[] |
|
|
select(.status.containerStatuses[]?.state.waiting?.reason |
|
|
test("ImagePull|ErrImage")) |
|
|
"\(.metadata.namespace)/\(.metadata.name)"' 2>/dev/null || true)
|
|
|
|
if [[ -n "$FAILED_PODS" ]]; then
|
|
log_warning "检测到镜像拉取失败"
|
|
echo ""
|
|
|
|
# 去重 Deployment
|
|
declare -A PROCESSED
|
|
|
|
while IFS= read -r pod; do
|
|
[[ -z "$pod" ]] && continue
|
|
|
|
NS="${pod%/*}"
|
|
POD_NAME="${pod##*/}"
|
|
|
|
# 获取 Pod 的 owner
|
|
OWNER_INFO=$(kubectl get pod "$POD_NAME" -n "$NS" -o json 2>/dev/null | \
|
|
jq -r '.metadata.ownerReferences[]? |
|
|
select(.kind | test("ReplicaSet|StatefulSet")) |
|
|
"\(.kind)/\(.name)"' || true)
|
|
|
|
if [[ -z "$OWNER_INFO" ]]; then
|
|
log_warning " 跳过 $pod (无法获取 owner)"
|
|
continue
|
|
fi
|
|
|
|
OWNER_KIND="${OWNER_INFO%/*}"
|
|
OWNER_NAME="${OWNER_INFO##*/}"
|
|
|
|
# ReplicaSet 需要获取 Deployment
|
|
if [[ "$OWNER_KIND" == "ReplicaSet" ]]; then
|
|
if [[ "$OWNER_NAME" =~ ^(.+)-[a-z0-9]+$ ]]; then
|
|
DEPLOY_NAME="${BASH_REMATCH[1]}"
|
|
DEPLOY_KEY="$NS/$DEPLOY_NAME"
|
|
RESOURCE_TYPE="deployment"
|
|
else
|
|
log_warning " 跳过 $pod (无法解析 Deployment 名称)"
|
|
continue
|
|
fi
|
|
elif [[ "$OWNER_KIND" == "StatefulSet" ]]; then
|
|
DEPLOY_KEY="$NS/$OWNER_NAME"
|
|
DEPLOY_NAME="$OWNER_NAME"
|
|
RESOURCE_TYPE="statefulset"
|
|
else
|
|
log_warning " 跳过 $pod (不支持的类型: $OWNER_KIND)"
|
|
continue
|
|
fi
|
|
|
|
# 避免重复处理
|
|
if [[ -n "${PROCESSED[$DEPLOY_KEY]}" ]]; then
|
|
continue
|
|
fi
|
|
|
|
log_info "🔧 自动修复: $DEPLOY_KEY ($RESOURCE_TYPE)"
|
|
|
|
if "$CACHE_SCRIPT" --update-deployment "$DEPLOY_KEY" >> "$FIX_LOG" 2>&1; then
|
|
log_success " 修复成功"
|
|
((TOTAL_FIXES++))
|
|
PROCESSED[$DEPLOY_KEY]=1
|
|
else
|
|
log_error " 修复失败 (查看日志: $FIX_LOG)"
|
|
((TOTAL_FAILURES++))
|
|
fi
|
|
|
|
echo ""
|
|
done <<< "$FAILED_PODS"
|
|
|
|
if [[ ${#PROCESSED[@]} -gt 0 ]]; then
|
|
log_success "本轮共修复 ${#PROCESSED[@]} 个资源"
|
|
log_info "等待 Pod 重启..."
|
|
sleep 10
|
|
fi
|
|
|
|
echo ""
|
|
fi
|
|
|
|
# 每隔一段时间显示统计
|
|
if (( TOTAL_CHECKS % 10 == 0 )); then
|
|
log_info "统计: 检查 $TOTAL_CHECKS 次, 修复 $TOTAL_FIXES 次, 失败 $TOTAL_FAILURES 次"
|
|
fi
|
|
|
|
sleep "$CHECK_INTERVAL"
|
|
done
|