304 lines
9.0 KiB
Bash
Executable File
304 lines
9.0 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# 平台治理修复脚本
|
||
# 覆盖：ArgoCD 应用重叠检查、镜像策略巡检、infra ownerless Pod 清理、cert-manager 凭证/issuer 巡检
|
||
|
||
set -euo pipefail
|
||
|
||
KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
|
||
JQ_BIN="${JQ_BIN:-jq}"
|
||
MAX_ORPHAN_AGE_HOURS="${MAX_ORPHAN_AGE_HOURS:-72}"
|
||
|
||
ISSUE_COUNT=0
|
||
|
||
log_info() {
|
||
echo "[INFO] $1"
|
||
}
|
||
|
||
log_warn() {
|
||
echo "[WARN] $1"
|
||
}
|
||
|
||
log_ok() {
|
||
echo "[OK] $1"
|
||
}
|
||
|
||
inc_issue() {
|
||
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
||
}
|
||
|
||
usage() {
|
||
cat <<USAGE
|
||
用法: $0 <command>
|
||
|
||
commands:
|
||
audit 执行全部治理检查（只读）
|
||
fix-image-policy 修复 latest + imagePullPolicy=Never（改为 Always）
|
||
cleanup-orphan-pods 清理 infra 中超过 ${MAX_ORPHAN_AGE_HOURS}h 的无 owner 一次性 Pod
|
||
all 先 audit，再执行 fix-image-policy 与 cleanup-orphan-pods
|
||
USAGE
|
||
}
|
||
|
||
require_tools() {
|
||
command -v "$KUBECTL_BIN" >/dev/null 2>&1 || { echo "kubectl 不存在"; exit 1; }
|
||
command -v "$JQ_BIN" >/dev/null 2>&1 || { echo "jq 不存在"; exit 1; }
|
||
}
|
||
|
||
ensure_cluster_access() {
|
||
if ! "$KUBECTL_BIN" version --request-timeout='5s' >/dev/null 2>&1; then
|
||
echo "[WARN] 无法连接 Kubernetes API，请先确认 kubeconfig 和集群连通性"
|
||
exit 2
|
||
fi
|
||
}
|
||
|
||
decode_b64() {
|
||
if base64 --help 2>&1 | grep -q -- "--decode"; then
|
||
base64 --decode
|
||
else
|
||
base64 -D
|
||
fi
|
||
}
|
||
|
||
audit_argocd_overlap() {
|
||
log_info "检查 ArgoCD 应用重叠（SharedResourceWarning）..."
|
||
|
||
local apps_json
|
||
if ! apps_json="$($KUBECTL_BIN get applications.argoproj.io -A -o json 2>/dev/null)"; then
|
||
log_warn "集群未发现 ArgoCD Application CRD，跳过该检查"
|
||
return 0
|
||
fi
|
||
|
||
local overlaps
|
||
overlaps="$(echo "$apps_json" | "$JQ_BIN" -r '
|
||
.items[]?
|
||
| [ .status.conditions[]? | select(.type == "SharedResourceWarning") | .message ] as $msgs
|
||
| select(($msgs | length) > 0)
|
||
| "\(.metadata.namespace)/\(.metadata.name)\t\($msgs | join(" | "))"
|
||
')"
|
||
|
||
if [[ -z "$overlaps" ]]; then
|
||
log_ok "未发现 SharedResourceWarning"
|
||
else
|
||
inc_issue
|
||
log_warn "发现应用重叠，请按输出逐项收敛资源归属（一个资源只能由一个 Application 管理）"
|
||
echo "$overlaps"
|
||
fi
|
||
}
|
||
|
||
collect_bad_image_policy() {
|
||
"$KUBECTL_BIN" get deployment,statefulset,daemonset,job,cronjob -A -o json 2>/dev/null | "$JQ_BIN" -r '
|
||
.items[] as $w
|
||
| (if $w.kind == "CronJob" then $w.spec.jobTemplate.spec.template.spec else $w.spec.template.spec end) as $pod
|
||
| ($pod.containers // [])
|
||
| to_entries[]
|
||
| . as $c
|
||
| ($c.value.imagePullPolicy // "IfNotPresent") as $pull
|
||
| select(($c.value.image | test(":latest($|@)")) and $pull == "Never")
|
||
| "\($w.kind)\t\($w.metadata.namespace)\t\($w.metadata.name)\t\($c.key)\t\($c.value.name)\t\($c.value.image)\t\($pull)"
|
||
'
|
||
}
|
||
|
||
audit_image_policy() {
|
||
log_info "检查 latest + imagePullPolicy=Never 组合..."
|
||
|
||
local bad
|
||
bad="$(collect_bad_image_policy || true)"
|
||
|
||
if [[ -z "$bad" ]]; then
|
||
log_ok "未发现 latest + Never 组合"
|
||
else
|
||
inc_issue
|
||
log_warn "发现不合规镜像策略："
|
||
echo -e "KIND\tNAMESPACE\tWORKLOAD\tIDX\tCONTAINER\tIMAGE\tPULL_POLICY"
|
||
echo "$bad"
|
||
fi
|
||
}
|
||
|
||
fix_image_policy() {
|
||
log_info "修复 latest + imagePullPolicy=Never -> Always..."
|
||
|
||
local bad
|
||
bad="$(collect_bad_image_policy || true)"
|
||
|
||
if [[ -z "$bad" ]]; then
|
||
log_ok "无需修复"
|
||
return 0
|
||
fi
|
||
|
||
while IFS=$'\t' read -r kind ns workload idx cname image pull; do
|
||
[[ -z "$kind" ]] && continue
|
||
|
||
local patch_path
|
||
if [[ "$kind" == "CronJob" ]]; then
|
||
patch_path="/spec/jobTemplate/spec/template/spec/containers/${idx}/imagePullPolicy"
|
||
"$KUBECTL_BIN" -n "$ns" patch cronjob "$workload" --type='json' \
|
||
-p="[{\"op\":\"replace\",\"path\":\"${patch_path}\",\"value\":\"Always\"}]" >/dev/null
|
||
elif [[ "$kind" == "Deployment" ]]; then
|
||
patch_path="/spec/template/spec/containers/${idx}/imagePullPolicy"
|
||
"$KUBECTL_BIN" -n "$ns" patch deployment "$workload" --type='json' \
|
||
-p="[{\"op\":\"replace\",\"path\":\"${patch_path}\",\"value\":\"Always\"}]" >/dev/null
|
||
elif [[ "$kind" == "StatefulSet" ]]; then
|
||
patch_path="/spec/template/spec/containers/${idx}/imagePullPolicy"
|
||
"$KUBECTL_BIN" -n "$ns" patch statefulset "$workload" --type='json' \
|
||
-p="[{\"op\":\"replace\",\"path\":\"${patch_path}\",\"value\":\"Always\"}]" >/dev/null
|
||
elif [[ "$kind" == "DaemonSet" ]]; then
|
||
patch_path="/spec/template/spec/containers/${idx}/imagePullPolicy"
|
||
"$KUBECTL_BIN" -n "$ns" patch daemonset "$workload" --type='json' \
|
||
-p="[{\"op\":\"replace\",\"path\":\"${patch_path}\",\"value\":\"Always\"}]" >/dev/null
|
||
elif [[ "$kind" == "Job" ]]; then
|
||
patch_path="/spec/template/spec/containers/${idx}/imagePullPolicy"
|
||
"$KUBECTL_BIN" -n "$ns" patch job "$workload" --type='json' \
|
||
-p="[{\"op\":\"replace\",\"path\":\"${patch_path}\",\"value\":\"Always\"}]" >/dev/null
|
||
else
|
||
log_warn "不支持的资源类型: $kind ($ns/$workload)"
|
||
continue
|
||
fi
|
||
|
||
log_ok "已修复: ${kind} ${ns}/${workload} container=${cname} (${image})"
|
||
done <<< "$bad"
|
||
}
|
||
|
||
collect_orphan_pods() {
|
||
"$KUBECTL_BIN" get pods -n infra -o json 2>/dev/null | "$JQ_BIN" -r --argjson max_age "$((MAX_ORPHAN_AGE_HOURS * 3600))" '
|
||
.items[]?
|
||
| select((.metadata.ownerReferences // []) | length == 0)
|
||
| select((now - (.metadata.creationTimestamp | fromdateiso8601)) > $max_age)
|
||
| "\(.metadata.name)\t\(.status.phase)\t\(.metadata.creationTimestamp)"
|
||
'
|
||
}
|
||
|
||
audit_orphan_pods() {
|
||
log_info "检查 infra 中长期无 owner Pod（>${MAX_ORPHAN_AGE_HOURS}h）..."
|
||
|
||
local pods
|
||
pods="$(collect_orphan_pods || true)"
|
||
|
||
if [[ -z "$pods" ]]; then
|
||
log_ok "未发现长期无 owner Pod"
|
||
else
|
||
inc_issue
|
||
log_warn "发现长期无 owner Pod："
|
||
echo -e "NAME\tPHASE\tCREATED_AT"
|
||
echo "$pods"
|
||
fi
|
||
}
|
||
|
||
cleanup_orphan_pods() {
|
||
log_info "清理 infra 中长期无 owner 一次性 Pod（Succeeded/Failed）..."
|
||
|
||
local pods
|
||
pods="$(collect_orphan_pods || true)"
|
||
|
||
if [[ -z "$pods" ]]; then
|
||
log_ok "无需清理"
|
||
return 0
|
||
fi
|
||
|
||
while IFS=$'\t' read -r name phase created_at; do
|
||
[[ -z "$name" ]] && continue
|
||
if [[ "$phase" == "Succeeded" || "$phase" == "Failed" ]]; then
|
||
"$KUBECTL_BIN" delete pod -n infra "$name" --wait=false >/dev/null
|
||
log_ok "已删除: ${name} (${phase}, ${created_at})"
|
||
else
|
||
log_warn "保留（非一次性状态）: ${name} (${phase}, ${created_at})"
|
||
fi
|
||
done <<< "$pods"
|
||
}
|
||
|
||
audit_cert_manager() {
|
||
log_info "检查 cert-manager 凭证格式与 issuer 路由..."
|
||
|
||
local dnspod_b64=""
|
||
local cloudflare_b64=""
|
||
|
||
dnspod_b64="$($KUBECTL_BIN get secret dnspod-api-token -n cert-manager -o jsonpath='{.data.api-token}' 2>/dev/null || true)"
|
||
if [[ -n "$dnspod_b64" ]]; then
|
||
local dnspod_token
|
||
dnspod_token="$(echo "$dnspod_b64" | decode_b64 2>/dev/null || true)"
|
||
if [[ "$dnspod_token" == *,* ]]; then
|
||
log_ok "DNSPod token 格式正确（ID,Token）"
|
||
else
|
||
inc_issue
|
||
log_warn "DNSPod token 格式异常，期望 ID,Token"
|
||
fi
|
||
else
|
||
log_warn "未找到 secret cert-manager/dnspod-api-token"
|
||
fi
|
||
|
||
cloudflare_b64="$($KUBECTL_BIN get secret cloudflare-api-token -n cert-manager -o jsonpath='{.data.api-token}' 2>/dev/null || true)"
|
||
if [[ -n "$cloudflare_b64" ]]; then
|
||
local cloudflare_token
|
||
cloudflare_token="$(echo "$cloudflare_b64" | decode_b64 2>/dev/null || true)"
|
||
if [[ "$cloudflare_token" == *,* ]]; then
|
||
inc_issue
|
||
log_warn "Cloudflare token 格式异常，不应使用 ID,Token"
|
||
else
|
||
log_ok "Cloudflare token 格式正确（单 token）"
|
||
fi
|
||
else
|
||
log_warn "未找到 secret cert-manager/cloudflare-api-token"
|
||
fi
|
||
|
||
local dnspod_group
|
||
dnspod_group="$($KUBECTL_BIN get clusterissuer dnspod-letsencrypt -o jsonpath='{.spec.acme.solvers[0].dns01.webhook.groupName}' 2>/dev/null || true)"
|
||
if [[ -n "$dnspod_group" ]]; then
|
||
if [[ "$dnspod_group" == "acme.omniverseai.net" ]]; then
|
||
log_ok "DNSPod issuer groupName 已统一: ${dnspod_group}"
|
||
else
|
||
inc_issue
|
||
log_warn "DNSPod issuer groupName 非预期: ${dnspod_group}（应为 acme.omniverseai.net）"
|
||
fi
|
||
else
|
||
log_warn "未找到 clusterissuer/dnspod-letsencrypt"
|
||
fi
|
||
|
||
for deprecated in letsencrypt-prod letsencrypt-staging; do
|
||
if $KUBECTL_BIN get clusterissuer "$deprecated" >/dev/null 2>&1; then
|
||
inc_issue
|
||
log_warn "发现已弃用 ClusterIssuer: ${deprecated}（建议迁移到 dnspod-/cloudflare-/http01-前缀）"
|
||
fi
|
||
done
|
||
}
|
||
|
||
run_audit() {
|
||
audit_argocd_overlap
|
||
audit_image_policy
|
||
audit_orphan_pods
|
||
audit_cert_manager
|
||
|
||
if [[ "$ISSUE_COUNT" -eq 0 ]]; then
|
||
log_ok "治理检查通过"
|
||
else
|
||
log_warn "治理检查完成，发现 ${ISSUE_COUNT} 个问题"
|
||
fi
|
||
}
|
||
|
||
main() {
|
||
require_tools
|
||
ensure_cluster_access
|
||
|
||
local cmd="${1:-audit}"
|
||
case "$cmd" in
|
||
audit)
|
||
run_audit
|
||
;;
|
||
fix-image-policy)
|
||
fix_image_policy
|
||
;;
|
||
cleanup-orphan-pods)
|
||
cleanup_orphan_pods
|
||
;;
|
||
all)
|
||
run_audit
|
||
fix_image_policy
|
||
cleanup_orphan_pods
|
||
;;
|
||
*)
|
||
usage
|
||
exit 1
|
||
;;
|
||
esac
|
||
}
|
||
|
||
main "$@"
|