286 lines
8.7 KiB
Bash
Executable File
286 lines
8.7 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
set -e
|
||
|
||
echo "======================================"
|
||
echo "启用 Prometheus 告警规则"
|
||
echo "======================================"
|
||
echo ""
|
||
|
||
# 1. 备份当前配置
|
||
echo "1. 备份当前 Prometheus 配置..."
|
||
kubectl get configmap prometheus-config -n monitoring -o yaml > /tmp/prometheus-config-backup-$(date +%Y%m%d-%H%M%S).yaml
|
||
echo " ✅ 备份保存到: /tmp/prometheus-config-backup-*.yaml"
|
||
echo ""
|
||
|
||
# 2. 检查 prometheus-rules ConfigMap 是否存在
|
||
echo "2. 检查告警规则 ConfigMap..."
|
||
if kubectl get configmap prometheus-rules -n monitoring &> /dev/null; then
|
||
echo " ✅ prometheus-rules ConfigMap 存在"
|
||
kubectl get configmap prometheus-rules -n monitoring -o jsonpath='{.data}' | grep -o '\.yml' | wc -l | xargs echo " 发现规则文件数量:"
|
||
else
|
||
echo " ❌ prometheus-rules ConfigMap 不存在"
|
||
echo " 请先创建告警规则"
|
||
exit 1
|
||
fi
|
||
echo ""
|
||
|
||
# 3. 更新 Prometheus ConfigMap 添加 rule_files
|
||
echo "3. 更新 Prometheus ConfigMap 添加 rule_files..."
|
||
|
||
cat <<'EOF' > /tmp/prometheus-config-patch.yaml
|
||
apiVersion: v1
|
||
kind: ConfigMap
|
||
metadata:
|
||
name: prometheus-config
|
||
namespace: monitoring
|
||
data:
|
||
prometheus.yml: |
|
||
global:
|
||
scrape_interval: 15s
|
||
evaluation_interval: 15s
|
||
|
||
# 告警规则文件
|
||
rule_files:
|
||
- /etc/prometheus/rules/*.yml
|
||
|
||
alerting:
|
||
alertmanagers:
|
||
- static_configs:
|
||
- targets:
|
||
- alertmanager:9093
|
||
|
||
scrape_configs:
|
||
# ==================== 基础设施监控 ====================
|
||
|
||
# Prometheus 自身
|
||
- job_name: 'prometheus'
|
||
static_configs:
|
||
- targets: ['localhost:9090']
|
||
|
||
# Kubernetes API Server
|
||
- job_name: 'kubernetes-apiservers'
|
||
kubernetes_sd_configs:
|
||
- role: endpoints
|
||
scheme: https
|
||
tls_config:
|
||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
|
||
action: keep
|
||
regex: default;kubernetes;https
|
||
|
||
# Kubernetes Nodes
|
||
- job_name: 'kubernetes-nodes'
|
||
kubernetes_sd_configs:
|
||
- role: node
|
||
scheme: https
|
||
tls_config:
|
||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||
relabel_configs:
|
||
- action: labelmap
|
||
regex: __meta_kubernetes_node_label_(.+)
|
||
|
||
# Node Exporter
|
||
- job_name: 'node-exporter'
|
||
kubernetes_sd_configs:
|
||
- role: endpoints
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_service_name]
|
||
action: keep
|
||
regex: node-exporter
|
||
- source_labels: [__meta_kubernetes_namespace]
|
||
target_label: namespace
|
||
- source_labels: [__meta_kubernetes_pod_name]
|
||
target_label: pod
|
||
|
||
# kube-state-metrics
|
||
- job_name: 'kube-state-metrics'
|
||
static_configs:
|
||
- targets: ['kube-state-metrics.monitoring.svc.cluster.local:8080']
|
||
|
||
# Kubernetes Pods
|
||
- job_name: 'kubernetes-pods'
|
||
kubernetes_sd_configs:
|
||
- role: pod
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||
action: keep
|
||
regex: true
|
||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||
action: replace
|
||
target_label: __metrics_path__
|
||
regex: (.+)
|
||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||
action: replace
|
||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||
replacement: $1:$2
|
||
target_label: __address__
|
||
- action: labelmap
|
||
regex: __meta_kubernetes_pod_label_(.+)
|
||
- source_labels: [__meta_kubernetes_namespace]
|
||
action: replace
|
||
target_label: kubernetes_namespace
|
||
- source_labels: [__meta_kubernetes_pod_name]
|
||
action: replace
|
||
target_label: kubernetes_pod_name
|
||
|
||
# ==================== 数据库和缓存监控（data namespace）====================
|
||
|
||
# Postgres Exporter
|
||
- job_name: 'postgres-exporter'
|
||
static_configs:
|
||
- targets: ['postgres-postgresql-metrics.data.svc.cluster.local:9187']
|
||
labels:
|
||
namespace: data
|
||
service: postgres-postgresql
|
||
component: database
|
||
|
||
# Redis Exporter
|
||
- job_name: 'redis-exporter'
|
||
static_configs:
|
||
- targets: ['redis-metrics.data.svc.cluster.local:9121']
|
||
labels:
|
||
namespace: data
|
||
service: redis-master
|
||
component: cache
|
||
|
||
# ==================== 可观测性组件自监控 ====================
|
||
|
||
# Loki
|
||
- job_name: 'loki'
|
||
static_configs:
|
||
- targets: ['loki.logging.svc.cluster.local:3100']
|
||
labels:
|
||
component: loki
|
||
|
||
# Tempo
|
||
- job_name: 'tempo'
|
||
static_configs:
|
||
- targets: ['tempo.tracing.svc.cluster.local:3200']
|
||
labels:
|
||
component: tempo
|
||
|
||
# Grafana
|
||
- job_name: 'grafana'
|
||
static_configs:
|
||
- targets: ['grafana.monitoring.svc.cluster.local:3000']
|
||
labels:
|
||
component: grafana
|
||
|
||
# ==================== 业务应用监控 ====================
|
||
|
||
# 业务应用 - 通过 Service annotation 自动发现
|
||
- job_name: 'business-services'
|
||
kubernetes_sd_configs:
|
||
- role: service
|
||
relabel_configs:
|
||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
||
action: keep
|
||
regex: true
|
||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
||
action: replace
|
||
target_label: __metrics_path__
|
||
regex: (.+)
|
||
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
||
action: replace
|
||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||
replacement: $1:$2
|
||
target_label: __address__
|
||
- action: labelmap
|
||
regex: __meta_kubernetes_service_label_(.+)
|
||
- source_labels: [__meta_kubernetes_namespace]
|
||
target_label: namespace
|
||
- source_labels: [__meta_kubernetes_service_name]
|
||
target_label: service
|
||
EOF
|
||
|
||
kubectl apply -f /tmp/prometheus-config-patch.yaml
|
||
echo " ✅ ConfigMap 已更新（添加 rule_files）"
|
||
echo ""
|
||
|
||
# 4. 更新 Prometheus Deployment 挂载 rules ConfigMap
|
||
echo "4. 更新 Prometheus Deployment 挂载告警规则..."
|
||
|
||
# 检查是否已经挂载
|
||
if kubectl get deployment prometheus -n monitoring -o yaml | grep -q "prometheus-rules"; then
|
||
echo " ℹ️ 告警规则已挂载，跳过"
|
||
else
|
||
# 添加 volume
|
||
kubectl patch deployment prometheus -n monitoring --type='json' -p='[
|
||
{
|
||
"op": "add",
|
||
"path": "/spec/template/spec/volumes/-",
|
||
"value": {
|
||
"name": "prometheus-rules",
|
||
"configMap": {
|
||
"name": "prometheus-rules"
|
||
}
|
||
}
|
||
}
|
||
]'
|
||
|
||
# 添加 volumeMount
|
||
kubectl patch deployment prometheus -n monitoring --type='json' -p='[
|
||
{
|
||
"op": "add",
|
||
"path": "/spec/template/spec/containers/0/volumeMounts/-",
|
||
"value": {
|
||
"name": "prometheus-rules",
|
||
"mountPath": "/etc/prometheus/rules"
|
||
}
|
||
}
|
||
]'
|
||
|
||
echo " ✅ Deployment 已更新（挂载告警规则）"
|
||
fi
|
||
echo ""
|
||
|
||
# 5. 等待 Prometheus 重启
|
||
echo "5. 等待 Prometheus Pod 重启..."
|
||
kubectl rollout status deployment prometheus -n monitoring --timeout=120s
|
||
echo " ✅ Prometheus 重启完成"
|
||
echo ""
|
||
|
||
# 6. 验证告警规则加载
|
||
echo "6. 验证告警规则加载..."
|
||
sleep 10
|
||
|
||
# 启动端口转发
|
||
kubectl port-forward -n monitoring svc/prometheus 9090:9090 > /dev/null 2>&1 &
|
||
PF_PID=$!
|
||
sleep 5
|
||
|
||
# 检查规则
|
||
RULES_COUNT=$(curl -s http://localhost:9090/api/v1/rules | jq -r '.data.groups | length')
|
||
|
||
# 停止端口转发
|
||
kill $PF_PID 2>/dev/null || true
|
||
|
||
if [ "$RULES_COUNT" -gt 0 ]; then
|
||
echo " ✅ 告警规则已加载: $RULES_COUNT 个规则组"
|
||
else
|
||
echo " ⚠️ 未检测到告警规则"
|
||
echo " 请访问 http://localhost:9090/rules 手动验证"
|
||
fi
|
||
echo ""
|
||
|
||
echo "======================================"
|
||
echo "✅ 告警规则启用完成"
|
||
echo "======================================"
|
||
echo ""
|
||
echo "验证步骤:"
|
||
echo "1. 启动端口转发:"
|
||
echo " kubectl port-forward -n monitoring svc/prometheus 9090:9090"
|
||
echo ""
|
||
echo "2. 访问 Prometheus Rules 页面:"
|
||
echo " http://localhost:9090/rules"
|
||
echo ""
|
||
echo "3. 预期看到:"
|
||
echo " - postgres-alerts (20+ 条规则)"
|
||
echo " - redis-alerts (20+ 条规则)"
|
||
echo " - k3s-operational-alerts (30+ 条规则)"
|
||
echo ""
|