ljwx-k8s/prometheus-alerts.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: prometheus-alerts
  namespace: monitoring
  labels:
    prometheus: kube-prometheus
    role: alert-rules
spec:
  groups:
      # ==================== Node 级别告警 ====================
      - name: node-alerts
        interval: 30s
        rules:
          # 节点磁盘空间告警
          - alert: NodeDiskSpaceWarning
            expr: |
              (node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|fuse.*"} / node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|fuse.*"}) * 100 < 20
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "节点 {{ $labels.instance }} 磁盘空间不足"
              description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 剩余空间低于 20%，当前为 {{ $value | humanize }}%"

          - alert: NodeDiskSpaceCritical
            expr: |
              (node_filesystem_avail_bytes{job="node-exporter",fstype!~"tmpfs|fuse.*"} / node_filesystem_size_bytes{job="node-exporter",fstype!~"tmpfs|fuse.*"}) * 100 < 10
            for: 2m
            labels:
              severity: critical
            annotations:
              summary: "节点 {{ $labels.instance }} 磁盘空间严重不足"
              description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 剩余空间低于 10%，当前为 {{ $value | humanize }}%"

          # 节点 inode 告警
          - alert: NodeInodeSpaceWarning
            expr: |
              (node_filesystem_files_free{job="node-exporter",fstype!~"tmpfs|fuse.*"} / node_filesystem_files{job="node-exporter",fstype!~"tmpfs|fuse.*"}) * 100 < 20
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "节点 {{ $labels.instance }} inode 不足"
              description: "节点 {{ $labels.instance }} 挂载点 {{ $labels.mountpoint }} 剩余 inode 低于 20%"

          # 节点内存压力告警
          - alert: NodeMemoryPressure
            expr: |
              (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "节点 {{ $labels.instance }} 内存使用率过高"
              description: "节点 {{ $labels.instance }} 内存使用率超过 85%，当前为 {{ $value | humanize }}%"

          # 节点 CPU 使用率告警
          - alert: NodeCPUHigh
            expr: |
              100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "节点 {{ $labels.instance }} CPU 使用率过高"
              description: "节点 {{ $labels.instance }} CPU 使用率持续 10 分钟超过 80%，当前为 {{ $value | humanize }}%"

      # ==================== Pod 级别告警 ====================
      - name: pod-alerts
        interval: 30s
        rules:
          # Pod CrashLoopBackOff
          - alert: PodCrashLoopBackOff
            expr: |
              rate(kube_pod_container_status_restarts_total[15m]) > 0
            for: 5m
            labels:
              severity: critical
            annotations:
              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 频繁重启"
              description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 容器 {{ $labels.container }} 在过去 15 分钟内重启了 {{ $value }} 次"

          # Pod Pending 时间过长
          - alert: PodPendingTooLong
            expr: |
              kube_pod_status_phase{phase="Pending"} > 0
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 长时间处于 Pending 状态"
              description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 已经 Pending 超过 10 分钟"

          # Pod 内存使用率告警
          - alert: PodMemoryHigh
            expr: |
              (sum by (namespace, pod) (container_memory_working_set_bytes{container!=""}) / sum by (namespace, pod) (kube_pod_container_resource_limits{resource="memory",container!=""})) * 100 > 90
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 内存使用率过高"
              description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 内存使用率超过限制的 90%"

          # Pod CPU 使用率告警
          - alert: PodCPUHigh
            expr: |
              (sum by (namespace, pod) (rate(container_cpu_usage_seconds_total{container!=""}[5m])) / sum by (namespace, pod) (kube_pod_container_resource_limits{resource="cpu",container!=""})) * 100 > 90
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} CPU 使用率过高"
              description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} CPU 使用率超过限制的 90%"

      # ==================== Workload 级别告警 ====================
      - name: workload-alerts
        interval: 30s
        rules:
          # Deployment 副本数不足
          - alert: DeploymentReplicasMismatch
            expr: |
              kube_deployment_spec_replicas != kube_deployment_status_replicas_available
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 副本数不匹配"
              description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 期望 {{ $value }} 个副本，但当前可用副本数不足"

          # StatefulSet 副本数不足
          - alert: StatefulSetReplicasMismatch
            expr: |
              kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 副本数不匹配"
              description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} 就绪副本数不足"

          # DaemonSet 未完全部署
          - alert: DaemonSetNotScheduled
            expr: |
              kube_daemonset_status_desired_number_scheduled != kube_daemonset_status_current_number_scheduled
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} 未完全部署"
              description: "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} 期望在所有节点上运行，但当前未完全部署"

          # Job 失败
          - alert: JobFailed
            expr: |
              kube_job_status_failed > 0
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} 执行失败"
              description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} 执行失败"

      # ==================== 可观测性组件告警 ====================
      - name: observability-alerts
        interval: 30s
        rules:
          # Prometheus 采集失败
          - alert: PrometheusScrapeFailure
            expr: |
              up == 0
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Prometheus 无法采集目标 {{ $labels.job }}"
              description: "Prometheus 无法采集目标 {{ $labels.job }} ({{ $labels.instance }})"

          # Loki 日志丢失
          - alert: LokiLogEntriesDropped
            expr: |
              rate(loki_distributor_lines_received_total[5m]) > rate(loki_ingester_lines_received_total[5m]) * 1.1
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "Loki 可能正在丢失日志"
              description: "Loki distributor 接收的日志量明显高于 ingester，可能存在日志丢失"