gao/ljwx-health
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): implement enterprise-grade k3s monitoring guardrails

Browse Source
This commit is contained in:
BrunoGaoSZ
2026-02-21 08:46:03 +01:00
parent e8256d5dbe
commit bc5593ed8b
11 changed files with 517 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/observability-guardrails.yml
+72
View File
@@ -0,0 +1,72 @@
name: Observability Guardrails
on:
pull_request:
paths:
- "deploy/monitoring/**"
- "scripts/check-observability-k3s.sh"
- "scripts/deploy-observability.sh"
- "docs/04-operations/monitoring/**"
push:
branches: [main]
paths:
- "deploy/monitoring/**"
- "scripts/check-observability-k3s.sh"
- "scripts/deploy-observability.sh"
- "docs/04-operations/monitoring/**"
jobs:
validate-observability:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Validate shell scripts syntax
run: |
bash -n scripts/check-observability-k3s.sh
bash -n scripts/deploy-observability.sh
- name: Validate dashboard JSON
run: |
jq -e . deploy/monitoring/grafana/provisioning/dashboards/ljwx-boot-dashboard.json >/dev/null
jq -e . deploy/monitoring/grafana/provisioning/dashboards/ljwx-bigscreen-dashboard.json >/dev/null
jq -e . deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json >/dev/null
- name: Validate YAML structure and required resources
run: |
python - <<'PY'
import sys, yaml, pathlib
files = [
"deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml",
"deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml",
"deploy/monitoring/grafana/grafana.yaml",
]
for f in files:
p = pathlib.Path(f)
if not p.exists():
raise SystemExit(f"missing required file: {f}")
with p.open("r", encoding="utf-8") as fp:
docs = [d for d in yaml.safe_load_all(fp) if d]
if not docs:
raise SystemExit(f"empty yaml: {f}")
with open("deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml", "r", encoding="utf-8") as fp:
rule = yaml.safe_load(fp)
if rule.get("kind") != "PrometheusRule":
raise SystemExit("ljwx-health-k3s-prometheusrule.yaml kind must be PrometheusRule")
groups = rule.get("spec", {}).get("groups", [])
names = {g.get("name") for g in groups}
required = {
"ljwx_health_k3s_availability",
"ljwx_health_k3s_latency_and_errors",
"ljwx_health_k3s_dependencies",
"ljwx_health_k3s_recording_rules",
"ljwx_health_k3s_slo_alerts",
}
missing = required - names
if missing:
raise SystemExit(f"PrometheusRule missing groups: {sorted(missing)}")
print("observability yaml validation passed")
PY
deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml
+54
View File
@@ -0,0 +1,54 @@
apiVersion: monitoring.coreos.com/v1alpha1
kind: AlertmanagerConfig
metadata:
name: ljwx-health-routing
namespace: monitoring
labels:
release: monitoring
app.kubernetes.io/name: ljwx-health
spec:
route:
groupBy:
- alertname
- service
- category
- severity
groupWait: 30s
groupInterval: 5m
repeatInterval: 2h
receiver: ljwx-health-warning
routes:
- matchers:
- name: severity
value: critical
matchType: "="
receiver: ljwx-health-critical
continue: false
- matchers:
- name: severity
value: warning
matchType: "="
receiver: ljwx-health-warning
continue: false
- matchers:
- name: severity
value: info
matchType: "="
receiver: ljwx-health-info
continue: false
receivers:
- name: ljwx-health-critical
webhookConfigs:
- url: http://alert-webhook.monitoring.svc.cluster.local:8080/critical
sendResolved: true
maxAlerts: 0
- name: ljwx-health-warning
webhookConfigs:
- url: http://alert-webhook.monitoring.svc.cluster.local:8080/warning
sendResolved: true
maxAlerts: 0
- name: ljwx-health-info
webhookConfigs:
- url: http://alert-webhook.monitoring.svc.cluster.local:8080/info
sendResolved: true
maxAlerts: 0
deploy/monitoring/grafana/README.md
+5
View File
@@ -83,6 +83,11 @@ kubectl create configmap grafana-dashboard-ljwx-bigscreen \
--from-file=ljwx-bigscreen-dashboard.json=deploy/monitoring/grafana/provisioning/dashboards/ljwx-bigscreen-dashboard.json \
-n infra \
--dry-run=client -o yaml | kubectl apply -f -
kubectl create configmap grafana-dashboard-ljwx-health-enterprise-slo \
--from-file=ljwx-health-enterprise-slo-dashboard.json=deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json \
-n infra \
--dry-run=client -o yaml | kubectl apply -f -
```
### 方法 2: 更新 Grafana Deployment
deploy/monitoring/grafana/grafana.yaml
+12
View File
@@ -217,6 +217,12 @@ spec:
subPath: ljwx-bigscreen-dashboard.json
readOnly: true
# 挂载 Enterprise SLO Dashboard JSON
- name: dashboard-ljwx-health-enterprise-slo
mountPath: /etc/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json
subPath: ljwx-health-enterprise-slo-dashboard.json
readOnly: true
# 挂载数据持久化目录（可选）
- name: grafana-storage
mountPath: /var/lib/grafana
@@ -272,6 +278,12 @@ spec:
name: grafana-dashboard-ljwx-bigscreen
defaultMode: 0644
# Enterprise SLO Dashboard ConfigMap
- name: dashboard-ljwx-health-enterprise-slo
configMap:
name: grafana-dashboard-ljwx-health-enterprise-slo
defaultMode: 0644
# 数据持久化（可选，使用 emptyDir 或 PVC）
- name: grafana-storage
emptyDir: {}
deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json
+157
View File
@@ -0,0 +1,157 @@
{
"id": null,
"uid": "ljwx-health-enterprise-slo",
"title": "LJWX Health - Enterprise SLO",
"tags": ["ljwx", "slo", "enterprise"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"panels": [
{
"type": "stat",
"title": "Boot Error Ratio (5m)",
"gridPos": {"h": 6, "w": 6, "x": 0, "y": 0},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "ljwx_health:boot:http_error_ratio:rate5m",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null},
{"color": "yellow", "value": 0.001},
{"color": "red", "value": 0.005}
]
}
},
"overrides": []
},
"options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
},
{
"type": "stat",
"title": "Boot Request Rate (5m)",
"gridPos": {"h": 6, "w": 6, "x": 6, "y": 0},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "ljwx_health:boot:http_requests:rate5m",
"refId": "A"
}
],
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
"options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
},
{
"type": "stat",
"title": "Boot P95 Latency",
"gridPos": {"h": 6, "w": 6, "x": 12, "y": 0},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "ljwx_health:boot:http_latency_p95:5m",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null},
{"color": "yellow", "value": 0.5},
{"color": "red", "value": 1.0}
]
}
},
"overrides": []
},
"options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
},
{
"type": "stat",
"title": "Critical Alerts Firing",
"gridPos": {"h": 6, "w": 6, "x": 18, "y": 0},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "sum(ALERTS{alertstate=\"firing\",severity=\"critical\",namespace=\"monitoring\",alertname=~\"LjwxHealth.*\"})",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null},
{"color": "red", "value": 1}
]
}
},
"overrides": []
},
"options": {"reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""}}
},
{
"type": "timeseries",
"title": "SLO Error Ratio Trends",
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 6},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "ljwx_health:boot:http_error_ratio:rate5m",
"legendFormat": "5m",
"refId": "A"
},
{
"expr": "ljwx_health:boot:http_error_ratio:rate1h",
"legendFormat": "1h",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {
"legend": {"displayMode": "table", "placement": "bottom"},
"tooltip": {"mode": "multi"}
}
},
{
"type": "timeseries",
"title": "Availability/Saturation Signals",
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 6},
"datasource": {"type": "prometheus", "uid": "prometheus"},
"targets": [
{
"expr": "kube_deployment_status_replicas_available{namespace=\"ljwx-health\",deployment=~\"ljwx-health-(boot|admin|bigscreen)\"}",
"legendFormat": "{{deployment}} available",
"refId": "A"
},
{
"expr": "kube_deployment_spec_replicas{namespace=\"ljwx-health\",deployment=~\"ljwx-health-(boot|admin|bigscreen)\"}",
"legendFormat": "{{deployment}} desired",
"refId": "B"
}
],
"fieldConfig": {"defaults": {"unit": "none"}, "overrides": []},
"options": {
"legend": {"displayMode": "table", "placement": "bottom"},
"tooltip": {"mode": "multi"}
}
}
]
}
deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml
+79
View File
@@ -154,3 +154,82 @@ spec:
summary: "ljwx-health 副本未达到期望值"
description: "Deployment {{ $labels.deployment }} 可用副本持续低于期望值。"
runbook: "docs/04-operations/monitoring/observability-review-checklist-k3s.md"
- name: ljwx_health_k3s_recording_rules
interval: 30s
rules:
- record: ljwx_health:boot:http_requests:rate5m
expr: sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[5m]))
- record: ljwx_health:boot:http_errors_5xx:rate5m
expr: sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[5m]))
- record: ljwx_health:boot:http_error_ratio:rate5m
expr: |
ljwx_health:boot:http_errors_5xx:rate5m
/
clamp_min(ljwx_health:boot:http_requests:rate5m, 0.001)
- record: ljwx_health:boot:http_error_ratio:rate1h
expr: |
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[1h]))
/
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[1h])), 0.001)
- record: ljwx_health:boot:http_latency_p95:5m
expr: |
histogram_quantile(
0.95,
sum(rate(http_server_requests_seconds_bucket{namespace="ljwx-health",service="ljwx-health-boot"}[5m])) by (le)
)
- name: ljwx_health_k3s_slo_alerts
interval: 30s
rules:
- alert: LjwxHealthBootSLOErrorBudgetBurnFast
expr: |
(
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[5m]))
/
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[5m])), 0.001)
) > (14.4 * (1 - 0.999))
and
(
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[1h]))
/
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[1h])), 0.001)
) > (14.4 * (1 - 0.999))
for: 2m
labels:
severity: critical
category: slo
service: ljwx-health-boot
slo_target: "99.9"
annotations:
summary: "ljwx-health-boot 错误预算快速燃烧"
description: "5m/1h 多窗口 burn-rate 超标，SLO(99.9%) 可能快速失守。"
runbook: "docs/04-operations/monitoring/observability-review-checklist-k3s.md"
- alert: LjwxHealthBootSLOErrorBudgetBurnSlow
expr: |
(
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[30m]))
/
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[30m])), 0.001)
) > (6 * (1 - 0.999))
and
(
sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot",status=~"5.."}[6h]))
/
clamp_min(sum(rate(http_server_requests_seconds_count{namespace="ljwx-health",service="ljwx-health-boot"}[6h])), 0.001)
) > (6 * (1 - 0.999))
for: 15m
labels:
severity: warning
category: slo
service: ljwx-health-boot
slo_target: "99.9"
annotations:
summary: "ljwx-health-boot 错误预算持续燃烧"
description: "30m/6h 多窗口 burn-rate 超标，需在值班窗口内处理。"
runbook: "docs/04-operations/monitoring/observability-review-checklist-k3s.md"
docs/04-operations/monitoring/observability-enterprise-implementation.md
+88
View File
@@ -0,0 +1,88 @@
# 企业级可观测性实施方案（LJWX Health）
## 目标
- 建立企业级三支柱能力：Metrics、Logs、Traces
- 建立 SLO 与错误预算治理机制（99.9% 可用性目标）
- 建立值班可执行机制：分级告警、路由、Runbook、门禁脚本
## 已落地资产
- Prometheus 告警与记录规则：
- `deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml`
- Alertmanager 路由配置（CRD）：
- `deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml`
- SLO 总览仪表盘：
- `deploy/monitoring/grafana/provisioning/dashboards/ljwx-health-enterprise-slo-dashboard.json`
- 发布门禁脚本：
- `scripts/check-observability-k3s.sh`
- 监控变更 CI 守护：
- `.github/workflows/observability-guardrails.yml`
## 企业级能力映射
### 1) 可用性治理
- Deployment 不可用告警（boot/admin/bigscreen）
- CrashLoopBackOff 告警
- 副本覆盖率不足告警
### 2) 质量治理（错误率与延迟）
- 5xx 错误率告警（warning/critical）
- P95 延迟告警
- SLI recording rules：
- 请求率
- 错误率（5m/1h）
- P95 延迟
### 3) SLO 与错误预算
- SLO 目标：可用性 99.9%
- 多窗口 burn-rate 告警：
- Fast burn：5m/1h（critical）
- Slow burn：30m/6h（warning）
### 4) 值班分级路由
- severity = critical/warning/info 分流
- 支持 resolved 回调，减少人工噪音
## 部署步骤（生产/准生产）
```bash
# 1. 应用告警和记录规则
kubectl apply -f deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml
# 2. 应用 Alertmanager 路由
kubectl apply -f deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml
# 3. 部署/刷新 Grafana（含 enterprise SLO dashboard）
bash scripts/deploy-observability.sh
# 4. 运行门禁验收
./scripts/check-observability-k3s.sh
```
## 发布门禁（必须全部通过）
- ArgoCD：`Synced + Healthy + selfHeal=true`
- 业务核心服务：`boot/admin/bigscreen` 均为 `Ready`
- PrometheusRule 存在并可被 Prometheus Operator 校验
- 对外健康探针返回 `HTTP 200`
## 运行期管理建议
- 值班流程：
- Critical：5 分钟内响应，必须回填事件编号
- Warning：30 分钟内确认影响与处置
- Info：日内归档处理
- 周期审计：
- 每周审计 Top 10 告警噪音来源
- 每周审计 SLO burn 事件与根因分类
- 每月审计阈值是否与业务峰值匹配
## 已知边界
- AlertmanagerConfig 中 webhook 地址为占位服务，需要替换成企业内部通知网关。
- 若集群未安装 `AlertmanagerConfig` CRD，路由对象不会生效，但不会影响 PrometheusRule。
docs/04-operations/monitoring/observability-guide.md
+1
View File
@@ -7,6 +7,7 @@
- 评审清单: `docs/04-operations/monitoring/observability-review-checklist-k3s.md`
- 门禁脚本: `scripts/check-observability-k3s.sh`
- K3s 告警规则: `deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml`
- 企业级实施方案: `docs/04-operations/monitoring/observability-enterprise-implementation.md`
## 概述
scripts/apply-observability-enterprise.sh
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
echo "[1/4] Apply PrometheusRule"
kubectl apply -f deploy/monitoring/prometheus/alerts/ljwx-health-k3s-prometheusrule.yaml
echo "[2/4] Apply AlertmanagerConfig"
kubectl apply -f deploy/monitoring/alertmanager/ljwx-health-alertmanagerconfig.yaml
echo "[3/4] Deploy/refresh Grafana observability assets"
bash scripts/deploy-observability.sh
echo "[4/4] Run observability gate"
./scripts/check-observability-k3s.sh
echo "Observability enterprise rollout completed."
scripts/check-observability-k3s.sh
+22 -2
View File
@@ -4,6 +4,7 @@ set -u
NS_APP="${NS_APP:-ljwx-health}"
NS_ARGO="${NS_ARGO:-argocd}"
NS_GRAFANA="${NS_GRAFANA:-monitoring}"
APP_NAME="${APP_NAME:-ljwx-health-k3s-eu}"
API_HEALTH_URL="${API_HEALTH_URL:-https://health-admin.lingjingwanxiang.cn/api/actuator/health}"
@@ -74,7 +75,26 @@ else
bad "ServiceMonitor 数量不足：$sm_count"
fi
info "4) 对外健康检查"
info "4) 规则与路由资源"
if kubectl -n monitoring get prometheusrule ljwx-health-k3s-alerts >/dev/null 2>&1; then
ok "PrometheusRule ljwx-health-k3s-alerts 存在"
else
bad "PrometheusRule ljwx-health-k3s-alerts 缺失"
fi
if kubectl -n monitoring get alertmanagerconfig ljwx-health-routing >/dev/null 2>&1; then
ok "AlertmanagerConfig ljwx-health-routing 存在"
else
warn "AlertmanagerConfig ljwx-health-routing 缺失（若未启用该 CRD 可忽略）"
fi
if kubectl -n "$NS_GRAFANA" get configmap grafana-dashboard-ljwx-health-enterprise-slo >/dev/null 2>&1; then
ok "Grafana enterprise SLO dashboard ConfigMap 存在"
else
warn "Grafana enterprise SLO dashboard ConfigMap 缺失（请运行 scripts/deploy-observability.sh）"
fi
info "5) 对外健康检查"
http_code="$(curl -skI --max-time 8 "$API_HEALTH_URL" | awk '/^HTTP/{print $2}' | tail -n 1)"
if [[ "$http_code" == "200" ]]; then
ok "API health 返回 HTTP 200"
@@ -82,7 +102,7 @@ else
bad "API health 返回 HTTP ${http_code:-N/A}"
fi
info "5) 近 10 分钟 boot 错误日志扫描"
info "6) 近 10 分钟 boot 错误日志扫描"
err_lines="$(kubectl -n "$NS_APP" logs deploy/ljwx-health-boot --since=10m 2>/dev/null | grep -E 'ERROR|Exception' | wc -l | tr -d ' ')"
if [[ "$err_lines" -eq 0 ]]; then
ok "boot 最近 10 分钟无 ERROR/Exception"
scripts/deploy-observability.sh
+10
View File
@@ -165,6 +165,16 @@ deploy_grafana() {
log_warning " ✗ ljwx-bigscreen-dashboard.json 未找到"
fi
if [ -f "$DASHBOARD_DIR/ljwx-health-enterprise-slo-dashboard.json" ]; then
kubectl create configmap grafana-dashboard-ljwx-health-enterprise-slo \
--from-file=ljwx-health-enterprise-slo-dashboard.json="$DASHBOARD_DIR/ljwx-health-enterprise-slo-dashboard.json" \
-n $NAMESPACE_INFRA \
--dry-run=client -o yaml | kubectl apply -f -
log_success " ✓ ljwx-health Enterprise SLO Dashboard ConfigMap 已创建"
else
log_warning " ✗ ljwx-health-enterprise-slo-dashboard.json 未找到"
fi
# 2. 部署 Grafana Deployment 和 Service
log_info "部署 Grafana Deployment 和 Service..."
kubectl apply -f "$MONITORING_DIR/grafana/grafana.yaml" -n $NAMESPACE_INFRA