fix(test): align nightly and local system testing with k3s

This commit is contained in:
BrunoGaoSZ
2026-03-07 21:00:41 +08:00
committed by BrunoGao
parent e30da7fd19
commit cf55b2caa4
6 changed files with 444 additions and 32 deletions
+83 -10
View File
@@ -13,10 +13,40 @@ permissions:
jobs:
nightly_regression:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_DB: ljwx_platform
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres -d ljwx_platform"
--health-interval 5s
--health-timeout 3s
--health-retries 20
redis:
image: redis:7-alpine
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 3s
--health-retries 20
env:
GH_TOKEN: ${{ github.token }}
BASE_URL: http://localhost:8080
ENABLE_E2E: "1"
LOGIN_PATH: /api/auth/login
FORBIDDEN_PATH: /api/users
RESOURCE_LIST_PATH: /api/v1/menus
TENANT_A_USER: admin
TENANT_A_PASS: Admin@12345
TENANT_B_USER: tenantB_admin
TENANT_B_PASS: Admin@12345
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -36,23 +66,56 @@ jobs:
command -v gh >/dev/null
command -v docker >/dev/null
- name: Start MySQL/Redis test environment
- name: Build and install all modules
shell: bash
run: |
set -euo pipefail
START_APP_CONTAINER=false bash scripts/setup-test-env.sh up
mvn install -DskipTests -q
- name: Wait for PostgreSQL
shell: bash
run: |
set -euo pipefail
for i in {1..30}; do
if (echo > /dev/tcp/127.0.0.1/5432) >/dev/null 2>&1; then
echo "PostgreSQL is ready"
exit 0
fi
sleep 2
done
echo "PostgreSQL is not reachable on 127.0.0.1:5432" >&2
exit 1
- name: Wait for Redis
shell: bash
run: |
set -euo pipefail
for i in {1..30}; do
if (echo > /dev/tcp/127.0.0.1/6379) >/dev/null 2>&1; then
echo "Redis is ready"
exit 0
fi
sleep 2
done
echo "Redis is not reachable on 127.0.0.1:6379" >&2
exit 1
- name: Start app locally
shell: bash
run: |
set -euo pipefail
SPRING_PROFILES_ACTIVE=test \
SPRING_DATASOURCE_URL='jdbc:mysql://127.0.0.1:3307/ljwx_platform?useUnicode=true&characterEncoding=UTF-8&serverTimezone=UTC' \
SPRING_DATASOURCE_USERNAME=ljwx \
SPRING_DATASOURCE_PASSWORD=ljwx \
SPRING_REDIS_HOST=127.0.0.1 \
SPRING_REDIS_PORT=6380 \
mvn -f ljwx-platform-app/pom.xml -DskipTests spring-boot:run > app.log 2>&1 &
SPRING_PROFILES_ACTIVE=test,dev \
SPRING_DATASOURCE_URL='jdbc:postgresql://127.0.0.1:5432/ljwx_platform' \
SPRING_DATASOURCE_USERNAME=postgres \
SPRING_DATASOURCE_PASSWORD=postgres \
REDIS_HOST=127.0.0.1 \
REDIS_PORT=6379 \
mvn -f ljwx-platform-app/pom.xml \
-e \
-DskipTests \
-Dspring-boot.run.profiles=test,dev \
-Dspring-boot.run.useTestClasspath=true \
spring-boot:run > app.log 2>&1 &
echo $! > app.pid
- name: Wait app health
@@ -60,6 +123,11 @@ jobs:
run: |
set -euo pipefail
for i in {1..240}; do
if [[ -f app.pid ]] && ! kill -0 "$(cat app.pid)" 2>/dev/null; then
echo "app process exited before health check passed" >&2
tail -n 300 app.log || true
exit 1
fi
if curl -fsS "${BASE_URL}/actuator/health" >/dev/null 2>&1; then
echo "app is healthy"
exit 0
@@ -70,6 +138,12 @@ jobs:
tail -n 200 app.log || true
exit 1
- name: Seed E2E fixtures
shell: bash
run: |
set -euo pipefail
bash scripts/e2e/seed-fixtures.sh
- name: Run R10 E2E
id: r10
continue-on-error: true
@@ -154,7 +228,6 @@ jobs:
run: |
set -euo pipefail
if [[ -f app.pid ]]; then kill "$(cat app.pid)" || true; fi
bash scripts/setup-test-env.sh clean || true
- name: Fail workflow on regression
if: always()
+10 -2
View File
@@ -95,7 +95,7 @@ pnpm dev
数据大屏默认访问 `http://localhost:5174`。
### 6. 一键 Docker Compose(推荐端到端验收)
### 6. 一键 Docker Compose(快速本地栈)
```bash
cp .env.compose.example .env.compose
@@ -143,12 +143,15 @@ DEPLOY_MODE=delivery bash scripts/local/compose-stack.sh e2e
DEPLOY_MODE=delivery bash scripts/local/compose-stack.sh down
```
### 8. 本地 k3s 纯交付件部署(不依赖 Argo)
### 8. 本地 k3s 纯交付件部署与系统测试(推荐)
```bash
cp .env.k3s.delivery.example .env.k3s.delivery
# 必填:镜像引用、数据库口令;按需设置拉取凭据
bash scripts/local/k3s-delivery.sh apply
bash scripts/local/k3s-delivery.sh smoke
bash scripts/local/k3s-delivery.sh e2e
bash scripts/local/k3s-delivery.sh perf
bash scripts/local/k3s-delivery.sh status
```
@@ -157,6 +160,8 @@ bash scripts/local/k3s-delivery.sh status
- Secret 参考模板:`deploy/k3s/artifact/ljwx-platform-db.secret.example.yaml`
- `MANAGE_DB_SECRET=1` 时脚本会根据 `.env.k3s.delivery` 创建/更新 `ljwx-platform-db`
- `MANAGE_PULL_SECRET=1` 时脚本会创建镜像拉取 secret 并补丁 default service account
- `smoke` / `e2e` / `perf` 通过 `kubectl port-forward` 访问本地 `127.0.0.1`
- `system-test` 可串行执行 `smoke + e2e + perf`
### 9. 运行 Gate 检查
@@ -357,6 +362,9 @@ ljwx-platform/
若只需要验证“交付件可直接落地 k3s”(不经过 Argo),可使用:
- `bash scripts/local/k3s-delivery.sh apply`
- `bash scripts/local/k3s-delivery.sh smoke`
- `bash scripts/local/k3s-delivery.sh e2e`
- `bash scripts/local/k3s-delivery.sh perf`
- `bash scripts/local/k3s-delivery.sh status`
## Grafana 可观测看板(日志 + 链路 + 指标)
+33 -10
View File
@@ -12,31 +12,55 @@ Cross-tenant contract is fixed:
## Local Run
1. Start test environment:
本地 system-testing 默认走 k3s 纯交付件路径,不再以 `setup-test-env.sh` / docker-compose 作为主入口。
1. Apply delivery artifacts to local k3s:
```bash
bash scripts/setup-test-env.sh up
cp .env.k3s.delivery.example .env.k3s.delivery
bash scripts/local/k3s-delivery.sh apply
```
2. Run E2E smoke:
2. Run smoke checks through local port-forward:
```bash
BASE_URL=http://localhost:8080 bash tests/k6-run.sh e2e-01
BASE_URL=http://localhost:8080 bash tests/k6-run.sh e2e-02
bash scripts/local/k3s-delivery.sh smoke
```
3. Run performance baseline:
3. Seed fixtures and run `R10`:
```bash
BASE_URL=http://localhost:8080 bash tests/k6-run.sh perf-baseline
bash scripts/local/k3s-delivery.sh e2e
```
4. Stop environment:
4. Run `R11`:
```bash
bash scripts/setup-test-env.sh down
bash scripts/local/k3s-delivery.sh perf
```
5. Run the full local system-testing chain:
```bash
bash scripts/local/k3s-delivery.sh system-test
```
6. Delete the local test namespace when finished:
```bash
bash scripts/local/k3s-delivery.sh delete
```
说明:
- `smoke` / `e2e` / `perf` 都通过 `kubectl port-forward` 访问本地 `127.0.0.1`
- `seed` 在集群内启动临时 PostgreSQL client Pod 写入夹具,不依赖本机 Docker 容器
- 若本地端口 `18080/18081/18082` 被占用,可通过环境变量 `BACKEND_LOCAL_PORT` / `ADMIN_LOCAL_PORT` / `SCREEN_LOCAL_PORT` 覆盖
## Legacy Compose Path
如果当前机器没有 k3s,可继续使用 compose 仅做临时调试,但这不再是 system-testing 的标准本地路径。
## Gate Integration
- `scripts/gates/gate-e2e.sh` writes `/tmp/ljwx-gate-results/R10.json`
@@ -54,4 +78,3 @@ bash scripts/setup-test-env.sh down
1. Observation phase: collect p95/avg only (`R11` pass unless execution error)
2. Alert phase: comment + issue when p95 crosses baseline band
3. Blocking phase: enforce threshold in gate after baseline stabilizes
+113 -9
View File
@@ -3,19 +3,22 @@ set -euo pipefail
DB_NAME="${DB_NAME:-ljwx_platform}"
DB_USERNAME="${DB_USERNAME:-postgres}"
DB_HOST="${DB_HOST:-127.0.0.1}"
DB_PORT="${DB_PORT:-5432}"
DB_PASSWORD="${DB_PASSWORD:-}"
POSTGRES_IMAGE_FILTER="${POSTGRES_IMAGE_FILTER:-postgres:16-alpine}"
POSTGRES_CONTAINER_ID="${POSTGRES_CONTAINER_ID:-}"
PSQL_MODE="${PSQL_MODE:-auto}"
K8S_NAMESPACE="${K8S_NAMESPACE:-}"
K8S_PSQL_IMAGE="${K8S_PSQL_IMAGE:-postgres:16-alpine}"
if [[ -z "${POSTGRES_CONTAINER_ID}" ]]; then
POSTGRES_CONTAINER_ID="$(docker ps --filter "ancestor=${POSTGRES_IMAGE_FILTER}" --format '{{.ID}}' | head -n1)"
fi
TMP_SQL_FILE="$(mktemp)"
cleanup() {
rm -f "${TMP_SQL_FILE}"
}
trap cleanup EXIT
if [[ -z "${POSTGRES_CONTAINER_ID}" ]]; then
echo "未找到 PostgreSQL 容器,请确认 services.postgres 已启动" >&2
exit 1
fi
docker exec -i "${POSTGRES_CONTAINER_ID}" psql -v ON_ERROR_STOP=1 -U "${DB_USERNAME}" -d "${DB_NAME}" <<'SQL'
cat >"${TMP_SQL_FILE}" <<'SQL'
-- Ensure tenant B exists with tenant_id=200001, and keep seeded entities aligned.
INSERT INTO sys_tenant (id, name, code, status, tenant_id, created_by, updated_by, version)
VALUES (200001, 'Tenant B', 'tenant_b', 1, 0, 0, 0, 1)
@@ -125,4 +128,105 @@ WHERE NOT EXISTS (
);
SQL
run_via_docker() {
local container_id="${POSTGRES_CONTAINER_ID}"
if [[ -z "${container_id}" && -n "$(command -v docker || true)" ]]; then
container_id="$(docker ps --filter "ancestor=${POSTGRES_IMAGE_FILTER}" --format '{{.ID}}' | head -n1)"
fi
if [[ -z "${container_id}" ]]; then
echo "未找到 PostgreSQL 容器,请确认 compose/services.postgres 已启动" >&2
return 1
fi
docker exec -i "${container_id}" psql -v ON_ERROR_STOP=1 -U "${DB_USERNAME}" -d "${DB_NAME}" <"${TMP_SQL_FILE}"
}
run_via_local_psql() {
if ! command -v psql >/dev/null 2>&1; then
echo "未安装 psql,无法使用本地 PostgreSQL 直连写入夹具" >&2
return 1
fi
if [[ -z "${DB_PASSWORD}" ]]; then
echo "DB_PASSWORD 为空,无法使用本地 PostgreSQL 直连写入夹具" >&2
return 1
fi
PGPASSWORD="${DB_PASSWORD}" psql \
-v ON_ERROR_STOP=1 \
-h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USERNAME}" \
-d "${DB_NAME}" \
-f "${TMP_SQL_FILE}"
}
run_via_kubectl() {
if ! command -v kubectl >/dev/null 2>&1; then
echo "未安装 kubectl,无法通过 k3s/Kubernetes 临时 Pod 写入夹具" >&2
return 1
fi
if [[ -z "${K8S_NAMESPACE}" ]]; then
echo "K8S_NAMESPACE 未设置,无法通过 k3s/Kubernetes 临时 Pod 写入夹具" >&2
return 1
fi
if [[ -z "${DB_PASSWORD}" ]]; then
echo "DB_PASSWORD 为空,无法通过 k3s/Kubernetes 临时 Pod 写入夹具" >&2
return 1
fi
local pod_name
pod_name="ljwx-seed-fixtures-$(date +%s)"
set +e
kubectl -n "${K8S_NAMESPACE}" run "${pod_name}" \
--image="${K8S_PSQL_IMAGE}" \
--restart=Never \
--rm -i --attach=true \
--env="PGPASSWORD=${DB_PASSWORD}" \
--command -- \
psql \
-v ON_ERROR_STOP=1 \
-h "${DB_HOST}" \
-p "${DB_PORT}" \
-U "${DB_USERNAME}" \
-d "${DB_NAME}" <"${TMP_SQL_FILE}"
local rc=$?
set -e
kubectl -n "${K8S_NAMESPACE}" delete pod "${pod_name}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
return "${rc}"
}
case "${PSQL_MODE}" in
auto)
if run_via_docker; then
:
elif [[ -n "${K8S_NAMESPACE}" ]] && run_via_kubectl; then
:
elif run_via_local_psql; then
:
else
echo "E2E fixture 写入失败:未找到可用 PostgreSQL 访问方式(docker / kubectl / psql)" >&2
exit 1
fi
;;
docker)
run_via_docker
;;
psql)
run_via_local_psql
;;
kubectl)
run_via_kubectl
;;
*)
echo "PSQL_MODE 非法: ${PSQL_MODE},仅支持 auto/docker/psql/kubectl" >&2
exit 1
;;
esac
echo "E2E fixtures seeded for tenantB_admin"
+24 -1
View File
@@ -17,7 +17,30 @@ run_k6() {
return
fi
if command -v docker >/dev/null 2>&1; then
docker run --rm -i -v "$PROJECT_ROOT:/work" -w /work grafana/k6:0.49.0 "$@"
local network_args=()
local env_args=()
if [[ -n "${K6_DOCKER_NETWORK:-}" ]]; then
network_args+=(--network "${K6_DOCKER_NETWORK}")
elif [[ "$(uname -s)" == "Linux" ]]; then
network_args+=(--network host)
fi
local env_key
for env_key in \
BASE_URL LOGIN_PATH TENANT_A_USER TENANT_A_PASS \
K6_VUS K6_DURATION PERF_ENDPOINTS; do
if [[ -n "${!env_key:-}" ]]; then
env_args+=(-e "${env_key}")
fi
done
docker run --rm -i \
"${network_args[@]}" \
"${env_args[@]}" \
-v "$PROJECT_ROOT:/work" \
-w /work \
grafana/k6:0.49.0 "$@"
return
fi
echo "k6 is not available and docker fallback is unavailable" >&2
+181
View File
@@ -40,6 +40,36 @@ set +a
: "${REGISTRY_USERNAME:=}"
: "${REGISTRY_PASSWORD:=}"
: "${ROLLOUT_TIMEOUT:=300s}"
: "${BACKEND_SERVICE_NAME:=ljwx-platform}"
: "${ADMIN_SERVICE_NAME:=ljwx-platform-admin-ui}"
: "${SCREEN_SERVICE_NAME:=ljwx-platform-screen}"
: "${BACKEND_LOCAL_PORT:=18080}"
: "${ADMIN_LOCAL_PORT:=18081}"
: "${SCREEN_LOCAL_PORT:=18082}"
: "${SERVICE_PORT:=80}"
: "${TENANT_A_USER:=admin}"
: "${TENANT_A_PASS:=Admin@12345}"
: "${TENANT_B_USER:=tenantB_admin}"
: "${TENANT_B_PASS:=Admin@12345}"
: "${LOGIN_PATH:=/api/auth/login}"
: "${FORBIDDEN_PATH:=/api/users}"
: "${RESOURCE_LIST_PATH:=/api/v1/menus}"
: "${K8S_PSQL_IMAGE:=postgres:16-alpine}"
PORT_FORWARD_PIDS=()
cleanup_port_forwards() {
local pid
for pid in "${PORT_FORWARD_PIDS[@]:-}"; do
if [[ -n "${pid}" ]]; then
kill "${pid}" >/dev/null 2>&1 || true
wait "${pid}" >/dev/null 2>&1 || true
fi
done
PORT_FORWARD_PIDS=()
}
trap cleanup_port_forwards EXIT
require_command() {
local cmd="$1"
@@ -170,6 +200,137 @@ print_logs() {
kubectl -n "${NAMESPACE}" logs deploy/ljwx-platform --tail=200
}
decode_secret_value() {
local secret_name="$1"
local key="$2"
kubectl -n "${NAMESPACE}" get secret "${secret_name}" -o "jsonpath={.data.${key}}" 2>/dev/null | base64 -d 2>/dev/null || true
}
resolve_db_credentials() {
if ! kubectl -n "${NAMESPACE}" get secret "${DB_SECRET_NAME}" >/dev/null 2>&1; then
return
fi
if [[ "${MANAGE_DB_SECRET}" != "1" || -z "${DB_PASSWORD}" || "${DB_PASSWORD}" == *"CHANGE_ME"* ]]; then
local secret_username secret_password
secret_username="$(decode_secret_value "${DB_SECRET_NAME}" DB_USERNAME)"
secret_password="$(decode_secret_value "${DB_SECRET_NAME}" DB_PASSWORD)"
if [[ -n "${secret_username}" ]]; then
DB_USERNAME="${secret_username}"
fi
if [[ -n "${secret_password}" ]]; then
DB_PASSWORD="${secret_password}"
fi
fi
}
start_port_forward() {
local service_name="$1"
local local_port="$2"
local service_port="$3"
local health_url="$4"
local log_file
log_file="$(mktemp "/tmp/${service_name}.port-forward.XXXXXX.log")"
kubectl -n "${NAMESPACE}" port-forward "svc/${service_name}" "${local_port}:${service_port}" >"${log_file}" 2>&1 &
local pf_pid=$!
PORT_FORWARD_PIDS+=("${pf_pid}")
local i
for i in $(seq 1 120); do
if curl -fsS "${health_url}" >/dev/null 2>&1; then
echo "${service_name} 已通过 port-forward 就绪: ${health_url}"
return 0
fi
if ! kill -0 "${pf_pid}" >/dev/null 2>&1; then
echo "port-forward 进程异常退出: svc/${service_name}" >&2
cat "${log_file}" >&2 || true
return 1
fi
sleep 2
done
echo "port-forward 健康检查超时: svc/${service_name} -> ${health_url}" >&2
cat "${log_file}" >&2 || true
return 1
}
run_smoke() {
require_command kubectl
require_command curl
cleanup_port_forwards
start_port_forward "${BACKEND_SERVICE_NAME}" "${BACKEND_LOCAL_PORT}" "${SERVICE_PORT}" "http://127.0.0.1:${BACKEND_LOCAL_PORT}/actuator/health"
start_port_forward "${ADMIN_SERVICE_NAME}" "${ADMIN_LOCAL_PORT}" "${SERVICE_PORT}" "http://127.0.0.1:${ADMIN_LOCAL_PORT}/"
start_port_forward "${SCREEN_SERVICE_NAME}" "${SCREEN_LOCAL_PORT}" "${SERVICE_PORT}" "http://127.0.0.1:${SCREEN_LOCAL_PORT}/"
echo "k3s smoke 检查通过"
cleanup_port_forwards
}
seed_fixtures() {
require_command kubectl
resolve_db_credentials
if [[ -z "${DB_PASSWORD}" || "${DB_PASSWORD}" == *"CHANGE_ME"* ]]; then
echo "缺少可用 DB_PASSWORD,无法在 k3s 集群内写入 E2E 夹具" >&2
exit 1
fi
PSQL_MODE=kubectl \
K8S_NAMESPACE="${NAMESPACE}" \
K8S_PSQL_IMAGE="${K8S_PSQL_IMAGE}" \
DB_HOST="${DB_HOST}" \
DB_PORT="${DB_PORT}" \
DB_NAME="${DB_NAME}" \
DB_USERNAME="${DB_USERNAME}" \
DB_PASSWORD="${DB_PASSWORD}" \
bash "${PROJECT_ROOT}/scripts/e2e/seed-fixtures.sh"
}
run_e2e() {
require_command kubectl
require_command curl
cleanup_port_forwards
seed_fixtures
start_port_forward "${BACKEND_SERVICE_NAME}" "${BACKEND_LOCAL_PORT}" "${SERVICE_PORT}" "http://127.0.0.1:${BACKEND_LOCAL_PORT}/actuator/health"
K6_FORCE_DOCKER="${K6_FORCE_DOCKER:-0}" \
BASE_URL="http://127.0.0.1:${BACKEND_LOCAL_PORT}" \
TENANT_A_USER="${TENANT_A_USER}" \
TENANT_A_PASS="${TENANT_A_PASS}" \
TENANT_B_USER="${TENANT_B_USER}" \
TENANT_B_PASS="${TENANT_B_PASS}" \
LOGIN_PATH="${LOGIN_PATH}" \
FORBIDDEN_PATH="${FORBIDDEN_PATH}" \
RESOURCE_LIST_PATH="${RESOURCE_LIST_PATH}" \
bash "${PROJECT_ROOT}/scripts/gates/gate-e2e.sh"
cleanup_port_forwards
}
run_perf() {
require_command kubectl
require_command curl
cleanup_port_forwards
start_port_forward "${BACKEND_SERVICE_NAME}" "${BACKEND_LOCAL_PORT}" "${SERVICE_PORT}" "http://127.0.0.1:${BACKEND_LOCAL_PORT}/actuator/health"
BASE_URL="http://127.0.0.1:${BACKEND_LOCAL_PORT}" \
TENANT_A_USER="${TENANT_A_USER}" \
TENANT_A_PASS="${TENANT_A_PASS}" \
LOGIN_PATH="${LOGIN_PATH}" \
bash "${PROJECT_ROOT}/scripts/gates/gate-perf.sh"
cleanup_port_forwards
}
run_system_tests() {
run_smoke
run_e2e
run_perf
}
run_apply() {
require_command kubectl
validate_images
@@ -195,6 +356,21 @@ case "${CMD}" in
require_command kubectl
print_logs
;;
smoke)
run_smoke
;;
seed)
seed_fixtures
;;
e2e)
run_e2e
;;
perf)
run_perf
;;
system-test)
run_system_tests
;;
render)
require_command kubectl
kubectl kustomize "${KUSTOMIZE_DIR}"
@@ -209,6 +385,11 @@ case "${CMD}" in
bash scripts/local/k3s-delivery.sh apply # 按交付镜像部署 backend/admin-ui/screen
bash scripts/local/k3s-delivery.sh status # 查看部署状态
bash scripts/local/k3s-delivery.sh logs # 查看 backend 日志
bash scripts/local/k3s-delivery.sh smoke # 通过 port-forward 做 backend/admin/screen 健康检查
bash scripts/local/k3s-delivery.sh seed # 在集群内写入 E2E 夹具
bash scripts/local/k3s-delivery.sh e2e # 通过 port-forward 执行 R10 E2E
bash scripts/local/k3s-delivery.sh perf # 通过 port-forward 执行 R11 基线
bash scripts/local/k3s-delivery.sh system-test # 串行执行 smoke + e2e + perf
bash scripts/local/k3s-delivery.sh render # 渲染 kustomize 清单
bash scripts/local/k3s-delivery.sh delete # 删除命名空间(慎用)
EOF