feat: add runner conversion scripts and strengthen cutover automation
This commit is contained in:
S
358
runners-conversion/periodVault/validate-test-quality.sh
Executable file
358
runners-conversion/periodVault/validate-test-quality.sh
Executable file
@@ -0,0 +1,358 @@
|
||||
#!/usr/bin/env bash
|
||||
# validate-test-quality.sh — Enforce anti-pattern regression thresholds for UI tests.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/validate-test-quality.sh [baseline_file]
|
||||
# scripts/validate-test-quality.sh --help
|
||||
#
|
||||
# Behavior:
|
||||
# - Loads metric baselines from JSON.
|
||||
# - Counts pattern matches in configured roots via ripgrep.
|
||||
# - Fails if any metric exceeds baseline + allowed_growth.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
DEFAULT_BASELINE_FILE="$PROJECT_ROOT/audit/test-quality-baseline.json"
|
||||
BASELINE_FILE="${1:-$DEFAULT_BASELINE_FILE}"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
validate-test-quality.sh: enforce UI test anti-pattern regression thresholds.
|
||||
|
||||
Usage:
|
||||
scripts/validate-test-quality.sh [baseline_file]
|
||||
scripts/validate-test-quality.sh --help
|
||||
EOF
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
FAILURES=0
|
||||
|
||||
fail() {
|
||||
echo -e "${RED}FAIL:${NC} $1"
|
||||
FAILURES=$((FAILURES + 1))
|
||||
}
|
||||
|
||||
pass() {
|
||||
echo -e "${GREEN}PASS:${NC} $1"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
fail "Missing required command: $1"
|
||||
fi
|
||||
}
|
||||
|
||||
relative_path() {
|
||||
local path="$1"
|
||||
if [[ "$path" == "$PROJECT_ROOT/"* ]]; then
|
||||
echo "${path#$PROJECT_ROOT/}"
|
||||
else
|
||||
echo "$path"
|
||||
fi
|
||||
}
|
||||
|
||||
count_matches() {
|
||||
local pattern="$1"
|
||||
local root="$2"
|
||||
local glob="$3"
|
||||
local output=""
|
||||
local status=0
|
||||
|
||||
set +e
|
||||
output="$(rg --count-matches --no-messages --pcre2 -N -g "$glob" "$pattern" "$root" 2>/dev/null)"
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ "$status" -eq 1 ]]; then
|
||||
echo "0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$status" -ne 0 ]]; then
|
||||
return "$status"
|
||||
fi
|
||||
|
||||
if [[ -z "$output" ]]; then
|
||||
echo "0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "$output" | awk -F: '{sum += $NF} END {print sum + 0}'
|
||||
}
|
||||
|
||||
count_matches_multiline() {
|
||||
local pattern="$1"
|
||||
local root="$2"
|
||||
local glob="$3"
|
||||
local output=""
|
||||
local status=0
|
||||
|
||||
set +e
|
||||
output="$(rg --count-matches --no-messages --pcre2 -U -N -g "$glob" "$pattern" "$root" 2>/dev/null)"
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ "$status" -eq 1 ]]; then
|
||||
echo "0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ "$status" -ne 0 ]]; then
|
||||
return "$status"
|
||||
fi
|
||||
|
||||
if [[ -z "$output" ]]; then
|
||||
echo "0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "$output" | awk -F: '{sum += $NF} END {print sum + 0}'
|
||||
}
|
||||
|
||||
list_metric_files() {
|
||||
local root="$1"
|
||||
local glob="$2"
|
||||
rg --files "$root" -g "$glob" 2>/dev/null || true
|
||||
}
|
||||
|
||||
count_swift_test_body_pattern_matches() {
|
||||
local pattern="$1"
|
||||
local root="$2"
|
||||
local glob="$3"
|
||||
local files=()
|
||||
|
||||
while IFS= read -r file_path; do
|
||||
[[ -n "$file_path" ]] && files+=("$file_path")
|
||||
done < <(list_metric_files "$root" "$glob")
|
||||
if [[ "${#files[@]}" -eq 0 ]]; then
|
||||
echo "0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
awk -v pattern="$pattern" '
|
||||
function update_depth(line, i, c) {
|
||||
for (i = 1; i <= length(line); i++) {
|
||||
c = substr(line, i, 1)
|
||||
if (c == "{") depth++
|
||||
else if (c == "}") depth--
|
||||
}
|
||||
}
|
||||
|
||||
/^[[:space:]]*func[[:space:]]+test[[:alnum:]_]+[[:space:]]*\(.*\)[[:space:]]*(throws)?[[:space:]]*\{/ {
|
||||
in_test = 1
|
||||
depth = 0
|
||||
update_depth($0)
|
||||
if ($0 ~ pattern) count++
|
||||
if (depth <= 0) {
|
||||
in_test = 0
|
||||
depth = 0
|
||||
}
|
||||
next
|
||||
}
|
||||
|
||||
{
|
||||
if (!in_test) next
|
||||
|
||||
if ($0 ~ pattern) count++
|
||||
update_depth($0)
|
||||
if (depth <= 0) {
|
||||
in_test = 0
|
||||
depth = 0
|
||||
}
|
||||
}
|
||||
|
||||
END { print count + 0 }
|
||||
' "${files[@]}"
|
||||
}
|
||||
|
||||
count_swift_empty_test_bodies() {
|
||||
local root="$1"
|
||||
local glob="$2"
|
||||
local files=()
|
||||
|
||||
while IFS= read -r file_path; do
|
||||
[[ -n "$file_path" ]] && files+=("$file_path")
|
||||
done < <(list_metric_files "$root" "$glob")
|
||||
if [[ "${#files[@]}" -eq 0 ]]; then
|
||||
echo "0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
awk '
|
||||
function update_depth(line, i, c) {
|
||||
for (i = 1; i <= length(line); i++) {
|
||||
c = substr(line, i, 1)
|
||||
if (c == "{") depth++
|
||||
else if (c == "}") depth--
|
||||
}
|
||||
}
|
||||
|
||||
function test_body_has_code(body, cleaned) {
|
||||
cleaned = body
|
||||
gsub(/\/\/.*/, "", cleaned)
|
||||
gsub(/[ \t\r\n{}]/, "", cleaned)
|
||||
return cleaned != ""
|
||||
}
|
||||
|
||||
/^[[:space:]]*func[[:space:]]+test[[:alnum:]_]+[[:space:]]*\(.*\)[[:space:]]*(throws)?[[:space:]]*\{/ {
|
||||
in_test = 1
|
||||
depth = 0
|
||||
body = ""
|
||||
update_depth($0)
|
||||
if (depth <= 0) {
|
||||
empty_count++
|
||||
in_test = 0
|
||||
depth = 0
|
||||
body = ""
|
||||
}
|
||||
next
|
||||
}
|
||||
|
||||
{
|
||||
if (!in_test) next
|
||||
|
||||
body = body $0 "\n"
|
||||
update_depth($0)
|
||||
|
||||
if (depth <= 0) {
|
||||
if (!test_body_has_code(body)) {
|
||||
empty_count++
|
||||
}
|
||||
in_test = 0
|
||||
depth = 0
|
||||
body = ""
|
||||
}
|
||||
}
|
||||
|
||||
END { print empty_count + 0 }
|
||||
' "${files[@]}"
|
||||
}
|
||||
|
||||
count_metric() {
|
||||
local mode="$1"
|
||||
local pattern="$2"
|
||||
local root="$3"
|
||||
local glob="$4"
|
||||
|
||||
case "$mode" in
|
||||
rg)
|
||||
count_matches "$pattern" "$root" "$glob"
|
||||
;;
|
||||
rg_multiline)
|
||||
count_matches_multiline "$pattern" "$root" "$glob"
|
||||
;;
|
||||
swift_test_body_pattern)
|
||||
count_swift_test_body_pattern_matches "$pattern" "$root" "$glob"
|
||||
;;
|
||||
swift_empty_test_bodies)
|
||||
count_swift_empty_test_bodies "$root" "$glob"
|
||||
;;
|
||||
*)
|
||||
echo "__INVALID_MODE__:$mode"
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
require_cmd jq
|
||||
require_cmd rg
|
||||
require_cmd awk
|
||||
|
||||
echo "=== Test Quality Gate ==="
|
||||
echo "Baseline: $(relative_path "$BASELINE_FILE")"
|
||||
|
||||
if [[ ! -f "$BASELINE_FILE" ]]; then
|
||||
fail "Baseline file not found: $(relative_path "$BASELINE_FILE")"
|
||||
echo ""
|
||||
echo -e "${RED}Test quality gate failed with $FAILURES issue(s).${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if jq -e '.metrics | type == "array" and length > 0' "$BASELINE_FILE" >/dev/null; then
|
||||
pass "Baseline includes metric definitions"
|
||||
else
|
||||
fail "Baseline file has no metrics"
|
||||
fi
|
||||
|
||||
while IFS= read -r metric; do
|
||||
[[ -z "$metric" ]] && continue
|
||||
|
||||
metric_id="$(jq -r '.id // empty' <<<"$metric")"
|
||||
description="$(jq -r '.description // empty' <<<"$metric")"
|
||||
mode="$(jq -r '.mode // "rg"' <<<"$metric")"
|
||||
root_rel="$(jq -r '.root // empty' <<<"$metric")"
|
||||
glob="$(jq -r '.glob // empty' <<<"$metric")"
|
||||
pattern="$(jq -r '.pattern // ""' <<<"$metric")"
|
||||
baseline="$(jq -r '.baseline // empty' <<<"$metric")"
|
||||
allowed_growth="$(jq -r '.allowed_growth // empty' <<<"$metric")"
|
||||
|
||||
if [[ -z "$metric_id" || -z "$root_rel" || -z "$glob" || -z "$baseline" || -z "$allowed_growth" ]]; then
|
||||
fail "Metric entry is missing required fields: $metric"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$mode" != "swift_empty_test_bodies" && -z "$pattern" ]]; then
|
||||
fail "Metric '$metric_id' requires non-empty pattern for mode '$mode'"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! [[ "$baseline" =~ ^[0-9]+$ ]]; then
|
||||
fail "Metric '$metric_id' has non-numeric baseline: $baseline"
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! [[ "$allowed_growth" =~ ^[0-9]+$ ]]; then
|
||||
fail "Metric '$metric_id' has non-numeric allowed_growth: $allowed_growth"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$root_rel" == /* ]]; then
|
||||
root_path="$root_rel"
|
||||
else
|
||||
root_path="$PROJECT_ROOT/$root_rel"
|
||||
fi
|
||||
|
||||
if [[ ! -d "$root_path" ]]; then
|
||||
fail "Metric '$metric_id' root directory not found: $(relative_path "$root_path")"
|
||||
continue
|
||||
fi
|
||||
|
||||
current_count="0"
|
||||
if ! current_count="$(count_metric "$mode" "$pattern" "$root_path" "$glob")"; then
|
||||
fail "Metric '$metric_id' failed while counting matches"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$current_count" == __INVALID_MODE__:* ]]; then
|
||||
fail "Metric '$metric_id' uses unsupported mode '$mode'"
|
||||
continue
|
||||
fi
|
||||
|
||||
max_allowed=$((baseline + allowed_growth))
|
||||
delta=$((current_count - baseline))
|
||||
|
||||
if [[ "$current_count" -le "$max_allowed" ]]; then
|
||||
pass "$metric_id ($description): current=$current_count baseline=$baseline allowed_growth=$allowed_growth threshold=$max_allowed delta=$delta"
|
||||
else
|
||||
fail "$metric_id ($description): current=$current_count exceeds threshold=$max_allowed (baseline=$baseline allowed_growth=$allowed_growth delta=$delta)"
|
||||
fi
|
||||
done < <(jq -c '.metrics[]' "$BASELINE_FILE")
|
||||
|
||||
echo ""
|
||||
if [[ "$FAILURES" -eq 0 ]]; then
|
||||
echo -e "${GREEN}Test quality gate passed.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -e "${RED}Test quality gate failed with $FAILURES issue(s).${NC}"
|
||||
exit 1
|
||||
Reference in New Issue
Block a user