359 lines
8.9 KiB
Bash
Executable File
359 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# validate-test-quality.sh — Enforce anti-pattern regression thresholds for UI tests.
|
|
#
|
|
# Usage:
|
|
# scripts/validate-test-quality.sh [baseline_file]
|
|
# scripts/validate-test-quality.sh --help
|
|
#
|
|
# Behavior:
|
|
# - Loads metric baselines from JSON.
|
|
# - Counts pattern matches in configured roots via ripgrep.
|
|
# - Fails if any metric exceeds baseline + allowed_growth.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
DEFAULT_BASELINE_FILE="$PROJECT_ROOT/audit/test-quality-baseline.json"
|
|
BASELINE_FILE="${1:-$DEFAULT_BASELINE_FILE}"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
validate-test-quality.sh: enforce UI test anti-pattern regression thresholds.
|
|
|
|
Usage:
|
|
scripts/validate-test-quality.sh [baseline_file]
|
|
scripts/validate-test-quality.sh --help
|
|
EOF
|
|
}
|
|
|
|
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
NC='\033[0m'
|
|
|
|
FAILURES=0
|
|
|
|
fail() {
|
|
echo -e "${RED}FAIL:${NC} $1"
|
|
FAILURES=$((FAILURES + 1))
|
|
}
|
|
|
|
pass() {
|
|
echo -e "${GREEN}PASS:${NC} $1"
|
|
}
|
|
|
|
require_cmd() {
|
|
if ! command -v "$1" >/dev/null 2>&1; then
|
|
fail "Missing required command: $1"
|
|
fi
|
|
}
|
|
|
|
relative_path() {
|
|
local path="$1"
|
|
if [[ "$path" == "$PROJECT_ROOT/"* ]]; then
|
|
echo "${path#$PROJECT_ROOT/}"
|
|
else
|
|
echo "$path"
|
|
fi
|
|
}
|
|
|
|
count_matches() {
|
|
local pattern="$1"
|
|
local root="$2"
|
|
local glob="$3"
|
|
local output=""
|
|
local status=0
|
|
|
|
set +e
|
|
output="$(rg --count-matches --no-messages --pcre2 -N -g "$glob" "$pattern" "$root" 2>/dev/null)"
|
|
status=$?
|
|
set -e
|
|
|
|
if [[ "$status" -eq 1 ]]; then
|
|
echo "0"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$status" -ne 0 ]]; then
|
|
return "$status"
|
|
fi
|
|
|
|
if [[ -z "$output" ]]; then
|
|
echo "0"
|
|
return 0
|
|
fi
|
|
|
|
echo "$output" | awk -F: '{sum += $NF} END {print sum + 0}'
|
|
}
|
|
|
|
count_matches_multiline() {
|
|
local pattern="$1"
|
|
local root="$2"
|
|
local glob="$3"
|
|
local output=""
|
|
local status=0
|
|
|
|
set +e
|
|
output="$(rg --count-matches --no-messages --pcre2 -U -N -g "$glob" "$pattern" "$root" 2>/dev/null)"
|
|
status=$?
|
|
set -e
|
|
|
|
if [[ "$status" -eq 1 ]]; then
|
|
echo "0"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$status" -ne 0 ]]; then
|
|
return "$status"
|
|
fi
|
|
|
|
if [[ -z "$output" ]]; then
|
|
echo "0"
|
|
return 0
|
|
fi
|
|
|
|
echo "$output" | awk -F: '{sum += $NF} END {print sum + 0}'
|
|
}
|
|
|
|
list_metric_files() {
|
|
local root="$1"
|
|
local glob="$2"
|
|
rg --files "$root" -g "$glob" 2>/dev/null || true
|
|
}
|
|
|
|
count_swift_test_body_pattern_matches() {
|
|
local pattern="$1"
|
|
local root="$2"
|
|
local glob="$3"
|
|
local files=()
|
|
|
|
while IFS= read -r file_path; do
|
|
[[ -n "$file_path" ]] && files+=("$file_path")
|
|
done < <(list_metric_files "$root" "$glob")
|
|
if [[ "${#files[@]}" -eq 0 ]]; then
|
|
echo "0"
|
|
return 0
|
|
fi
|
|
|
|
awk -v pattern="$pattern" '
|
|
function update_depth(line, i, c) {
|
|
for (i = 1; i <= length(line); i++) {
|
|
c = substr(line, i, 1)
|
|
if (c == "{") depth++
|
|
else if (c == "}") depth--
|
|
}
|
|
}
|
|
|
|
/^[[:space:]]*func[[:space:]]+test[[:alnum:]_]+[[:space:]]*\(.*\)[[:space:]]*(throws)?[[:space:]]*\{/ {
|
|
in_test = 1
|
|
depth = 0
|
|
update_depth($0)
|
|
if ($0 ~ pattern) count++
|
|
if (depth <= 0) {
|
|
in_test = 0
|
|
depth = 0
|
|
}
|
|
next
|
|
}
|
|
|
|
{
|
|
if (!in_test) next
|
|
|
|
if ($0 ~ pattern) count++
|
|
update_depth($0)
|
|
if (depth <= 0) {
|
|
in_test = 0
|
|
depth = 0
|
|
}
|
|
}
|
|
|
|
END { print count + 0 }
|
|
' "${files[@]}"
|
|
}
|
|
|
|
count_swift_empty_test_bodies() {
|
|
local root="$1"
|
|
local glob="$2"
|
|
local files=()
|
|
|
|
while IFS= read -r file_path; do
|
|
[[ -n "$file_path" ]] && files+=("$file_path")
|
|
done < <(list_metric_files "$root" "$glob")
|
|
if [[ "${#files[@]}" -eq 0 ]]; then
|
|
echo "0"
|
|
return 0
|
|
fi
|
|
|
|
awk '
|
|
function update_depth(line, i, c) {
|
|
for (i = 1; i <= length(line); i++) {
|
|
c = substr(line, i, 1)
|
|
if (c == "{") depth++
|
|
else if (c == "}") depth--
|
|
}
|
|
}
|
|
|
|
function test_body_has_code(body, cleaned) {
|
|
cleaned = body
|
|
gsub(/\/\/.*/, "", cleaned)
|
|
gsub(/[ \t\r\n{}]/, "", cleaned)
|
|
return cleaned != ""
|
|
}
|
|
|
|
/^[[:space:]]*func[[:space:]]+test[[:alnum:]_]+[[:space:]]*\(.*\)[[:space:]]*(throws)?[[:space:]]*\{/ {
|
|
in_test = 1
|
|
depth = 0
|
|
body = ""
|
|
update_depth($0)
|
|
if (depth <= 0) {
|
|
empty_count++
|
|
in_test = 0
|
|
depth = 0
|
|
body = ""
|
|
}
|
|
next
|
|
}
|
|
|
|
{
|
|
if (!in_test) next
|
|
|
|
body = body $0 "\n"
|
|
update_depth($0)
|
|
|
|
if (depth <= 0) {
|
|
if (!test_body_has_code(body)) {
|
|
empty_count++
|
|
}
|
|
in_test = 0
|
|
depth = 0
|
|
body = ""
|
|
}
|
|
}
|
|
|
|
END { print empty_count + 0 }
|
|
' "${files[@]}"
|
|
}
|
|
|
|
count_metric() {
|
|
local mode="$1"
|
|
local pattern="$2"
|
|
local root="$3"
|
|
local glob="$4"
|
|
|
|
case "$mode" in
|
|
rg)
|
|
count_matches "$pattern" "$root" "$glob"
|
|
;;
|
|
rg_multiline)
|
|
count_matches_multiline "$pattern" "$root" "$glob"
|
|
;;
|
|
swift_test_body_pattern)
|
|
count_swift_test_body_pattern_matches "$pattern" "$root" "$glob"
|
|
;;
|
|
swift_empty_test_bodies)
|
|
count_swift_empty_test_bodies "$root" "$glob"
|
|
;;
|
|
*)
|
|
echo "__INVALID_MODE__:$mode"
|
|
return 0
|
|
;;
|
|
esac
|
|
}
|
|
|
|
require_cmd jq
|
|
require_cmd rg
|
|
require_cmd awk
|
|
|
|
echo "=== Test Quality Gate ==="
|
|
echo "Baseline: $(relative_path "$BASELINE_FILE")"
|
|
|
|
if [[ ! -f "$BASELINE_FILE" ]]; then
|
|
fail "Baseline file not found: $(relative_path "$BASELINE_FILE")"
|
|
echo ""
|
|
echo -e "${RED}Test quality gate failed with $FAILURES issue(s).${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
if jq -e '.metrics | type == "array" and length > 0' "$BASELINE_FILE" >/dev/null; then
|
|
pass "Baseline includes metric definitions"
|
|
else
|
|
fail "Baseline file has no metrics"
|
|
fi
|
|
|
|
while IFS= read -r metric; do
|
|
[[ -z "$metric" ]] && continue
|
|
|
|
metric_id="$(jq -r '.id // empty' <<<"$metric")"
|
|
description="$(jq -r '.description // empty' <<<"$metric")"
|
|
mode="$(jq -r '.mode // "rg"' <<<"$metric")"
|
|
root_rel="$(jq -r '.root // empty' <<<"$metric")"
|
|
glob="$(jq -r '.glob // empty' <<<"$metric")"
|
|
pattern="$(jq -r '.pattern // ""' <<<"$metric")"
|
|
baseline="$(jq -r '.baseline // empty' <<<"$metric")"
|
|
allowed_growth="$(jq -r '.allowed_growth // empty' <<<"$metric")"
|
|
|
|
if [[ -z "$metric_id" || -z "$root_rel" || -z "$glob" || -z "$baseline" || -z "$allowed_growth" ]]; then
|
|
fail "Metric entry is missing required fields: $metric"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$mode" != "swift_empty_test_bodies" && -z "$pattern" ]]; then
|
|
fail "Metric '$metric_id' requires non-empty pattern for mode '$mode'"
|
|
continue
|
|
fi
|
|
|
|
if ! [[ "$baseline" =~ ^[0-9]+$ ]]; then
|
|
fail "Metric '$metric_id' has non-numeric baseline: $baseline"
|
|
continue
|
|
fi
|
|
|
|
if ! [[ "$allowed_growth" =~ ^[0-9]+$ ]]; then
|
|
fail "Metric '$metric_id' has non-numeric allowed_growth: $allowed_growth"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$root_rel" == /* ]]; then
|
|
root_path="$root_rel"
|
|
else
|
|
root_path="$PROJECT_ROOT/$root_rel"
|
|
fi
|
|
|
|
if [[ ! -d "$root_path" ]]; then
|
|
fail "Metric '$metric_id' root directory not found: $(relative_path "$root_path")"
|
|
continue
|
|
fi
|
|
|
|
current_count="0"
|
|
if ! current_count="$(count_metric "$mode" "$pattern" "$root_path" "$glob")"; then
|
|
fail "Metric '$metric_id' failed while counting matches"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$current_count" == __INVALID_MODE__:* ]]; then
|
|
fail "Metric '$metric_id' uses unsupported mode '$mode'"
|
|
continue
|
|
fi
|
|
|
|
max_allowed=$((baseline + allowed_growth))
|
|
delta=$((current_count - baseline))
|
|
|
|
if [[ "$current_count" -le "$max_allowed" ]]; then
|
|
pass "$metric_id ($description): current=$current_count baseline=$baseline allowed_growth=$allowed_growth threshold=$max_allowed delta=$delta"
|
|
else
|
|
fail "$metric_id ($description): current=$current_count exceeds threshold=$max_allowed (baseline=$baseline allowed_growth=$allowed_growth delta=$delta)"
|
|
fi
|
|
done < <(jq -c '.metrics[]' "$BASELINE_FILE")
|
|
|
|
echo ""
|
|
if [[ "$FAILURES" -eq 0 ]]; then
|
|
echo -e "${GREEN}Test quality gate passed.${NC}"
|
|
exit 0
|
|
fi
|
|
|
|
echo -e "${RED}Test quality gate failed with $FAILURES issue(s).${NC}"
|
|
exit 1
|