#!/usr/bin/env bash # validate-test-quality.sh — Enforce anti-pattern regression thresholds for UI tests. # # Usage: # scripts/validate-test-quality.sh [baseline_file] # scripts/validate-test-quality.sh --help # # Behavior: # - Loads metric baselines from JSON. # - Counts pattern matches in configured roots via ripgrep. # - Fails if any metric exceeds baseline + allowed_growth. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" DEFAULT_BASELINE_FILE="$PROJECT_ROOT/audit/test-quality-baseline.json" BASELINE_FILE="${1:-$DEFAULT_BASELINE_FILE}" usage() { cat <<'EOF' validate-test-quality.sh: enforce UI test anti-pattern regression thresholds. Usage: scripts/validate-test-quality.sh [baseline_file] scripts/validate-test-quality.sh --help EOF } if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then usage exit 0 fi RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' FAILURES=0 fail() { echo -e "${RED}FAIL:${NC} $1" FAILURES=$((FAILURES + 1)) } pass() { echo -e "${GREEN}PASS:${NC} $1" } require_cmd() { if ! command -v "$1" >/dev/null 2>&1; then fail "Missing required command: $1" fi } relative_path() { local path="$1" if [[ "$path" == "$PROJECT_ROOT/"* ]]; then echo "${path#$PROJECT_ROOT/}" else echo "$path" fi } count_matches() { local pattern="$1" local root="$2" local glob="$3" local output="" local status=0 set +e output="$(rg --count-matches --no-messages --pcre2 -N -g "$glob" "$pattern" "$root" 2>/dev/null)" status=$? set -e if [[ "$status" -eq 1 ]]; then echo "0" return 0 fi if [[ "$status" -ne 0 ]]; then return "$status" fi if [[ -z "$output" ]]; then echo "0" return 0 fi echo "$output" | awk -F: '{sum += $NF} END {print sum + 0}' } count_matches_multiline() { local pattern="$1" local root="$2" local glob="$3" local output="" local status=0 set +e output="$(rg --count-matches --no-messages --pcre2 -U -N -g "$glob" "$pattern" "$root" 2>/dev/null)" status=$? set -e if [[ "$status" -eq 1 ]]; then echo "0" return 0 fi if [[ "$status" -ne 0 ]]; then return "$status" fi if [[ -z "$output" ]]; then echo "0" return 0 fi echo "$output" | awk -F: '{sum += $NF} END {print sum + 0}' } list_metric_files() { local root="$1" local glob="$2" rg --files "$root" -g "$glob" 2>/dev/null || true } count_swift_test_body_pattern_matches() { local pattern="$1" local root="$2" local glob="$3" local files=() while IFS= read -r file_path; do [[ -n "$file_path" ]] && files+=("$file_path") done < <(list_metric_files "$root" "$glob") if [[ "${#files[@]}" -eq 0 ]]; then echo "0" return 0 fi awk -v pattern="$pattern" ' function update_depth(line, i, c) { for (i = 1; i <= length(line); i++) { c = substr(line, i, 1) if (c == "{") depth++ else if (c == "}") depth-- } } /^[[:space:]]*func[[:space:]]+test[[:alnum:]_]+[[:space:]]*\(.*\)[[:space:]]*(throws)?[[:space:]]*\{/ { in_test = 1 depth = 0 update_depth($0) if ($0 ~ pattern) count++ if (depth <= 0) { in_test = 0 depth = 0 } next } { if (!in_test) next if ($0 ~ pattern) count++ update_depth($0) if (depth <= 0) { in_test = 0 depth = 0 } } END { print count + 0 } ' "${files[@]}" } count_swift_empty_test_bodies() { local root="$1" local glob="$2" local files=() while IFS= read -r file_path; do [[ -n "$file_path" ]] && files+=("$file_path") done < <(list_metric_files "$root" "$glob") if [[ "${#files[@]}" -eq 0 ]]; then echo "0" return 0 fi awk ' function update_depth(line, i, c) { for (i = 1; i <= length(line); i++) { c = substr(line, i, 1) if (c == "{") depth++ else if (c == "}") depth-- } } function test_body_has_code(body, cleaned) { cleaned = body gsub(/\/\/.*/, "", cleaned) gsub(/[ \t\r\n{}]/, "", cleaned) return cleaned != "" } /^[[:space:]]*func[[:space:]]+test[[:alnum:]_]+[[:space:]]*\(.*\)[[:space:]]*(throws)?[[:space:]]*\{/ { in_test = 1 depth = 0 body = "" update_depth($0) if (depth <= 0) { empty_count++ in_test = 0 depth = 0 body = "" } next } { if (!in_test) next body = body $0 "\n" update_depth($0) if (depth <= 0) { if (!test_body_has_code(body)) { empty_count++ } in_test = 0 depth = 0 body = "" } } END { print empty_count + 0 } ' "${files[@]}" } count_metric() { local mode="$1" local pattern="$2" local root="$3" local glob="$4" case "$mode" in rg) count_matches "$pattern" "$root" "$glob" ;; rg_multiline) count_matches_multiline "$pattern" "$root" "$glob" ;; swift_test_body_pattern) count_swift_test_body_pattern_matches "$pattern" "$root" "$glob" ;; swift_empty_test_bodies) count_swift_empty_test_bodies "$root" "$glob" ;; *) echo "__INVALID_MODE__:$mode" return 0 ;; esac } require_cmd jq require_cmd rg require_cmd awk echo "=== Test Quality Gate ===" echo "Baseline: $(relative_path "$BASELINE_FILE")" if [[ ! -f "$BASELINE_FILE" ]]; then fail "Baseline file not found: $(relative_path "$BASELINE_FILE")" echo "" echo -e "${RED}Test quality gate failed with $FAILURES issue(s).${NC}" exit 1 fi if jq -e '.metrics | type == "array" and length > 0' "$BASELINE_FILE" >/dev/null; then pass "Baseline includes metric definitions" else fail "Baseline file has no metrics" fi while IFS= read -r metric; do [[ -z "$metric" ]] && continue metric_id="$(jq -r '.id // empty' <<<"$metric")" description="$(jq -r '.description // empty' <<<"$metric")" mode="$(jq -r '.mode // "rg"' <<<"$metric")" root_rel="$(jq -r '.root // empty' <<<"$metric")" glob="$(jq -r '.glob // empty' <<<"$metric")" pattern="$(jq -r '.pattern // ""' <<<"$metric")" baseline="$(jq -r '.baseline // empty' <<<"$metric")" allowed_growth="$(jq -r '.allowed_growth // empty' <<<"$metric")" if [[ -z "$metric_id" || -z "$root_rel" || -z "$glob" || -z "$baseline" || -z "$allowed_growth" ]]; then fail "Metric entry is missing required fields: $metric" continue fi if [[ "$mode" != "swift_empty_test_bodies" && -z "$pattern" ]]; then fail "Metric '$metric_id' requires non-empty pattern for mode '$mode'" continue fi if ! [[ "$baseline" =~ ^[0-9]+$ ]]; then fail "Metric '$metric_id' has non-numeric baseline: $baseline" continue fi if ! [[ "$allowed_growth" =~ ^[0-9]+$ ]]; then fail "Metric '$metric_id' has non-numeric allowed_growth: $allowed_growth" continue fi if [[ "$root_rel" == /* ]]; then root_path="$root_rel" else root_path="$PROJECT_ROOT/$root_rel" fi if [[ ! -d "$root_path" ]]; then fail "Metric '$metric_id' root directory not found: $(relative_path "$root_path")" continue fi current_count="0" if ! current_count="$(count_metric "$mode" "$pattern" "$root_path" "$glob")"; then fail "Metric '$metric_id' failed while counting matches" continue fi if [[ "$current_count" == __INVALID_MODE__:* ]]; then fail "Metric '$metric_id' uses unsupported mode '$mode'" continue fi max_allowed=$((baseline + allowed_growth)) delta=$((current_count - baseline)) if [[ "$current_count" -le "$max_allowed" ]]; then pass "$metric_id ($description): current=$current_count baseline=$baseline allowed_growth=$allowed_growth threshold=$max_allowed delta=$delta" else fail "$metric_id ($description): current=$current_count exceeds threshold=$max_allowed (baseline=$baseline allowed_growth=$allowed_growth delta=$delta)" fi done < <(jq -c '.metrics[]' "$BASELINE_FILE") echo "" if [[ "$FAILURES" -eq 0 ]]; then echo -e "${GREEN}Test quality gate passed.${NC}" exit 0 fi echo -e "${RED}Test quality gate failed with $FAILURES issue(s).${NC}" exit 1