Files
gitea-migration/lib/common.sh
S 0e0aeda658 feat: extract .env validators to common.sh and add validate_env()
Move 10 validation functions from configure_env.sh to lib/common.sh as
shared utilities. Define variable-to-validator mapping using parallel
arrays (bash 3.2 compatible). validate_env() checks all ~50 .env
variables against their expected format and reports all failures at once.

Wired into preflight.sh (Check 6b) and bitwarden_to_env.sh (post-restore).
configure_env.sh now sources validators from common.sh instead of
defining its own copies.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-28 22:08:01 -05:00

697 lines
21 KiB
Bash

#!/usr/bin/env bash
# =============================================================================
# lib/common.sh — Shared functions for Gitea Migration Toolkit
# Source this file in every script: source "$(dirname "$0")/lib/common.sh"
# =============================================================================
# This file defines functions only. No side effects at source time.
# ---------------------------------------------------------------------------
# Colors (only if stderr is a terminal)
# ---------------------------------------------------------------------------
if [[ -t 2 ]]; then
_C_RESET='\033[0m'
_C_RED='\033[0;31m'
_C_GREEN='\033[0;32m'
_C_YELLOW='\033[0;33m'
_C_BLUE='\033[0;34m'
_C_BOLD='\033[1m'
else
_C_RESET='' _C_RED='' _C_GREEN='' _C_YELLOW='' _C_BLUE='' _C_BOLD=''
fi
# ---------------------------------------------------------------------------
# Logging — all output to stderr so stdout stays clean for data (JSON, etc.)
# ---------------------------------------------------------------------------
log_info() {
printf '%b[INFO]%b %s\n' "$_C_BLUE" "$_C_RESET" "$*" >&2
}
log_warn() {
printf '%b[WARN]%b %s\n' "$_C_YELLOW" "$_C_RESET" "$*" >&2
}
log_error() {
printf '%b[ERROR]%b %s\n' "$_C_RED" "$_C_RESET" "$*" >&2
}
log_success() {
printf '%b[OK]%b %s\n' "$_C_GREEN" "$_C_RESET" "$*" >&2
}
log_step() {
local step_num="$1"; shift
printf ' %b[%s]%b %s\n' "$_C_BOLD" "$step_num" "$_C_RESET" "$*" >&2
}
phase_header() {
local num="$1" name="$2"
printf '\n%b=== Phase %s: %s ===%b\n\n' "$_C_BOLD" "$num" "$name" "$_C_RESET" >&2
}
# ---------------------------------------------------------------------------
# Environment
# ---------------------------------------------------------------------------
# Resolve the project root (directory containing .env / .env.example)
_project_root() {
local dir
dir="$(cd "$(dirname "${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}")/.." && pwd)"
# Walk up until we find .env or .env.example
while [[ "$dir" != "/" ]]; do
if [[ -f "$dir/.env" ]] || [[ -f "$dir/.env.example" ]]; then
printf '%s' "$dir"
return 0
fi
dir="$(dirname "$dir")"
done
# Fallback: script's parent dir
cd "$(dirname "${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}")/.." && pwd
}
# Source .env and export all variables.
# Uses set -a/+a to auto-export every variable defined in the file,
# making them available to child processes (envsubst, ssh, etc.).
load_env() {
local env_file
env_file="$(_project_root)/.env"
if [[ ! -f "$env_file" ]]; then
log_error ".env file not found at $env_file"
log_error "Copy .env.example to .env and populate values."
return 1
fi
set -a # auto-export all vars defined below
# shellcheck source=/dev/null
source "$env_file"
set +a # stop auto-exporting
}
save_env_var() {
local key="$1" value="$2"
local env_file
env_file="$(_project_root)/.env"
if [[ ! -f "$env_file" ]]; then
log_error ".env file not found at $env_file"
return 1
fi
# Escape special characters in value for sed
local escaped_value
escaped_value=$(printf '%s' "$value" | sed 's/[&/\]/\\&/g')
if grep -q "^${key}=" "$env_file"; then
# Replace existing line — match KEY= followed by anything (value + optional comment)
sed -i.bak "s|^${key}=.*|${key}=${escaped_value}|" "$env_file"
rm -f "${env_file}.bak"
else
# Append new line
printf '%s=%s\n' "$key" "$value" >> "$env_file"
fi
# Also export it in current shell
export "${key}=${value}"
}
require_vars() {
local var
for var in "$@"; do
if [[ -z "${!var:-}" ]]; then
log_error "Missing required var: $var"
return 1
fi
done
}
# ---------------------------------------------------------------------------
# .env value validators — shared by configure_env.sh, preflight.sh,
# bitwarden_to_env.sh. Each returns 0 (valid) or 1 (invalid).
# ---------------------------------------------------------------------------
validate_ip() {
[[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]
}
validate_port() {
[[ "$1" =~ ^[0-9]+$ ]] && [[ "$1" -ge 1 ]] && [[ "$1" -le 65535 ]]
}
validate_email() {
[[ "$1" == *@* ]]
}
validate_path() {
[[ "$1" == /* ]]
}
validate_url() {
[[ "$1" =~ ^https?:// ]]
}
validate_bool() {
[[ "$1" == "true" ]] || [[ "$1" == "false" ]]
}
validate_integer() {
[[ "$1" =~ ^[0-9]+$ ]]
}
validate_nonempty() {
[[ -n "$1" ]]
}
validate_password() {
[[ ${#1} -ge 8 ]]
}
validate_ssl_mode() {
[[ "$1" == "letsencrypt" ]] || [[ "$1" == "existing" ]]
}
# ---------------------------------------------------------------------------
# .env format validation — validate_env()
# Checks every .env variable against its expected type.
# Requires .env to be loaded first (variables in environment).
# Returns 0 if all pass, 1 if any fail. Does NOT exit — caller decides.
# ---------------------------------------------------------------------------
# Parallel arrays: variable name → validator type (bash 3.2 compatible).
# Order matches configure_env.sh prompt_var calls.
_ENV_VAR_NAMES=(
UNRAID_IP UNRAID_SSH_USER UNRAID_SSH_PORT
UNRAID_GITEA_PORT UNRAID_GITEA_SSH_PORT UNRAID_GITEA_DATA_PATH
FEDORA_IP FEDORA_SSH_USER FEDORA_SSH_PORT
FEDORA_GITEA_PORT FEDORA_GITEA_SSH_PORT FEDORA_GITEA_DATA_PATH
GITEA_ADMIN_USER GITEA_ADMIN_PASSWORD GITEA_ADMIN_EMAIL
GITEA_ORG_NAME GITEA_INSTANCE_NAME GITEA_DB_TYPE GITEA_VERSION ACT_RUNNER_VERSION
GITEA_DOMAIN GITEA_INTERNAL_URL
GITEA_BACKUP_INTERNAL_URL GITEA_BACKUP_MIRROR_INTERVAL
BACKUP_STORAGE_PATH BACKUP_RETENTION_COUNT
GITHUB_USERNAME GITHUB_TOKEN
REPO_1_NAME REPO_2_NAME REPO_3_NAME
MIGRATE_ISSUES MIGRATE_LABELS MIGRATE_MILESTONES MIGRATE_WIKI
GITHUB_MIRROR_TOKEN GITHUB_MIRROR_INTERVAL
NGINX_CONTAINER_NAME NGINX_CONF_PATH SSL_MODE
PROTECTED_BRANCH REQUIRE_PR_REVIEW REQUIRED_APPROVALS
SEMGREP_VERSION TRIVY_VERSION GITLEAKS_VERSION SECURITY_FAIL_ON_ERROR
)
_ENV_VAR_TYPES=(
ip nonempty port
port port path
ip nonempty port
port port path
nonempty password email
nonempty nonempty nonempty nonempty nonempty
nonempty url
url nonempty
path integer
nonempty nonempty
nonempty nonempty nonempty
bool bool bool bool
nonempty nonempty
nonempty path ssl_mode
nonempty bool integer
nonempty nonempty nonempty bool
)
# Conditional variables — validated only when SSL_MODE matches.
_ENV_CONDITIONAL_NAMES=(SSL_EMAIL SSL_CERT_PATH SSL_KEY_PATH)
_ENV_CONDITIONAL_TYPES=(email path path)
_ENV_CONDITIONAL_WHEN=( letsencrypt existing existing)
# Human-readable format hints for error messages.
_validator_hint() {
case "$1" in
ip) echo "expected: x.x.x.x" ;;
port) echo "expected: 1-65535" ;;
email) echo "must contain @" ;;
path) echo "must start with /" ;;
url) echo "must start with http:// or https://" ;;
bool) echo "must be true or false" ;;
integer) echo "must be a number" ;;
nonempty) echo "cannot be empty" ;;
password) echo "must be at least 8 characters" ;;
ssl_mode) echo "must be letsencrypt or existing" ;;
*) echo "invalid" ;;
esac
}
validate_env() {
local errors=0
local i var_name var_type value
# Validate main variables
for ((i = 0; i < ${#_ENV_VAR_NAMES[@]}; i++)); do
var_name="${_ENV_VAR_NAMES[$i]}"
var_type="${_ENV_VAR_TYPES[$i]}"
value="${!var_name:-}"
if [[ -z "$value" ]]; then
log_error "$var_name is empty ($(_validator_hint "$var_type"))"
errors=$((errors + 1))
elif ! "validate_${var_type}" "$value"; then
log_error "$var_name='$value' ($(_validator_hint "$var_type"))"
errors=$((errors + 1))
fi
done
# Validate conditional variables (SSL_MODE-dependent)
local ssl_mode="${SSL_MODE:-}"
for ((i = 0; i < ${#_ENV_CONDITIONAL_NAMES[@]}; i++)); do
var_name="${_ENV_CONDITIONAL_NAMES[$i]}"
var_type="${_ENV_CONDITIONAL_TYPES[$i]}"
local required_when="${_ENV_CONDITIONAL_WHEN[$i]}"
if [[ "$ssl_mode" != "$required_when" ]]; then
continue
fi
value="${!var_name:-}"
if [[ -z "$value" ]]; then
log_error "$var_name is empty (required when SSL_MODE=$required_when, $(_validator_hint "$var_type"))"
errors=$((errors + 1))
elif ! "validate_${var_type}" "$value"; then
log_error "$var_name='$value' ($(_validator_hint "$var_type"))"
errors=$((errors + 1))
fi
done
if [[ $errors -gt 0 ]]; then
log_error "$errors .env variable(s) failed format validation"
return 1
fi
return 0
}
# ---------------------------------------------------------------------------
# OS compatibility checks
# ---------------------------------------------------------------------------
# Verify the local machine is running the expected OS.
# Usage: require_local_os "Darwin" "This script requires macOS"
# os_type: "Darwin" for macOS, "Linux" for Linux
# Checks `uname -s` against the expected value.
require_local_os() {
local expected="$1" msg="${2:-This script requires ${1}}"
local actual
actual="$(uname -s)"
if [[ "$actual" != "$expected" ]]; then
log_error "$msg"
log_error "Detected OS: $actual (expected: $expected)"
return 1
fi
}
# Verify a remote machine (via SSH) is running the expected OS.
# Usage: require_remote_os "UNRAID" "Linux" "Unraid must be a Linux machine"
# host_key: SSH host key (UNRAID, FEDORA) — uses ssh_exec
# os_type: expected `uname -s` output
require_remote_os() {
local host_key="$1" expected="$2" msg="${3:-Remote host $1 must be running $2}"
local actual
actual="$(ssh_exec "$host_key" "uname -s" 2>/dev/null)" || {
log_error "Cannot determine OS on ${host_key} (SSH failed)"
return 1
}
if [[ "$actual" != "$expected" ]]; then
log_error "$msg"
log_error "${host_key} detected OS: $actual (expected: $expected)"
return 1
fi
}
# Check if a remote host has a specific package manager available.
# Usage: require_remote_pkg_manager "FEDORA" "dnf" "Fedora requires dnf"
require_remote_pkg_manager() {
local host_key="$1" pkg_mgr="$2"
local msg="${3:-${host_key} requires ${pkg_mgr}}"
if ! ssh_exec "$host_key" "command -v $pkg_mgr" &>/dev/null; then
log_error "$msg"
log_error "${host_key} does not have '${pkg_mgr}' — is this the right machine?"
return 1
fi
}
# ---------------------------------------------------------------------------
# SSH
# ---------------------------------------------------------------------------
# Execute a command on a remote host via SSH.
# Uses indirect variable expansion: ssh_exec "UNRAID" "ls" reads
# UNRAID_IP, UNRAID_SSH_USER, UNRAID_SSH_PORT from the environment.
# This pattern avoids passing connection details to every function call.
ssh_exec() {
local host_key="$1"; shift
local cmd="$*"
# Indirect expansion: ${!ip_var} dereferences the variable named by $ip_var
local ip_var="${host_key}_IP"
local user_var="${host_key}_SSH_USER"
local port_var="${host_key}_SSH_PORT"
local ip="${!ip_var:-}"
local user="${!user_var:-}"
local port="${!port_var:-22}"
if [[ -z "$ip" ]] || [[ -z "$user" ]]; then
log_error "SSH config incomplete for $host_key: ${ip_var}=${ip:-<empty>}, ${user_var}=${user:-<empty>}"
return 1
fi
# ConnectTimeout: fail fast if host is unreachable (don't hang for 60s)
# StrictHostKeyChecking=accept-new: auto-accept new hosts but reject changed keys
# BatchMode=yes: never prompt for password (fail if key auth doesn't work)
ssh -o ConnectTimeout=10 \
-o StrictHostKeyChecking=accept-new \
-o BatchMode=yes \
-p "$port" \
"${user}@${ip}" \
"$cmd"
}
ssh_check() {
local host_key="$1"
ssh_exec "$host_key" "true" 2>/dev/null
}
scp_to() {
local host_key="$1" src="$2" dst="$3"
local ip_var="${host_key}_IP"
local user_var="${host_key}_SSH_USER"
local port_var="${host_key}_SSH_PORT"
local ip="${!ip_var:-}"
local user="${!user_var:-}"
local port="${!port_var:-22}"
if [[ -z "$ip" ]] || [[ -z "$user" ]]; then
log_error "SCP config incomplete for $host_key"
return 1
fi
scp -o ConnectTimeout=10 \
-o StrictHostKeyChecking=accept-new \
-o BatchMode=yes \
-P "$port" \
"$src" "${user}@${ip}:${dst}"
}
# ---------------------------------------------------------------------------
# API wrappers — return JSON on stdout, logs go to stderr
# ---------------------------------------------------------------------------
# Internal API call helper. Writes response to a tmpfile so we can separate
# the HTTP status code (via curl -w) from the response body. This ensures
# JSON output goes to stdout and error messages go to stderr — callers can
# pipe JSON through jq without log noise contaminating the output.
_api_call() {
local base_url="$1" token="$2" method="$3" path="$4" data="${5:-}"
local http_code
local tmpfile
tmpfile=$(mktemp)
local -a curl_args=(
-s
-X "$method"
-H "Authorization: token ${token}"
-H "Content-Type: application/json"
-H "Accept: application/json"
-o "$tmpfile"
-w "%{http_code}"
)
if [[ -n "$data" ]]; then
curl_args+=(-d "$data")
fi
http_code=$(curl "${curl_args[@]}" "${base_url}${path}") || {
log_error "curl failed for $method ${base_url}${path}"
rm -f "$tmpfile"
return 1
}
if [[ "$http_code" -ge 400 ]]; then
log_error "API $method ${path} returned HTTP ${http_code}"
log_error "Response: $(cat "$tmpfile")"
rm -f "$tmpfile"
return 1
fi
cat "$tmpfile"
rm -f "$tmpfile"
}
gitea_api() {
local method="$1" path="$2" data="${3:-}"
_api_call "${GITEA_INTERNAL_URL}/api/v1" "${GITEA_ADMIN_TOKEN}" "$method" "$path" "$data"
}
gitea_backup_api() {
local method="$1" path="$2" data="${3:-}"
_api_call "${GITEA_BACKUP_INTERNAL_URL}/api/v1" "${GITEA_BACKUP_ADMIN_TOKEN}" "$method" "$path" "$data"
}
github_api() {
local method="$1" path="$2" data="${3:-}"
_api_call "https://api.github.com" "${GITHUB_TOKEN}" "$method" "$path" "$data"
}
# ---------------------------------------------------------------------------
# Templates
# ---------------------------------------------------------------------------
render_template() {
local src="$1" dest="$2" vars="${3:-}"
if [[ ! -f "$src" ]]; then
log_error "Template not found: $src"
return 1
fi
if [[ -z "$vars" ]]; then
log_error "render_template requires an explicit variable list (third argument)"
log_error "Example: render_template src dest '\${VAR1} \${VAR2}'"
return 1
fi
envsubst "$vars" < "$src" > "$dest"
}
# ---------------------------------------------------------------------------
# Polling / waiting
# ---------------------------------------------------------------------------
wait_for_http() {
local url="$1" max_secs="${2:-60}"
local elapsed=0
log_info "Waiting for HTTP 200 at ${url} (timeout: ${max_secs}s)..."
while [[ $elapsed -lt $max_secs ]]; do
if curl -sf -o /dev/null "$url" 2>/dev/null; then
log_success "HTTP 200 at ${url} (after ${elapsed}s)"
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
log_error "Timeout waiting for ${url} after ${max_secs}s"
return 1
}
wait_for_ssh() {
local host_key="$1" max_secs="${2:-60}"
local elapsed=0
log_info "Waiting for SSH to ${host_key} (timeout: ${max_secs}s)..."
while [[ $elapsed -lt $max_secs ]]; do
if ssh_check "$host_key"; then
log_success "SSH to ${host_key} connected (after ${elapsed}s)"
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
log_error "Timeout waiting for SSH to ${host_key} after ${max_secs}s"
return 1
}
# ---------------------------------------------------------------------------
# Version checking
# ---------------------------------------------------------------------------
# Compare two semver-like version strings (major.minor or major.minor.patch).
# Returns 0 if $1 >= $2, 1 otherwise.
# Works by comparing each numeric component left to right.
_version_gte() {
local ver="$1" min="$2"
# Split on dots into arrays
local IFS='.'
# shellcheck disable=SC2206
local -a v=($ver)
# shellcheck disable=SC2206
local -a m=($min)
local max_parts=${#m[@]}
local i
for ((i = 0; i < max_parts; i++)); do
local vp="${v[$i]:-0}"
local mp="${m[$i]:-0}"
if (( vp > mp )); then return 0; fi
if (( vp < mp )); then return 1; fi
done
return 0
}
# Extract the first semver-like string (X.Y or X.Y.Z) from arbitrary output.
# Handles common patterns like "jq-1.7.1", "Docker version 24.0.7", "v2.29.1", etc.
_extract_version() {
local raw="$1"
# Match the first occurrence of digits.digits (optionally .digits more)
if [[ "$raw" =~ ([0-9]+\.[0-9]+(\.[0-9]+)*) ]]; then
printf '%s' "${BASH_REMATCH[1]}"
else
printf ''
fi
}
# Check that a local command meets a minimum version.
# Usage: check_min_version "docker" "docker --version" "20.0"
# tool_name: display name for log messages
# version_cmd: command to run (must output version somewhere in its output)
# min_version: minimum required version (e.g. "1.6", "20.0.0")
# Returns 0 if version >= min, 1 otherwise.
check_min_version() {
local tool_name="$1" version_cmd="$2" min_version="$3"
local raw_output
raw_output=$(eval "$version_cmd" 2>&1) || {
log_error "$tool_name: failed to run '$version_cmd'"
return 1
}
local actual
actual=$(_extract_version "$raw_output")
if [[ -z "$actual" ]]; then
log_error "$tool_name: could not parse version from: $raw_output"
return 1
fi
if _version_gte "$actual" "$min_version"; then
log_success "$tool_name $actual (>= $min_version)"
return 0
else
log_error "$tool_name $actual is below minimum $min_version"
return 1
fi
}
# Same as check_min_version but runs the command on a remote host via SSH.
# Usage: check_remote_min_version "UNRAID" "docker" "docker --version" "20.0"
check_remote_min_version() {
local host_key="$1" tool_name="$2" version_cmd="$3" min_version="$4"
local raw_output
raw_output=$(ssh_exec "$host_key" "$version_cmd" 2>&1) || {
log_error "$tool_name on ${host_key}: failed to run '$version_cmd'"
return 1
}
local actual
actual=$(_extract_version "$raw_output")
if [[ -z "$actual" ]]; then
log_error "$tool_name on ${host_key}: could not parse version from: $raw_output"
return 1
fi
if _version_gte "$actual" "$min_version"; then
log_success "$tool_name $actual on ${host_key} (>= $min_version)"
return 0
else
log_error "$tool_name $actual on ${host_key} is below minimum $min_version"
return 1
fi
}
# ---------------------------------------------------------------------------
# Install manifest — tracks what each setup script installs for rollback
# ---------------------------------------------------------------------------
# Manifest files live in $PROJECT_ROOT/.manifests/<host>.manifest
# Each line: TYPE|TARGET|DETAILS
# Types:
# brew_pkg — Homebrew package on macOS (TARGET=package_name)
# dnf_pkg — DNF package on Fedora (TARGET=package_name)
# static_bin — Static binary installed to a path (TARGET=path)
# docker_group — User added to docker group (TARGET=username)
# systemd_svc — Systemd service enabled (TARGET=service_name)
# xcode_cli — Xcode CLI Tools installed (TARGET=xcode-select)
# directory — Directory created (TARGET=path)
_manifest_dir() {
printf '%s/.manifests' "$(_project_root)"
}
# Initialize manifest directory and file for a host.
# Usage: manifest_init "macbook"
manifest_init() {
local host="$1"
local dir
dir="$(_manifest_dir)"
mkdir -p "$dir"
local file="${dir}/${host}.manifest"
if [[ ! -f "$file" ]]; then
printf '# Install manifest for %s — created %s\n' "$host" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$file"
fi
}
# Record an action in the manifest. Skips duplicates.
# Usage: manifest_record "macbook" "brew_pkg" "jq"
# manifest_record "unraid" "static_bin" "/usr/local/bin/jq"
manifest_record() {
local host="$1" action_type="$2" target="$3" details="${4:-}"
local file
file="$(_manifest_dir)/${host}.manifest"
# Ensure manifest file exists
manifest_init "$host"
local entry="${action_type}|${target}|${details}"
# Skip if already recorded
if grep -qF "$entry" "$file" 2>/dev/null; then
return 0
fi
printf '%s\n' "$entry" >> "$file"
}
# Check if a manifest file exists and has entries (beyond the header).
# Usage: manifest_exists "macbook"
manifest_exists() {
local host="$1"
local file
file="$(_manifest_dir)/${host}.manifest"
[[ -f "$file" ]] && grep -qv '^#' "$file" 2>/dev/null
}
# Read all entries from a manifest (skipping comments).
# Usage: manifest_entries "macbook"
# Outputs lines to stdout: TYPE|TARGET|DETAILS
manifest_entries() {
local host="$1"
local file
file="$(_manifest_dir)/${host}.manifest"
if [[ ! -f "$file" ]]; then
return 0
fi
grep -v '^#' "$file" | grep -v '^$' || true
}
# Remove the manifest file for a host (after successful cleanup).
# Usage: manifest_clear "macbook"
manifest_clear() {
local host="$1"
local file
file="$(_manifest_dir)/${host}.manifest"
rm -f "$file"
}