dc08375ad0
- teardown_all.sh: replace `yes |` pipeline with `< <(yes)` process
substitution to avoid SIGPIPE (exit 141) false failures under pipefail
- phase6_teardown.sh: extract push mirror `.id` instead of `.remote_name`
to match the DELETE /push_mirrors/{id} API contract
- phase5_migrate_pipelines.sh: expand sed regex from `[a-z_]*` to
`[a-z_.]*` to handle nested GitHub contexts like
`github.event.pull_request.number`
- lib/common.sh: render_template now requires explicit variable list to
prevent envsubst from eating Nginx variables ($host, $proxy_add_...)
- backup scripts: remove MacBook relay, use direct Unraid↔Fedora SCP;
fix dump path to write to /data/ (mounted volume) instead of /tmp/
(container-only); add unzip -t integrity verification
- preflight.sh: add --skip-port-checks flag for resuming with
--start-from (ports already bound by earlier phases)
- run_all.sh: update run_step to pass extra args; use --skip-port-checks
when --start-from > 1
- post-checks (phase4/7/9): wrap API calls in helper functions with
>/dev/null redirection instead of passing -o /dev/null as API data
- phase8: replace GitHub archiving with [MIRROR] description marking
and disable wiki/projects/Pages (archived repos reject push mirrors)
- restore_to_primary.sh: add require_vars for Fedora SSH variables
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
536 lines
16 KiB
Bash
536 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# lib/common.sh — Shared functions for Gitea Migration Toolkit
|
|
# Source this file in every script: source "$(dirname "$0")/lib/common.sh"
|
|
# =============================================================================
|
|
# This file defines functions only. No side effects at source time.
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Colors (only if stderr is a terminal)
|
|
# ---------------------------------------------------------------------------
|
|
if [[ -t 2 ]]; then
|
|
_C_RESET='\033[0m'
|
|
_C_RED='\033[0;31m'
|
|
_C_GREEN='\033[0;32m'
|
|
_C_YELLOW='\033[0;33m'
|
|
_C_BLUE='\033[0;34m'
|
|
_C_BOLD='\033[1m'
|
|
else
|
|
_C_RESET='' _C_RED='' _C_GREEN='' _C_YELLOW='' _C_BLUE='' _C_BOLD=''
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Logging — all output to stderr so stdout stays clean for data (JSON, etc.)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
log_info() {
|
|
printf '%b[INFO]%b %s\n' "$_C_BLUE" "$_C_RESET" "$*" >&2
|
|
}
|
|
|
|
log_warn() {
|
|
printf '%b[WARN]%b %s\n' "$_C_YELLOW" "$_C_RESET" "$*" >&2
|
|
}
|
|
|
|
log_error() {
|
|
printf '%b[ERROR]%b %s\n' "$_C_RED" "$_C_RESET" "$*" >&2
|
|
}
|
|
|
|
log_success() {
|
|
printf '%b[OK]%b %s\n' "$_C_GREEN" "$_C_RESET" "$*" >&2
|
|
}
|
|
|
|
log_step() {
|
|
local step_num="$1"; shift
|
|
printf ' %b[%s]%b %s\n' "$_C_BOLD" "$step_num" "$_C_RESET" "$*" >&2
|
|
}
|
|
|
|
phase_header() {
|
|
local num="$1" name="$2"
|
|
printf '\n%b=== Phase %s: %s ===%b\n\n' "$_C_BOLD" "$num" "$name" "$_C_RESET" >&2
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Environment
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Resolve the project root (directory containing .env / .env.example)
|
|
_project_root() {
|
|
local dir
|
|
dir="$(cd "$(dirname "${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}")/.." && pwd)"
|
|
# Walk up until we find .env or .env.example
|
|
while [[ "$dir" != "/" ]]; do
|
|
if [[ -f "$dir/.env" ]] || [[ -f "$dir/.env.example" ]]; then
|
|
printf '%s' "$dir"
|
|
return 0
|
|
fi
|
|
dir="$(dirname "$dir")"
|
|
done
|
|
# Fallback: script's parent dir
|
|
cd "$(dirname "${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}")/.." && pwd
|
|
}
|
|
|
|
# Source .env and export all variables.
|
|
# Uses set -a/+a to auto-export every variable defined in the file,
|
|
# making them available to child processes (envsubst, ssh, etc.).
|
|
load_env() {
|
|
local env_file
|
|
env_file="$(_project_root)/.env"
|
|
if [[ ! -f "$env_file" ]]; then
|
|
log_error ".env file not found at $env_file"
|
|
log_error "Copy .env.example to .env and populate values."
|
|
return 1
|
|
fi
|
|
set -a # auto-export all vars defined below
|
|
# shellcheck source=/dev/null
|
|
source "$env_file"
|
|
set +a # stop auto-exporting
|
|
}
|
|
|
|
save_env_var() {
|
|
local key="$1" value="$2"
|
|
local env_file
|
|
env_file="$(_project_root)/.env"
|
|
|
|
if [[ ! -f "$env_file" ]]; then
|
|
log_error ".env file not found at $env_file"
|
|
return 1
|
|
fi
|
|
|
|
# Escape special characters in value for sed
|
|
local escaped_value
|
|
escaped_value=$(printf '%s' "$value" | sed 's/[&/\]/\\&/g')
|
|
|
|
if grep -q "^${key}=" "$env_file"; then
|
|
# Replace existing line — match KEY= followed by anything (value + optional comment)
|
|
sed -i.bak "s|^${key}=.*|${key}=${escaped_value}|" "$env_file"
|
|
rm -f "${env_file}.bak"
|
|
else
|
|
# Append new line
|
|
printf '%s=%s\n' "$key" "$value" >> "$env_file"
|
|
fi
|
|
|
|
# Also export it in current shell
|
|
export "${key}=${value}"
|
|
}
|
|
|
|
require_vars() {
|
|
local var
|
|
for var in "$@"; do
|
|
if [[ -z "${!var:-}" ]]; then
|
|
log_error "Missing required var: $var"
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# OS compatibility checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Verify the local machine is running the expected OS.
|
|
# Usage: require_local_os "Darwin" "This script requires macOS"
|
|
# os_type: "Darwin" for macOS, "Linux" for Linux
|
|
# Checks `uname -s` against the expected value.
|
|
require_local_os() {
|
|
local expected="$1" msg="${2:-This script requires ${1}}"
|
|
local actual
|
|
actual="$(uname -s)"
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
log_error "$msg"
|
|
log_error "Detected OS: $actual (expected: $expected)"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Verify a remote machine (via SSH) is running the expected OS.
|
|
# Usage: require_remote_os "UNRAID" "Linux" "Unraid must be a Linux machine"
|
|
# host_key: SSH host key (UNRAID, FEDORA) — uses ssh_exec
|
|
# os_type: expected `uname -s` output
|
|
require_remote_os() {
|
|
local host_key="$1" expected="$2" msg="${3:-Remote host $1 must be running $2}"
|
|
local actual
|
|
actual="$(ssh_exec "$host_key" "uname -s" 2>/dev/null)" || {
|
|
log_error "Cannot determine OS on ${host_key} (SSH failed)"
|
|
return 1
|
|
}
|
|
if [[ "$actual" != "$expected" ]]; then
|
|
log_error "$msg"
|
|
log_error "${host_key} detected OS: $actual (expected: $expected)"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Check if a remote host has a specific package manager available.
|
|
# Usage: require_remote_pkg_manager "FEDORA" "dnf" "Fedora requires dnf"
|
|
require_remote_pkg_manager() {
|
|
local host_key="$1" pkg_mgr="$2"
|
|
local msg="${3:-${host_key} requires ${pkg_mgr}}"
|
|
if ! ssh_exec "$host_key" "command -v $pkg_mgr" &>/dev/null; then
|
|
log_error "$msg"
|
|
log_error "${host_key} does not have '${pkg_mgr}' — is this the right machine?"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SSH
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Execute a command on a remote host via SSH.
|
|
# Uses indirect variable expansion: ssh_exec "UNRAID" "ls" reads
|
|
# UNRAID_IP, UNRAID_SSH_USER, UNRAID_SSH_PORT from the environment.
|
|
# This pattern avoids passing connection details to every function call.
|
|
ssh_exec() {
|
|
local host_key="$1"; shift
|
|
local cmd="$*"
|
|
|
|
# Indirect expansion: ${!ip_var} dereferences the variable named by $ip_var
|
|
local ip_var="${host_key}_IP"
|
|
local user_var="${host_key}_SSH_USER"
|
|
local port_var="${host_key}_SSH_PORT"
|
|
|
|
local ip="${!ip_var:-}"
|
|
local user="${!user_var:-}"
|
|
local port="${!port_var:-22}"
|
|
|
|
if [[ -z "$ip" ]] || [[ -z "$user" ]]; then
|
|
log_error "SSH config incomplete for $host_key: ${ip_var}=${ip:-<empty>}, ${user_var}=${user:-<empty>}"
|
|
return 1
|
|
fi
|
|
|
|
# ConnectTimeout: fail fast if host is unreachable (don't hang for 60s)
|
|
# StrictHostKeyChecking=accept-new: auto-accept new hosts but reject changed keys
|
|
# BatchMode=yes: never prompt for password (fail if key auth doesn't work)
|
|
ssh -o ConnectTimeout=10 \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
-o BatchMode=yes \
|
|
-p "$port" \
|
|
"${user}@${ip}" \
|
|
"$cmd"
|
|
}
|
|
|
|
ssh_check() {
|
|
local host_key="$1"
|
|
ssh_exec "$host_key" "true" 2>/dev/null
|
|
}
|
|
|
|
scp_to() {
|
|
local host_key="$1" src="$2" dst="$3"
|
|
|
|
local ip_var="${host_key}_IP"
|
|
local user_var="${host_key}_SSH_USER"
|
|
local port_var="${host_key}_SSH_PORT"
|
|
|
|
local ip="${!ip_var:-}"
|
|
local user="${!user_var:-}"
|
|
local port="${!port_var:-22}"
|
|
|
|
if [[ -z "$ip" ]] || [[ -z "$user" ]]; then
|
|
log_error "SCP config incomplete for $host_key"
|
|
return 1
|
|
fi
|
|
|
|
scp -o ConnectTimeout=10 \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
-o BatchMode=yes \
|
|
-P "$port" \
|
|
"$src" "${user}@${ip}:${dst}"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# API wrappers — return JSON on stdout, logs go to stderr
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Internal API call helper. Writes response to a tmpfile so we can separate
|
|
# the HTTP status code (via curl -w) from the response body. This ensures
|
|
# JSON output goes to stdout and error messages go to stderr — callers can
|
|
# pipe JSON through jq without log noise contaminating the output.
|
|
_api_call() {
|
|
local base_url="$1" token="$2" method="$3" path="$4" data="${5:-}"
|
|
|
|
local http_code
|
|
local tmpfile
|
|
tmpfile=$(mktemp)
|
|
|
|
local -a curl_args=(
|
|
-s
|
|
-X "$method"
|
|
-H "Authorization: token ${token}"
|
|
-H "Content-Type: application/json"
|
|
-H "Accept: application/json"
|
|
-o "$tmpfile"
|
|
-w "%{http_code}"
|
|
)
|
|
|
|
if [[ -n "$data" ]]; then
|
|
curl_args+=(-d "$data")
|
|
fi
|
|
|
|
http_code=$(curl "${curl_args[@]}" "${base_url}${path}") || {
|
|
log_error "curl failed for $method ${base_url}${path}"
|
|
rm -f "$tmpfile"
|
|
return 1
|
|
}
|
|
|
|
if [[ "$http_code" -ge 400 ]]; then
|
|
log_error "API $method ${path} returned HTTP ${http_code}"
|
|
log_error "Response: $(cat "$tmpfile")"
|
|
rm -f "$tmpfile"
|
|
return 1
|
|
fi
|
|
|
|
cat "$tmpfile"
|
|
rm -f "$tmpfile"
|
|
}
|
|
|
|
gitea_api() {
|
|
local method="$1" path="$2" data="${3:-}"
|
|
_api_call "${GITEA_INTERNAL_URL}/api/v1" "${GITEA_ADMIN_TOKEN}" "$method" "$path" "$data"
|
|
}
|
|
|
|
gitea_backup_api() {
|
|
local method="$1" path="$2" data="${3:-}"
|
|
_api_call "${GITEA_BACKUP_INTERNAL_URL}/api/v1" "${GITEA_BACKUP_ADMIN_TOKEN}" "$method" "$path" "$data"
|
|
}
|
|
|
|
github_api() {
|
|
local method="$1" path="$2" data="${3:-}"
|
|
_api_call "https://api.github.com" "${GITHUB_TOKEN}" "$method" "$path" "$data"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Templates
|
|
# ---------------------------------------------------------------------------
|
|
|
|
render_template() {
|
|
local src="$1" dest="$2" vars="${3:-}"
|
|
|
|
if [[ ! -f "$src" ]]; then
|
|
log_error "Template not found: $src"
|
|
return 1
|
|
fi
|
|
|
|
if [[ -z "$vars" ]]; then
|
|
log_error "render_template requires an explicit variable list (third argument)"
|
|
log_error "Example: render_template src dest '\${VAR1} \${VAR2}'"
|
|
return 1
|
|
fi
|
|
|
|
envsubst "$vars" < "$src" > "$dest"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Polling / waiting
|
|
# ---------------------------------------------------------------------------
|
|
|
|
wait_for_http() {
|
|
local url="$1" max_secs="${2:-60}"
|
|
local elapsed=0
|
|
|
|
log_info "Waiting for HTTP 200 at ${url} (timeout: ${max_secs}s)..."
|
|
while [[ $elapsed -lt $max_secs ]]; do
|
|
if curl -sf -o /dev/null "$url" 2>/dev/null; then
|
|
log_success "HTTP 200 at ${url} (after ${elapsed}s)"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
|
|
log_error "Timeout waiting for ${url} after ${max_secs}s"
|
|
return 1
|
|
}
|
|
|
|
wait_for_ssh() {
|
|
local host_key="$1" max_secs="${2:-60}"
|
|
local elapsed=0
|
|
|
|
log_info "Waiting for SSH to ${host_key} (timeout: ${max_secs}s)..."
|
|
while [[ $elapsed -lt $max_secs ]]; do
|
|
if ssh_check "$host_key"; then
|
|
log_success "SSH to ${host_key} connected (after ${elapsed}s)"
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
|
|
log_error "Timeout waiting for SSH to ${host_key} after ${max_secs}s"
|
|
return 1
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Version checking
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Compare two semver-like version strings (major.minor or major.minor.patch).
|
|
# Returns 0 if $1 >= $2, 1 otherwise.
|
|
# Works by comparing each numeric component left to right.
|
|
_version_gte() {
|
|
local ver="$1" min="$2"
|
|
|
|
# Split on dots into arrays
|
|
local IFS='.'
|
|
# shellcheck disable=SC2206
|
|
local -a v=($ver)
|
|
# shellcheck disable=SC2206
|
|
local -a m=($min)
|
|
|
|
local max_parts=${#m[@]}
|
|
local i
|
|
for ((i = 0; i < max_parts; i++)); do
|
|
local vp="${v[$i]:-0}"
|
|
local mp="${m[$i]:-0}"
|
|
if (( vp > mp )); then return 0; fi
|
|
if (( vp < mp )); then return 1; fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# Extract the first semver-like string (X.Y or X.Y.Z) from arbitrary output.
|
|
# Handles common patterns like "jq-1.7.1", "Docker version 24.0.7", "v2.29.1", etc.
|
|
_extract_version() {
|
|
local raw="$1"
|
|
# Match the first occurrence of digits.digits (optionally .digits more)
|
|
if [[ "$raw" =~ ([0-9]+\.[0-9]+(\.[0-9]+)*) ]]; then
|
|
printf '%s' "${BASH_REMATCH[1]}"
|
|
else
|
|
printf ''
|
|
fi
|
|
}
|
|
|
|
# Check that a local command meets a minimum version.
|
|
# Usage: check_min_version "docker" "docker --version" "20.0"
|
|
# tool_name: display name for log messages
|
|
# version_cmd: command to run (must output version somewhere in its output)
|
|
# min_version: minimum required version (e.g. "1.6", "20.0.0")
|
|
# Returns 0 if version >= min, 1 otherwise.
|
|
check_min_version() {
|
|
local tool_name="$1" version_cmd="$2" min_version="$3"
|
|
local raw_output
|
|
raw_output=$(eval "$version_cmd" 2>&1) || {
|
|
log_error "$tool_name: failed to run '$version_cmd'"
|
|
return 1
|
|
}
|
|
local actual
|
|
actual=$(_extract_version "$raw_output")
|
|
if [[ -z "$actual" ]]; then
|
|
log_error "$tool_name: could not parse version from: $raw_output"
|
|
return 1
|
|
fi
|
|
if _version_gte "$actual" "$min_version"; then
|
|
log_success "$tool_name $actual (>= $min_version)"
|
|
return 0
|
|
else
|
|
log_error "$tool_name $actual is below minimum $min_version"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Same as check_min_version but runs the command on a remote host via SSH.
|
|
# Usage: check_remote_min_version "UNRAID" "docker" "docker --version" "20.0"
|
|
check_remote_min_version() {
|
|
local host_key="$1" tool_name="$2" version_cmd="$3" min_version="$4"
|
|
local raw_output
|
|
raw_output=$(ssh_exec "$host_key" "$version_cmd" 2>&1) || {
|
|
log_error "$tool_name on ${host_key}: failed to run '$version_cmd'"
|
|
return 1
|
|
}
|
|
local actual
|
|
actual=$(_extract_version "$raw_output")
|
|
if [[ -z "$actual" ]]; then
|
|
log_error "$tool_name on ${host_key}: could not parse version from: $raw_output"
|
|
return 1
|
|
fi
|
|
if _version_gte "$actual" "$min_version"; then
|
|
log_success "$tool_name $actual on ${host_key} (>= $min_version)"
|
|
return 0
|
|
else
|
|
log_error "$tool_name $actual on ${host_key} is below minimum $min_version"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Install manifest — tracks what each setup script installs for rollback
|
|
# ---------------------------------------------------------------------------
|
|
# Manifest files live in $PROJECT_ROOT/.manifests/<host>.manifest
|
|
# Each line: TYPE|TARGET|DETAILS
|
|
# Types:
|
|
# brew_pkg — Homebrew package on macOS (TARGET=package_name)
|
|
# dnf_pkg — DNF package on Fedora (TARGET=package_name)
|
|
# static_bin — Static binary installed to a path (TARGET=path)
|
|
# docker_group — User added to docker group (TARGET=username)
|
|
# systemd_svc — Systemd service enabled (TARGET=service_name)
|
|
# xcode_cli — Xcode CLI Tools installed (TARGET=xcode-select)
|
|
# directory — Directory created (TARGET=path)
|
|
|
|
_manifest_dir() {
|
|
printf '%s/.manifests' "$(_project_root)"
|
|
}
|
|
|
|
# Initialize manifest directory and file for a host.
|
|
# Usage: manifest_init "macbook"
|
|
manifest_init() {
|
|
local host="$1"
|
|
local dir
|
|
dir="$(_manifest_dir)"
|
|
mkdir -p "$dir"
|
|
local file="${dir}/${host}.manifest"
|
|
if [[ ! -f "$file" ]]; then
|
|
printf '# Install manifest for %s — created %s\n' "$host" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$file"
|
|
fi
|
|
}
|
|
|
|
# Record an action in the manifest. Skips duplicates.
|
|
# Usage: manifest_record "macbook" "brew_pkg" "jq"
|
|
# manifest_record "unraid" "static_bin" "/usr/local/bin/jq"
|
|
manifest_record() {
|
|
local host="$1" action_type="$2" target="$3" details="${4:-}"
|
|
local file
|
|
file="$(_manifest_dir)/${host}.manifest"
|
|
|
|
# Ensure manifest file exists
|
|
manifest_init "$host"
|
|
|
|
local entry="${action_type}|${target}|${details}"
|
|
|
|
# Skip if already recorded
|
|
if grep -qF "$entry" "$file" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$entry" >> "$file"
|
|
}
|
|
|
|
# Check if a manifest file exists and has entries (beyond the header).
|
|
# Usage: manifest_exists "macbook"
|
|
manifest_exists() {
|
|
local host="$1"
|
|
local file
|
|
file="$(_manifest_dir)/${host}.manifest"
|
|
[[ -f "$file" ]] && grep -qv '^#' "$file" 2>/dev/null
|
|
}
|
|
|
|
# Read all entries from a manifest (skipping comments).
|
|
# Usage: manifest_entries "macbook"
|
|
# Outputs lines to stdout: TYPE|TARGET|DETAILS
|
|
manifest_entries() {
|
|
local host="$1"
|
|
local file
|
|
file="$(_manifest_dir)/${host}.manifest"
|
|
if [[ ! -f "$file" ]]; then
|
|
return 0
|
|
fi
|
|
grep -v '^#' "$file" | grep -v '^$' || true
|
|
}
|
|
|
|
# Remove the manifest file for a host (after successful cleanup).
|
|
# Usage: manifest_clear "macbook"
|
|
manifest_clear() {
|
|
local host="$1"
|
|
local file
|
|
file="$(_manifest_dir)/${host}.manifest"
|
|
rm -f "$file"
|
|
}
|