diff --git a/.env.example b/.env.example index 9e95c46..74dccb5 100644 --- a/.env.example +++ b/.env.example @@ -124,6 +124,8 @@ TLS_MODE=cloudflare # TLS mode: "cloudflare" (DNS-01 via CF API) o CADDY_DOMAIN= # Wildcard cert base domain (e.g. privacyindesign.com → cert for *.privacyindesign.com) CADDY_DATA_PATH= # Absolute path on host for Caddy data (e.g. /mnt/nvme/caddy) CLOUDFLARE_API_TOKEN= # Cloudflare API token with Zone:DNS:Edit (only if TLS_MODE=cloudflare) +PUBLIC_DNS_TARGET_IP= # Phase 8 Cloudflare A-record target for GITEA_DOMAIN (public ingress IP recommended) +PHASE8_ALLOW_PRIVATE_DNS_TARGET=false # true only for LAN-only/split-DNS setups using private RFC1918 target IPs SSL_CERT_PATH= # Absolute path to SSL cert (only if TLS_MODE=existing) SSL_KEY_PATH= # Absolute path to SSL key (only if TLS_MODE=existing) diff --git a/README.md b/README.md index 143f35e..e74c934 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ When `TLS_MODE=cloudflare`, Caddy handles certificate renewal automatically via | MacBook | macOS, Homebrew, jq >= 1.6, curl >= 7.70, git >= 2.30, shellcheck >= 0.8, gh >= 2.0, bw >= 2.0 | | Unraid | Linux, Docker >= 20.0, docker-compose >= 2.0, jq >= 1.6, passwordless sudo for SSH user | | Fedora | Linux with dnf, Docker CE >= 20.0, docker-compose >= 2.0, jq >= 1.6, passwordless sudo for SSH user | -| Network | MacBook can SSH to both servers, DNS A record pointing to Unraid (needed for Phase 8 TLS), Cloudflare API token (if using `TLS_MODE=cloudflare`) | +| Network | MacBook can SSH to both servers; for `TLS_MODE=cloudflare`, provide `CLOUDFLARE_API_TOKEN` plus `PUBLIC_DNS_TARGET_IP` (public ingress IP recommended; private IP requires `PHASE8_ALLOW_PRIVATE_DNS_TARGET=true`) | ## Quick Start diff --git a/USAGE_GUIDE.md b/USAGE_GUIDE.md index 440e81e..3c48def 100644 --- a/USAGE_GUIDE.md +++ b/USAGE_GUIDE.md @@ -31,8 +31,9 @@ Before running anything, confirm: DNS and TLS are only needed for Phase 8 (Caddy reverse proxy). You can set these up later: -- A DNS A record for your Gitea domain pointing to `UNRAID_IP` - If using `TLS_MODE=cloudflare`: a Cloudflare API token with Zone:DNS:Edit permission +- `PUBLIC_DNS_TARGET_IP` set to your ingress IP for `GITEA_DOMAIN` (public IP recommended) +- If you intentionally use LAN-only split DNS with a private IP target, set `PHASE8_ALLOW_PRIVATE_DNS_TARGET=true` ### 2. Passwordless sudo on remote hosts @@ -316,7 +317,7 @@ Then re-run Phase 4. Already-migrated repos will be skipped. **Symptom**: Preflight check 14 fails. -**Fix**: Add or update your DNS A record. If using a local DNS server or `/etc/hosts`, ensure the record points to `UNRAID_IP`. DNS propagation can take minutes to hours. +**Fix**: Phase 8 can auto-upsert the Cloudflare A record for `GITEA_DOMAIN` when `TLS_MODE=cloudflare`. Set `PUBLIC_DNS_TARGET_IP` first. Use a public ingress IP for public access. For LAN-only split DNS, set `PHASE8_ALLOW_PRIVATE_DNS_TARGET=true`. ### Caddy fails to start or obtain TLS certificate in Phase 8 diff --git a/lib/common.sh b/lib/common.sh index c4ace10..058c470 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -269,9 +269,9 @@ _ENV_VAR_TYPES=( ) # Conditional variables — validated only when TLS_MODE matches. -_ENV_CONDITIONAL_TLS_NAMES=(CLOUDFLARE_API_TOKEN SSL_CERT_PATH SSL_KEY_PATH) -_ENV_CONDITIONAL_TLS_TYPES=(nonempty path path) -_ENV_CONDITIONAL_TLS_WHEN=( cloudflare existing existing) +_ENV_CONDITIONAL_TLS_NAMES=(CLOUDFLARE_API_TOKEN PUBLIC_DNS_TARGET_IP PHASE8_ALLOW_PRIVATE_DNS_TARGET SSL_CERT_PATH SSL_KEY_PATH) +_ENV_CONDITIONAL_TLS_TYPES=(nonempty ip bool path path) +_ENV_CONDITIONAL_TLS_WHEN=( cloudflare cloudflare cloudflare existing existing) # Conditional variables — validated only when GITEA_DB_TYPE is NOT sqlite3. _ENV_CONDITIONAL_DB_NAMES=(GITEA_DB_PORT GITEA_DB_NAME GITEA_DB_USER GITEA_DB_PASSWD) diff --git a/phase7_5_nginx_to_caddy.sh b/phase7_5_nginx_to_caddy.sh index e41ac86..380bad1 100755 --- a/phase7_5_nginx_to_caddy.sh +++ b/phase7_5_nginx_to_caddy.sh @@ -513,12 +513,52 @@ if ! ssh_exec UNRAID "docker exec caddy caddy reload --config /etc/caddy/Caddyfi fi log_success "Caddy container is running with new config" +probe_http_code_ok() { + local code="$1" role="$2" + if [[ "$role" == "gitea_api" ]]; then + [[ "$code" == "200" ]] + return + fi + [[ "$code" =~ ^(2|3)[0-9][0-9]$ || "$code" == "401" || "$code" == "403" ]] +} + +probe_host_via_caddy() { + local host="$1" upstream="$2" role="$3" + local path="/" + if [[ "$role" == "gitea_api" ]]; then + path="/api/v1/version" + fi + + local tmp_body http_code + tmp_body=$(mktemp) + http_code=$(curl -sk --resolve "${host}:443:${UNRAID_CADDY_IP}" \ + -o "$tmp_body" -w "%{http_code}" "https://${host}${path}" 2>/dev/null || echo "000") + + if probe_http_code_ok "$http_code" "$role"; then + log_success "Probe passed: ${host} (HTTP ${http_code})" + rm -f "$tmp_body" + return 0 + fi + + log_error "Probe failed: ${host} (HTTP ${http_code})" + if [[ "$http_code" == "502" || "$http_code" == "503" || "$http_code" == "504" || "$http_code" == "000" ]]; then + local upstream_probe_raw upstream_code + upstream_probe_raw=$(ssh_exec UNRAID "curl -sk -o /dev/null -w '%{http_code}' '${upstream}' || true" 2>/dev/null || true) + upstream_code=$(printf '%s' "$upstream_probe_raw" | tr -cd '0-9') + if [[ -z "$upstream_code" ]]; then + upstream_code="000" + elif [[ ${#upstream_code} -gt 3 ]]; then + upstream_code="${upstream_code:$((${#upstream_code} - 3))}" + fi + log_warn "Upstream check from Unraid: ${upstream} -> HTTP ${upstream_code}" + fi + rm -f "$tmp_body" + return 1 +} + if [[ "$MODE" == "canary" ]]; then if confirm_action "Run canary HTTPS probe for tower.sintheus.com via Caddy IP now? [y/N] "; then - if curl -skf --resolve "tower.sintheus.com:443:${UNRAID_CADDY_IP}" \ - "https://tower.sintheus.com/" >/dev/null; then - log_success "Canary probe passed: tower.sintheus.com via ${UNRAID_CADDY_IP}" - else + if ! probe_host_via_caddy "tower.sintheus.com" "https://192.168.1.82:443" "generic"; then log_error "Canary probe failed for tower.sintheus.com via ${UNRAID_CADDY_IP}" exit 1 fi @@ -527,11 +567,12 @@ else log_step 5 "Probing all configured hosts via Caddy IP..." PROBE_FAILS=0 for entry in "${SELECTED_HOST_MAP[@]}"; do - IFS='|' read -r host _ <<< "$entry" - if curl -skf --resolve "${host}:443:${UNRAID_CADDY_IP}" "https://${host}/" >/dev/null; then - log_success "Probe passed: ${host}" - else - log_error "Probe failed: ${host}" + IFS='|' read -r host upstream _ <<< "$entry" + role="generic" + if [[ "$host" == "$GITEA_DOMAIN" ]]; then + role="gitea_api" + fi + if ! probe_host_via_caddy "$host" "$upstream" "$role"; then PROBE_FAILS=$((PROBE_FAILS + 1)) fi done diff --git a/phase8_cutover.sh b/phase8_cutover.sh index a9d387f..c5f5c84 100755 --- a/phase8_cutover.sh +++ b/phase8_cutover.sh @@ -25,7 +25,7 @@ require_vars UNRAID_IP UNRAID_SSH_USER UNRAID_GITEA_IP UNRAID_CADDY_IP \ REPO_NAMES if [[ "$TLS_MODE" == "cloudflare" ]]; then - require_vars CLOUDFLARE_API_TOKEN + require_vars CLOUDFLARE_API_TOKEN PUBLIC_DNS_TARGET_IP elif [[ "$TLS_MODE" == "existing" ]]; then require_vars SSL_CERT_PATH SSL_KEY_PATH else @@ -44,6 +44,13 @@ UNRAID_DOCKER_NETWORK_NAME="br0" CADDY_COMPOSE_DIR="${UNRAID_COMPOSE_DIR}/caddy" PHASE8_GITEA_ROUTE_BEGIN="# BEGIN_PHASE8_GITEA_ROUTE" PHASE8_GITEA_ROUTE_END="# END_PHASE8_GITEA_ROUTE" +PUBLIC_DNS_TARGET_IP="${PUBLIC_DNS_TARGET_IP:-}" +PHASE8_ALLOW_PRIVATE_DNS_TARGET="${PHASE8_ALLOW_PRIVATE_DNS_TARGET:-false}" + +if ! validate_bool "${PHASE8_ALLOW_PRIVATE_DNS_TARGET}"; then + log_error "Invalid PHASE8_ALLOW_PRIVATE_DNS_TARGET='${PHASE8_ALLOW_PRIVATE_DNS_TARGET}' (must be true or false)" + exit 1 +fi wait_for_https_public() { local host="$1" max_secs="${2:-30}" @@ -78,6 +85,149 @@ wait_for_https_via_resolve() { return 1 } +check_unraid_gitea_backend() { + local raw code + raw=$(ssh_exec UNRAID "curl -sS -o /dev/null -w '%{http_code}' 'http://${UNRAID_GITEA_IP}:3000/api/v1/version' || true" 2>/dev/null || true) + code=$(printf '%s' "$raw" | tr -cd '0-9') + if [[ -z "$code" ]]; then + code="000" + elif [[ ${#code} -gt 3 ]]; then + code="${code:$((${#code} - 3))}" + fi + + if [[ "$code" == "200" ]]; then + log_success "Unraid -> Gitea backend API reachable (HTTP 200)" + return 0 + fi + + log_error "Unraid -> Gitea backend API check failed (HTTP ${code}) at http://${UNRAID_GITEA_IP}:3000/api/v1/version" + return 1 +} + +is_private_ipv4() { + local ip="$1" + [[ "$ip" =~ ^10\. ]] || \ + [[ "$ip" =~ ^192\.168\. ]] || \ + [[ "$ip" =~ ^172\.(1[6-9]|2[0-9]|3[0-1])\. ]] +} + +cloudflare_api_call() { + local method="$1" path="$2" data="${3:-}" + local -a args=( + curl -sS + -X "$method" + -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" + -H "Content-Type: application/json" + "https://api.cloudflare.com/client/v4${path}" + ) + if [[ -n "$data" ]]; then + args+=(-d "$data") + fi + "${args[@]}" +} + +ensure_cloudflare_dns_for_gitea() { + local host="$1" target_ip="$2" zone_id zone_name + local allow_private="${PHASE8_ALLOW_PRIVATE_DNS_TARGET}" + + if [[ -z "$target_ip" ]]; then + log_error "PUBLIC_DNS_TARGET_IP is not set" + log_error "Set PUBLIC_DNS_TARGET_IP to your public ingress IP for ${host}" + log_error "For LAN-only/split-DNS use, also set PHASE8_ALLOW_PRIVATE_DNS_TARGET=true" + return 1 + fi + + if ! validate_ip "$target_ip"; then + log_error "Invalid PUBLIC_DNS_TARGET_IP='${target_ip}'" + log_error "Set PUBLIC_DNS_TARGET_IP in .env to the IP that should answer ${host}" + return 1 + fi + + zone_name="${host#*.}" + if [[ "$zone_name" == "$host" ]]; then + log_error "GITEA_DOMAIN='${host}' is not a valid FQDN for Cloudflare zone detection" + return 1 + fi + + if is_private_ipv4 "$target_ip"; then + if [[ "$allow_private" != "true" ]]; then + log_error "Refusing private DNS target ${target_ip} for Cloudflare public DNS" + log_error "Set PUBLIC_DNS_TARGET_IP to public ingress IP, or set PHASE8_ALLOW_PRIVATE_DNS_TARGET=true for LAN-only split-DNS" + return 1 + fi + log_warn "Using private DNS target ${target_ip} because PHASE8_ALLOW_PRIVATE_DNS_TARGET=true" + fi + + local zone_resp zone_err + zone_resp=$(cloudflare_api_call GET "/zones?name=${zone_name}&status=active") + if [[ "$(jq -r '.success // false' <<< "$zone_resp")" != "true" ]]; then + zone_err=$(jq -r '(.errors // []) | map(.message // tostring) | join("; ")' <<< "$zone_resp") + log_error "Cloudflare zone lookup failed for ${zone_name}: ${zone_err:-unknown error}" + return 1 + fi + + zone_id=$(jq -r '.result[0].id // empty' <<< "$zone_resp") + if [[ -z "$zone_id" ]]; then + log_error "Cloudflare zone not found or not accessible for ${zone_name}" + return 1 + fi + + local record_resp record_err record_count record_id old_ip + record_resp=$(cloudflare_api_call GET "/zones/${zone_id}/dns_records?type=A&name=${host}") + if [[ "$(jq -r '.success // false' <<< "$record_resp")" != "true" ]]; then + record_err=$(jq -r '(.errors // []) | map(.message // tostring) | join("; ")' <<< "$record_resp") + log_error "Cloudflare DNS query failed for ${host}: ${record_err:-unknown error}" + return 1 + fi + + record_count=$(jq -r '.result | length' <<< "$record_resp") + if [[ "$record_count" -eq 0 ]]; then + local create_payload create_resp create_err + create_payload=$(jq -n \ + --arg type "A" \ + --arg name "$host" \ + --arg content "$target_ip" \ + --argjson ttl 120 \ + --argjson proxied false \ + '{type:$type, name:$name, content:$content, ttl:$ttl, proxied:$proxied}') + create_resp=$(cloudflare_api_call POST "/zones/${zone_id}/dns_records" "$create_payload") + if [[ "$(jq -r '.success // false' <<< "$create_resp")" != "true" ]]; then + create_err=$(jq -r '(.errors // []) | map(.message // tostring) | join("; ")' <<< "$create_resp") + log_error "Failed to create Cloudflare A record ${host} -> ${target_ip}: ${create_err:-unknown error}" + return 1 + fi + log_success "Created Cloudflare A record: ${host} -> ${target_ip}" + return 0 + fi + + record_id=$(jq -r '.result[0].id // empty' <<< "$record_resp") + old_ip=$(jq -r '.result[0].content // empty' <<< "$record_resp") + if [[ -n "$old_ip" && "$old_ip" == "$target_ip" ]]; then + log_info "Cloudflare A record already correct: ${host} -> ${target_ip}" + return 0 + fi + + local update_payload update_resp update_err + update_payload=$(jq -n \ + --arg type "A" \ + --arg name "$host" \ + --arg content "$target_ip" \ + --argjson ttl 120 \ + --argjson proxied false \ + '{type:$type, name:$name, content:$content, ttl:$ttl, proxied:$proxied}') + update_resp=$(cloudflare_api_call PUT "/zones/${zone_id}/dns_records/${record_id}" "$update_payload") + if [[ "$(jq -r '.success // false' <<< "$update_resp")" != "true" ]]; then + update_err=$(jq -r '(.errors // []) | map(.message // tostring) | join("; ")' <<< "$update_resp") + log_error "Failed to update Cloudflare A record ${host}: ${update_err:-unknown error}" + return 1 + fi + + log_info "Updated Cloudflare A record: ${host}" + log_info " old: ${old_ip:-}" + log_info " new: ${target_ip}" + return 0 +} + caddyfile_has_domain_block() { local file="$1" domain="$2" awk -v domain="$domain" ' @@ -322,23 +472,39 @@ log_step 4 "Starting Caddy container..." CONTAINER_STATUS=$(ssh_exec UNRAID "docker ps --filter name=caddy --format '{{.Status}}'" 2>/dev/null || true) if [[ "$CONTAINER_STATUS" == *"Up"* ]]; then log_info "Caddy container already running" - if [[ "$CADDYFILE_UPDATED" -eq 1 ]]; then - log_info "Caddyfile changed — restarting caddy to apply updated config" - ssh_exec UNRAID "docker restart caddy >/dev/null" - log_success "Caddy container restarted with new config" + log_info "Reloading Caddy config from /etc/caddy/Caddyfile" + if ssh_exec UNRAID "docker exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile" >/dev/null 2>&1; then + log_success "Caddy config reloaded" else - log_info "Caddyfile unchanged — restart not required" + log_warn "Caddy reload failed; restarting caddy container" + ssh_exec UNRAID "docker restart caddy >/dev/null" + log_success "Caddy container restarted" fi else ssh_exec UNRAID "cd '${CADDY_COMPOSE_DIR}' && docker compose up -d 2>/dev/null || docker-compose up -d" - log_success "Caddy container started" + if ssh_exec UNRAID "docker exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile" >/dev/null 2>&1; then + log_success "Caddy container started and config loaded" + else + log_success "Caddy container started" + fi fi # --------------------------------------------------------------------------- -# Step 5: Wait for HTTPS to work +# Step 5: Ensure DNS points Gitea domain to target ingress IP +# --------------------------------------------------------------------------- +log_step 5 "Ensuring DNS for ${GITEA_DOMAIN}..." +if [[ "$TLS_MODE" == "cloudflare" ]]; then + ensure_cloudflare_dns_for_gitea "${GITEA_DOMAIN}" "${PUBLIC_DNS_TARGET_IP}" +else + log_info "TLS_MODE=${TLS_MODE}; skipping Cloudflare DNS automation" +fi + +# --------------------------------------------------------------------------- +# Step 6: Wait for HTTPS to work # Caddy auto-obtains certs — poll until HTTPS responds. # --------------------------------------------------------------------------- -log_step 5 "Waiting for HTTPS (Caddy auto-provisions cert)..." +log_step 6 "Waiting for HTTPS (Caddy auto-provisions cert)..." +check_unraid_gitea_backend if wait_for_https_public "${GITEA_DOMAIN}" 30; then log_success "HTTPS verified through current domain routing — https://${GITEA_DOMAIN} works" else @@ -348,7 +514,7 @@ else fi # --------------------------------------------------------------------------- -# Step 6: Mark GitHub repos as offsite backup only +# Step 7: Mark GitHub repos as offsite backup only # Updates description + homepage to indicate Gitea is primary. # Disables wiki and Pages to avoid unnecessary resource usage. # Does NOT archive — archived repos reject pushes, which would break @@ -356,7 +522,7 @@ fi # Persists original mutable settings to a local state file for teardown. # GitHub Actions already disabled in Phase 6 Step D. # --------------------------------------------------------------------------- -log_step 6 "Marking GitHub repos as offsite backup..." +log_step 7 "Marking GitHub repos as offsite backup..." init_phase8_state_store GITHUB_REPO_UPDATE_FAILURES=0 @@ -390,10 +556,11 @@ for repo in "${REPOS[@]}"; do --arg homepage "https://${GITEA_DOMAIN}/${GITEA_ORG_NAME}/${repo}" \ '{description: $description, homepage: $homepage, has_wiki: false, has_projects: false}') - if github_api PATCH "/repos/${GITHUB_USERNAME}/${repo}" "$UPDATE_PAYLOAD" >/dev/null 2>&1; then + if PATCH_OUT=$(github_api PATCH "/repos/${GITHUB_USERNAME}/${repo}" "$UPDATE_PAYLOAD" 2>&1); then log_success "Marked GitHub repo as mirror: ${repo}" else log_error "Failed to update GitHub repo: ${repo}" + log_error "GitHub API: $(printf '%s' "$PATCH_OUT" | tail -n 1)" GITHUB_REPO_UPDATE_FAILURES=$((GITHUB_REPO_UPDATE_FAILURES + 1)) fi diff --git a/setup/configure_env.sh b/setup/configure_env.sh index 5c9b053..d1818c8 100755 --- a/setup/configure_env.sh +++ b/setup/configure_env.sh @@ -65,7 +65,7 @@ get_env_val() { # Prompt function # --------------------------------------------------------------------------- # Base prompt count (56 fixed + 3 TLS conditional slots — repo/DB prompts added dynamically) -TOTAL_PROMPTS=59 +TOTAL_PROMPTS=61 CURRENT_PROMPT=0 LAST_SECTION="" @@ -374,11 +374,13 @@ prompt_var "CADDY_DATA_PATH" "Absolute path on host for Caddy data" # Conditional TLS prompts if [[ "$COLLECTED_TLS_MODE" == "cloudflare" ]]; then prompt_var "CLOUDFLARE_API_TOKEN" "Cloudflare API token (Zone:DNS:Edit)" nonempty "" "TLS / REVERSE PROXY" + prompt_var "PUBLIC_DNS_TARGET_IP" "Public DNS target IP for GITEA_DOMAIN" ip "" "TLS / REVERSE PROXY" + prompt_var "PHASE8_ALLOW_PRIVATE_DNS_TARGET" "Allow private RFC1918 DNS target (LAN-only/split-DNS)" bool "false" "TLS / REVERSE PROXY" # Skip cert path prompts but still count them for progress CURRENT_PROMPT=$((CURRENT_PROMPT + 2)) else # Skip cloudflare token prompt but count it - CURRENT_PROMPT=$((CURRENT_PROMPT + 1)) + CURRENT_PROMPT=$((CURRENT_PROMPT + 3)) prompt_var "SSL_CERT_PATH" "Absolute path to SSL cert" path "" "TLS / REVERSE PROXY" prompt_var "SSL_KEY_PATH" "Absolute path to SSL key" path "" "TLS / REVERSE PROXY" fi