From dc08375ad0c049490ed93248de6de1bde9052627 Mon Sep 17 00:00:00 2001 From: S Date: Sat, 28 Feb 2026 20:18:35 -0500 Subject: [PATCH] fix: address multiple bugs from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - teardown_all.sh: replace `yes |` pipeline with `< <(yes)` process substitution to avoid SIGPIPE (exit 141) false failures under pipefail - phase6_teardown.sh: extract push mirror `.id` instead of `.remote_name` to match the DELETE /push_mirrors/{id} API contract - phase5_migrate_pipelines.sh: expand sed regex from `[a-z_]*` to `[a-z_.]*` to handle nested GitHub contexts like `github.event.pull_request.number` - lib/common.sh: render_template now requires explicit variable list to prevent envsubst from eating Nginx variables ($host, $proxy_add_...) - backup scripts: remove MacBook relay, use direct Unraid↔Fedora SCP; fix dump path to write to /data/ (mounted volume) instead of /tmp/ (container-only); add unzip -t integrity verification - preflight.sh: add --skip-port-checks flag for resuming with --start-from (ports already bound by earlier phases) - run_all.sh: update run_step to pass extra args; use --skip-port-checks when --start-from > 1 - post-checks (phase4/7/9): wrap API calls in helper functions with >/dev/null redirection instead of passing -o /dev/null as API data - phase8: replace GitHub archiving with [MIRROR] description marking and disable wiki/projects/Pages (archived repos reject push mirrors) - restore_to_primary.sh: add require_vars for Fedora SSH variables Co-Authored-By: Claude Opus 4.6 --- backup/backup_primary.sh | 61 ++++++++++++++++++++---------------- backup/restore_to_primary.sh | 17 +++++----- lib/common.sh | 10 ++++-- manage_runner.sh | 12 ++++--- phase1_gitea_unraid.sh | 6 ++-- phase2_gitea_fedora.sh | 6 ++-- phase4_post_check.sh | 12 ++++--- phase5_migrate_pipelines.sh | 14 ++++----- phase6_teardown.sh | 2 +- phase7_post_check.sh | 6 ++-- phase8_cutover.sh | 58 ++++++++++++++++++++-------------- phase8_post_check.sh | 14 ++++----- phase8_teardown.sh | 28 ++++++++--------- phase9_post_check.sh | 6 ++-- phase9_security.sh | 3 +- preflight.sh | 54 ++++++++++++++++++++----------- run_all.sh | 15 +++++++-- teardown_all.sh | 8 +++-- 18 files changed, 199 insertions(+), 133 deletions(-) diff --git a/backup/backup_primary.sh b/backup/backup_primary.sh index 227aeb7..a187b66 100755 --- a/backup/backup_primary.sh +++ b/backup/backup_primary.sh @@ -12,10 +12,11 @@ set -euo pipefail # # Steps: # 1. Run `gitea dump` inside the container to create a zip archive -# 2. SCP the dump from Unraid to Fedora (offsite storage) -# 3. Clean up the dump from Unraid /tmp -# 4. Prune old backups beyond retention count -# 5. Print backup summary +# 2. SCP the dump directly from Unraid to Fedora (no MacBook relay) +# 3. Verify archive integrity on Fedora +# 4. Clean up the dump from Unraid /tmp +# 5. Prune old backups beyond retention count +# 6. Print backup summary # ============================================================================= SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" @@ -31,50 +32,56 @@ log_info "=== Gitea Primary Backup ===" # --------------------------------------------------------------------------- # Step 1: Run gitea dump inside the container # The -u git flag is important — gitea dump must run as the git user who -# owns the repository files. The dump is created in /tmp inside the container -# which maps to /tmp on the host via the default Docker tmpfs mount. +# owns the repository files. The dump is written to /data/ inside the +# container, which is mounted from ${DATA_PATH}/data on the host. +# Writing to /data/ (mounted volume) instead of /tmp/ (container-only +# filesystem) ensures the dump is accessible from the host for SCP. # --------------------------------------------------------------------------- +DATA_PATH="$UNRAID_GITEA_DATA_PATH" TIMESTAMP=$(date +%Y%m%d-%H%M%S) DUMP_FILENAME="gitea-dump-${TIMESTAMP}.zip" -DUMP_REMOTE_PATH="/tmp/${DUMP_FILENAME}" +DUMP_CONTAINER_PATH="/data/${DUMP_FILENAME}" +DUMP_HOST_PATH="${DATA_PATH}/data/${DUMP_FILENAME}" log_info "Creating Gitea dump on Unraid..." ssh_exec UNRAID "docker exec -u git gitea gitea dump \ -c /data/gitea/conf/app.ini \ - -f '${DUMP_REMOTE_PATH}'" + -f '${DUMP_CONTAINER_PATH}'" log_success "Dump created: ${DUMP_FILENAME}" # --------------------------------------------------------------------------- -# Step 2: Create backup storage directory on Fedora and transfer dump -# The dump goes from Unraid → local machine → Fedora because direct -# Unraid→Fedora SCP may not have SSH keys set up. Using the MacBook as -# a relay is more reliable with our existing SSH config. +# Step 2: Transfer dump directly from Unraid to Fedora +# Uses SSH from Unraid to SCP the file to Fedora. This avoids relaying +# through the MacBook, which would be slow for large dumps and requires +# the MacBook to be online. # --------------------------------------------------------------------------- log_info "Transferring dump to Fedora backup storage..." ssh_exec FEDORA "mkdir -p '${BACKUP_STORAGE_PATH}'" -# SCP from Unraid to local temp, then to Fedora -LOCAL_TMP=$(mktemp -d) -scp_to_local() { - local ip_var="UNRAID_IP" user_var="UNRAID_SSH_USER" port_var="UNRAID_SSH_PORT" - local ip="${!ip_var:-}" user="${!user_var:-}" port="${!port_var:-22}" - scp -o ConnectTimeout=10 -o BatchMode=yes -P "$port" \ - "${user}@${ip}:${DUMP_REMOTE_PATH}" "${LOCAL_TMP}/${DUMP_FILENAME}" -} -scp_to_local -scp_to FEDORA "${LOCAL_TMP}/${DUMP_FILENAME}" "${BACKUP_STORAGE_PATH}/${DUMP_FILENAME}" -rm -rf "$LOCAL_TMP" +FEDORA_PORT="${FEDORA_SSH_PORT:-22}" +ssh_exec UNRAID "scp -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + -o BatchMode=yes -P '${FEDORA_PORT}' \ + '${DUMP_HOST_PATH}' '${FEDORA_SSH_USER}@${FEDORA_IP}:${BACKUP_STORAGE_PATH}/${DUMP_FILENAME}'" log_success "Dump transferred to Fedora: ${BACKUP_STORAGE_PATH}/${DUMP_FILENAME}" # --------------------------------------------------------------------------- -# Step 3: Clean up dump from Unraid /tmp +# Step 3: Verify archive integrity on Fedora +# CRC-checks every file in the zip. If corrupt, set -e aborts before +# pruning old (known-good) backups. +# --------------------------------------------------------------------------- +log_info "Verifying archive integrity..." +ssh_exec FEDORA "unzip -t '${BACKUP_STORAGE_PATH}/${DUMP_FILENAME}'" >/dev/null +log_success "Archive integrity verified" + +# --------------------------------------------------------------------------- +# Step 4: Clean up dump from Unraid /tmp # No reason to keep the dump on Unraid — it's on Fedora now. # --------------------------------------------------------------------------- -ssh_exec UNRAID "rm -f '${DUMP_REMOTE_PATH}'" +ssh_exec UNRAID "rm -f '${DUMP_HOST_PATH}'" log_info "Cleaned up dump from Unraid" # --------------------------------------------------------------------------- -# Step 4: Prune old backups beyond retention count +# Step 5: Prune old backups beyond retention count # Lists all gitea-dump-*.zip files sorted by time (newest first), then # removes everything beyond BACKUP_RETENTION_COUNT. # --------------------------------------------------------------------------- @@ -85,7 +92,7 @@ REMAINING=$(ssh_exec FEDORA "ls -1 '${BACKUP_STORAGE_PATH}'/gitea-dump-*.zip 2>/ log_info "Backups remaining: ${REMAINING}" # --------------------------------------------------------------------------- -# Step 5: Summary +# Step 6: Summary # --------------------------------------------------------------------------- DUMP_SIZE=$(ssh_exec FEDORA "du -h '${BACKUP_STORAGE_PATH}/${DUMP_FILENAME}'" | awk '{print $1}') diff --git a/backup/restore_to_primary.sh b/backup/restore_to_primary.sh index f345835..ac13327 100755 --- a/backup/restore_to_primary.sh +++ b/backup/restore_to_primary.sh @@ -65,7 +65,7 @@ fi # --------------------------------------------------------------------------- # Step 1: Transfer archive to Unraid /tmp if needed # If the archive is a local file, SCP it directly. If it's on Fedora, -# we relay through the local machine. +# SCP directly from Fedora to Unraid (no MacBook relay). # --------------------------------------------------------------------------- log_step 1 "Preparing archive..." ARCHIVE_NAME=$(basename "$ARCHIVE_PATH") @@ -76,16 +76,13 @@ if [[ -f "$ARCHIVE_PATH" ]]; then log_info "Uploading local archive to Unraid..." scp_to UNRAID "$ARCHIVE_PATH" "$UNRAID_ARCHIVE" else - # Assume path is on Fedora — relay through local machine + # Assume path is on Fedora — SCP directly from Fedora to Unraid + require_vars FEDORA_IP FEDORA_SSH_USER UNRAID_IP UNRAID_SSH_USER log_info "Transferring archive from Fedora to Unraid..." - LOCAL_TMP=$(mktemp -d) - # SCP from Fedora to local - ip="${FEDORA_IP:-}" user="${FEDORA_SSH_USER:-}" port="${FEDORA_SSH_PORT:-22}" - scp -o ConnectTimeout=10 -o BatchMode=yes -P "$port" \ - "${user}@${ip}:${ARCHIVE_PATH}" "${LOCAL_TMP}/${ARCHIVE_NAME}" - # SCP from local to Unraid - scp_to UNRAID "${LOCAL_TMP}/${ARCHIVE_NAME}" "$UNRAID_ARCHIVE" - rm -rf "$LOCAL_TMP" + UNRAID_PORT="${UNRAID_SSH_PORT:-22}" + ssh_exec FEDORA "scp -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + -o BatchMode=yes -P '${UNRAID_PORT}' \ + '${ARCHIVE_PATH}' '${UNRAID_SSH_USER}@${UNRAID_IP}:${UNRAID_ARCHIVE}'" fi log_success "Archive ready on Unraid: ${UNRAID_ARCHIVE}" diff --git a/lib/common.sh b/lib/common.sh index 3f9f0bd..edd4b90 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -303,14 +303,20 @@ github_api() { # --------------------------------------------------------------------------- render_template() { - local src="$1" dest="$2" + local src="$1" dest="$2" vars="${3:-}" if [[ ! -f "$src" ]]; then log_error "Template not found: $src" return 1 fi - envsubst < "$src" > "$dest" + if [[ -z "$vars" ]]; then + log_error "render_template requires an explicit variable list (third argument)" + log_error "Example: render_template src dest '\${VAR1} \${VAR2}'" + return 1 + fi + + envsubst "$vars" < "$src" > "$dest" } # --------------------------------------------------------------------------- diff --git a/manage_runner.sh b/manage_runner.sh index 29db783..2494fd2 100755 --- a/manage_runner.sh +++ b/manage_runner.sh @@ -163,13 +163,15 @@ add_docker_runner() { tmpfile=$(mktemp) export RUNNER_NAME RUNNER_LABELS RUNNER_DATA_PATH export GITEA_RUNNER_REGISTRATION_TOKEN="${GITEA_RUNNER_REGISTRATION_TOKEN:-}" - render_template "${SCRIPT_DIR}/templates/docker-compose-runner.yml.tpl" "$tmpfile" + render_template "${SCRIPT_DIR}/templates/docker-compose-runner.yml.tpl" "$tmpfile" \ + '${ACT_RUNNER_VERSION} ${RUNNER_NAME} ${GITEA_INTERNAL_URL} ${GITEA_RUNNER_REGISTRATION_TOKEN} ${RUNNER_LABELS} ${RUNNER_DATA_PATH}' runner_scp "$tmpfile" "${RUNNER_DATA_PATH}/docker-compose.yml" rm -f "$tmpfile" # Render runner config tmpfile=$(mktemp) - render_template "${SCRIPT_DIR}/templates/runner-config.yaml.tpl" "$tmpfile" + render_template "${SCRIPT_DIR}/templates/runner-config.yaml.tpl" "$tmpfile" \ + '${RUNNER_NAME} ${RUNNER_LABELS}' runner_scp "$tmpfile" "${RUNNER_DATA_PATH}/config.yaml" rm -f "$tmpfile" @@ -246,13 +248,15 @@ add_native_runner() { local tmpfile tmpfile=$(mktemp) export RUNNER_NAME RUNNER_LABELS RUNNER_DATA_PATH - render_template "${SCRIPT_DIR}/templates/runner-config.yaml.tpl" "$tmpfile" + render_template "${SCRIPT_DIR}/templates/runner-config.yaml.tpl" "$tmpfile" \ + '${RUNNER_NAME} ${RUNNER_LABELS}' cp "$tmpfile" "${RUNNER_DATA_PATH}/config.yaml" rm -f "$tmpfile" # Render launchd plist tmpfile=$(mktemp) - render_template "${SCRIPT_DIR}/templates/com.gitea.runner.plist.tpl" "$tmpfile" + render_template "${SCRIPT_DIR}/templates/com.gitea.runner.plist.tpl" "$tmpfile" \ + '${RUNNER_NAME} ${RUNNER_DATA_PATH}' mkdir -p "$HOME/Library/LaunchAgents" cp "$tmpfile" "$plist_path" rm -f "$tmpfile" diff --git a/phase1_gitea_unraid.sh b/phase1_gitea_unraid.sh index ea52dc8..963ebb0 100755 --- a/phase1_gitea_unraid.sh +++ b/phase1_gitea_unraid.sh @@ -45,7 +45,8 @@ else TMPFILE=$(mktemp) # Set variables for template export DATA_PATH GITEA_PORT="${UNRAID_GITEA_PORT}" GITEA_SSH_PORT="${UNRAID_GITEA_SSH_PORT}" - render_template "${SCRIPT_DIR}/templates/docker-compose-gitea.yml.tpl" "$TMPFILE" + render_template "${SCRIPT_DIR}/templates/docker-compose-gitea.yml.tpl" "$TMPFILE" \ + '${GITEA_VERSION} ${DATA_PATH} ${GITEA_PORT} ${GITEA_SSH_PORT}' scp_to UNRAID "$TMPFILE" "${DATA_PATH}/docker-compose.yml" rm -f "$TMPFILE" log_success "docker-compose.yml deployed" @@ -62,7 +63,8 @@ else # Generate a random secret key for this instance GITEA_SECRET_KEY=$(openssl rand -hex 32) export GITEA_SECRET_KEY - render_template "${SCRIPT_DIR}/templates/app.ini.tpl" "$TMPFILE" + render_template "${SCRIPT_DIR}/templates/app.ini.tpl" "$TMPFILE" \ + '${GITEA_DOMAIN} ${GITEA_DB_TYPE} ${GITEA_SECRET_KEY}' scp_to UNRAID "$TMPFILE" "${DATA_PATH}/config/app.ini" rm -f "$TMPFILE" log_success "app.ini deployed" diff --git a/phase2_gitea_fedora.sh b/phase2_gitea_fedora.sh index fc877dd..6003885 100755 --- a/phase2_gitea_fedora.sh +++ b/phase2_gitea_fedora.sh @@ -43,7 +43,8 @@ if ssh_exec FEDORA "test -f '${DATA_PATH}/docker-compose.yml'"; then else TMPFILE=$(mktemp) export DATA_PATH GITEA_PORT="${FEDORA_GITEA_PORT}" GITEA_SSH_PORT="${FEDORA_GITEA_SSH_PORT}" - render_template "${SCRIPT_DIR}/templates/docker-compose-gitea.yml.tpl" "$TMPFILE" + render_template "${SCRIPT_DIR}/templates/docker-compose-gitea.yml.tpl" "$TMPFILE" \ + '${GITEA_VERSION} ${DATA_PATH} ${GITEA_PORT} ${GITEA_SSH_PORT}' scp_to FEDORA "$TMPFILE" "${DATA_PATH}/docker-compose.yml" rm -f "$TMPFILE" log_success "docker-compose.yml deployed" @@ -67,7 +68,8 @@ else # the Fedora instance doesn't have a public domain GITEA_DOMAIN="${FEDORA_IP}:${FEDORA_GITEA_PORT}" export GITEA_DOMAIN - render_template "${SCRIPT_DIR}/templates/app.ini.tpl" "$TMPFILE" + render_template "${SCRIPT_DIR}/templates/app.ini.tpl" "$TMPFILE" \ + '${GITEA_DOMAIN} ${GITEA_DB_TYPE} ${GITEA_SECRET_KEY}' scp_to FEDORA "$TMPFILE" "${DATA_PATH}/config/app.ini" rm -f "$TMPFILE" log_success "app.ini deployed" diff --git a/phase4_post_check.sh b/phase4_post_check.sh index d5130b2..337f0f9 100755 --- a/phase4_post_check.sh +++ b/phase4_post_check.sh @@ -44,8 +44,10 @@ for repo in "${REPOS[@]}"; do log_info "--- Checking repo: ${repo} ---" # Check 1: Repo exists on primary - run_check "Primary: ${GITEA_ORG_NAME}/${repo} exists" \ - gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}" -o /dev/null + check_repo_exists() { + gitea_api GET "/repos/${GITEA_ORG_NAME}/$1" >/dev/null + } + run_check "Primary: ${GITEA_ORG_NAME}/${repo} exists" check_repo_exists "$repo" # Check 2: Repo has commits (migration imported content) check_commits() { @@ -67,8 +69,10 @@ for repo in "${REPOS[@]}"; do run_check "Primary: ${repo} default branch matches GitHub" check_default_branch "$repo" # Check 4: Mirror exists on Fedora - run_check "Fedora: ${GITEA_ADMIN_USER}/${repo} exists" \ - gitea_backup_api GET "/repos/${GITEA_ADMIN_USER}/${repo}" -o /dev/null + check_mirror_exists() { + gitea_backup_api GET "/repos/${GITEA_ADMIN_USER}/$1" >/dev/null + } + run_check "Fedora: ${GITEA_ADMIN_USER}/${repo} exists" check_mirror_exists "$repo" # Check 5: Mirror has mirror=true check_mirror_flag() { diff --git a/phase5_migrate_pipelines.sh b/phase5_migrate_pipelines.sh index 0c9f92f..502dcd2 100755 --- a/phase5_migrate_pipelines.sh +++ b/phase5_migrate_pipelines.sh @@ -113,15 +113,15 @@ for repo in "${REPOS[@]}"; do cat "$dest" >> "$tmpwf" mv "$tmpwf" "$dest" - # Replace GitHub-specific context variables with Gitea equivalents - # Using sed with a temp file for portability (macOS sed -i requires '', - # GNU sed -i requires no arg — avoiding both by writing to a temp file) + # Replace GitHub-specific context variables with Gitea equivalents. + # Only match inside ${{ ... }} expression delimiters to avoid mangling + # URLs, comments, or other strings that happen to contain "github.". + # Two patterns: with spaces (${{ github.X }}) and without (${{github.X}}). + # Character class [a-z_.] covers nested contexts like github.event.pull_request.number. tmpwf=$(mktemp) sed \ - -e 's/github\.repository/gitea.repository/g' \ - -e 's/github\.event/gitea.event/g' \ - -e 's/github\.token/gitea.token/g' \ - -e 's/github\.server_url/gitea.server_url/g' \ + -e 's/\${{ github\.\([a-z_.]*\) }}/\${{ gitea.\1 }}/g' \ + -e 's/\${{github\.\([a-z_.]*\)}}/\${{gitea.\1}}/g' \ "$dest" > "$tmpwf" mv "$tmpwf" "$dest" diff --git a/phase6_teardown.sh b/phase6_teardown.sh index 34bf434..3955942 100755 --- a/phase6_teardown.sh +++ b/phase6_teardown.sh @@ -32,7 +32,7 @@ for repo in "${REPOS[@]}"; do # Get push mirror IDs (there could be multiple, delete all) MIRRORS=$(gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}/push_mirrors" 2>/dev/null || echo "[]") - MIRROR_IDS=$(printf '%s' "$MIRRORS" | jq -r '.[].remote_name' 2>/dev/null || true) + MIRROR_IDS=$(printf '%s' "$MIRRORS" | jq -r '.[].id' 2>/dev/null || true) if [[ -z "$MIRROR_IDS" ]]; then log_info "No push mirrors found for ${repo} — already clean" diff --git a/phase7_post_check.sh b/phase7_post_check.sh index e012235..4f6419f 100755 --- a/phase7_post_check.sh +++ b/phase7_post_check.sh @@ -37,8 +37,10 @@ for repo in "${REPOS[@]}"; do log_info "--- Checking repo: ${repo} ---" # Check 1: Protection rule exists - run_check "Branch protection exists for '${PROTECTED_BRANCH}' on ${repo}" \ - gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}/branch_protections/${PROTECTED_BRANCH}" -o /dev/null + check_protection_exists() { + gitea_api GET "/repos/${GITEA_ORG_NAME}/$1/branch_protections/${PROTECTED_BRANCH}" >/dev/null + } + run_check "Branch protection exists for '${PROTECTED_BRANCH}' on ${repo}" check_protection_exists "$repo" # Check 2: Push is blocked (enable_push should be false) check_push_blocked() { diff --git a/phase8_cutover.sh b/phase8_cutover.sh index 9457a45..27aefb7 100755 --- a/phase8_cutover.sh +++ b/phase8_cutover.sh @@ -58,7 +58,8 @@ render_nginx_http_only() { # Set dummy cert paths (not used in HTTP-only mode) export SSL_CERT_FULLPATH="/dev/null" export SSL_KEY_FULLPATH="/dev/null" - render_template "${SCRIPT_DIR}/templates/nginx-gitea.conf.tpl" "$rendered" + render_template "${SCRIPT_DIR}/templates/nginx-gitea.conf.tpl" "$rendered" \ + '${GITEA_DOMAIN} ${UNRAID_IP} ${UNRAID_GITEA_PORT} ${SSL_CERT_FULLPATH} ${SSL_KEY_FULLPATH}' # Strip the HTTPS server block (everything between markers inclusive) sed '/# SSL_HTTPS_BLOCK_START/,/# SSL_HTTPS_BLOCK_END/d' "$rendered" > "$tmpfile" @@ -75,7 +76,8 @@ render_nginx_https() { export GITEA_DOMAIN UNRAID_IP UNRAID_GITEA_PORT export SSL_CERT_FULLPATH="$cert_path" export SSL_KEY_FULLPATH="$key_path" - render_template "${SCRIPT_DIR}/templates/nginx-gitea.conf.tpl" "$rendered" + render_template "${SCRIPT_DIR}/templates/nginx-gitea.conf.tpl" "$rendered" \ + '${GITEA_DOMAIN} ${UNRAID_IP} ${UNRAID_GITEA_PORT} ${SSL_CERT_FULLPATH} ${SSL_KEY_FULLPATH}' # Replace the redirect block content with a 301 redirect to HTTPS # The block between markers gets replaced with just the redirect @@ -266,39 +268,47 @@ else fi # --------------------------------------------------------------------------- -# Step 11: Archive GitHub repos -# Marks repos as archived with a "[MOVED]" description pointing to Gitea. -# Preserves the original description by appending it after "— was: ". +# Step 11: Mark GitHub repos as offsite backup only +# Updates description + homepage to indicate Gitea is primary. +# Disables wiki and Pages to avoid unnecessary resource usage. +# Does NOT archive — archived repos reject pushes, which would break +# the push mirrors configured in Phase 6. +# Preserves original description by appending after "— was: ". +# GitHub Actions already disabled in Phase 6 Step D. # --------------------------------------------------------------------------- -log_step 11 "Archiving GitHub repos..." +log_step 11 "Marking GitHub repos as offsite backup..." for repo in "${REPOS[@]}"; do - # Check if already archived - IS_ARCHIVED=$(github_api GET "/repos/${GITHUB_USERNAME}/${repo}" 2>/dev/null | jq -r '.archived' || echo "false") - if [[ "$IS_ARCHIVED" == "true" ]]; then - log_info "GitHub repo ${repo} already archived — skipping" + # Fetch repo metadata (single API call) + REPO_DATA=$(github_api GET "/repos/${GITHUB_USERNAME}/${repo}" 2>/dev/null || echo "{}") + CURRENT_DESC=$(printf '%s' "$REPO_DATA" | jq -r '.description // ""') + + # Skip if already marked + if [[ "$CURRENT_DESC" == "[MIRROR]"* ]]; then + log_info "GitHub repo ${repo} already marked as mirror — skipping" continue fi - # Get original description to preserve it - ORIGINAL_DESC=$(github_api GET "/repos/${GITHUB_USERNAME}/${repo}" 2>/dev/null | jq -r '.description // ""' || echo "") - - # Build new description with moved notice - NEW_DESC="[MOVED] Now at https://${GITEA_DOMAIN}/${GITEA_ORG_NAME}/${repo}" - if [[ -n "$ORIGINAL_DESC" ]]; then - NEW_DESC="${NEW_DESC} — was: ${ORIGINAL_DESC}" + # Build new description preserving original + NEW_DESC="[MIRROR] Offsite backup — primary at https://${GITEA_DOMAIN}/${GITEA_ORG_NAME}/${repo}" + if [[ -n "$CURRENT_DESC" ]]; then + NEW_DESC="${NEW_DESC} — was: ${CURRENT_DESC}" fi - # Archive the repo with the new description - ARCHIVE_PAYLOAD=$(jq -n \ + # Update description + homepage, disable wiki and projects + UPDATE_PAYLOAD=$(jq -n \ --arg description "$NEW_DESC" \ - '{archived: true, description: $description}') + --arg homepage "https://${GITEA_DOMAIN}/${GITEA_ORG_NAME}/${repo}" \ + '{description: $description, homepage: $homepage, has_wiki: false, has_projects: false}') - if github_api PATCH "/repos/${GITHUB_USERNAME}/${repo}" "$ARCHIVE_PAYLOAD" >/dev/null 2>&1; then - log_success "Archived GitHub repo: ${repo}" + if github_api PATCH "/repos/${GITHUB_USERNAME}/${repo}" "$UPDATE_PAYLOAD" >/dev/null 2>&1; then + log_success "Marked GitHub repo as mirror: ${repo}" else - log_error "Failed to archive GitHub repo: ${repo}" + log_error "Failed to update GitHub repo: ${repo}" fi + + # Disable GitHub Pages if enabled (Pages can incur bandwidth costs) + github_api DELETE "/repos/${GITHUB_USERNAME}/${repo}/pages" >/dev/null 2>&1 || true done # --------------------------------------------------------------------------- @@ -306,4 +316,4 @@ done # --------------------------------------------------------------------------- printf '\n' log_success "Phase 8 complete — Gitea is live at https://${GITEA_DOMAIN}" -log_info "GitHub repos have been archived. Gitea is now the primary git host." +log_info "GitHub repos marked as offsite backup. Push mirrors remain active." diff --git a/phase8_post_check.sh b/phase8_post_check.sh index 1b558cc..e5d09c4 100755 --- a/phase8_post_check.sh +++ b/phase8_post_check.sh @@ -7,7 +7,7 @@ set -euo pipefail # 1. HTTPS works with valid cert # 2. HTTP redirects to HTTPS # 3. All repos accessible via HTTPS -# 4. GitHub repos are archived +# 4. GitHub repos are marked as offsite backup # Exits 0 only if ALL checks pass. # ============================================================================= @@ -64,14 +64,14 @@ for repo in "${REPOS[@]}"; do curl -sf -o /dev/null -H "Authorization: token ${GITEA_ADMIN_TOKEN}" "https://${GITEA_DOMAIN}/api/v1/repos/${GITEA_ORG_NAME}/${repo}" done -# Check 5: GitHub repos are archived +# Check 5: GitHub repos are marked as offsite backup for repo in "${REPOS[@]}"; do - check_archived() { - local is_archived - is_archived=$(github_api GET "/repos/${GITHUB_USERNAME}/$1" | jq -r '.archived') - [[ "$is_archived" == "true" ]] + check_mirror_marked() { + local desc + desc=$(github_api GET "/repos/${GITHUB_USERNAME}/$1" | jq -r '.description // ""') + [[ "$desc" == "[MIRROR]"* ]] } - run_check "GitHub repo ${repo} is archived" check_archived "$repo" + run_check "GitHub repo ${repo} marked as mirror" check_mirror_marked "$repo" done # Summary diff --git a/phase8_teardown.sh b/phase8_teardown.sh index 74baf51..15b97e0 100755 --- a/phase8_teardown.sh +++ b/phase8_teardown.sh @@ -2,12 +2,12 @@ set -euo pipefail # ============================================================================= -# phase8_teardown.sh — Reverse the cutover: remove HTTPS, un-archive GitHub +# phase8_teardown.sh — Reverse the cutover: remove HTTPS, restore GitHub repos # Steps: # 1. Remove Nginx gitea.conf + reload # 2. Remove cert renewal cron # 3. Optionally remove SSL certificates -# 4. Un-archive GitHub repos + restore original descriptions +# 4. Restore GitHub repo descriptions, re-enable wiki/projects # ============================================================================= SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -67,37 +67,35 @@ if [[ "$SSL_MODE" == "letsencrypt" ]]; then fi # --------------------------------------------------------------------------- -# Step 4: Un-archive GitHub repos + restore original descriptions -# The archive description format is: "[MOVED] ... — was: ORIGINAL_DESC" +# Step 4: Restore GitHub repos — description, wiki, projects +# The mirror description format is: "[MIRROR] ... — was: ORIGINAL_DESC" # We parse the original description from after "— was: " to restore it. # --------------------------------------------------------------------------- -printf 'Un-archive GitHub repos and restore descriptions? [y/N] ' +printf 'Restore GitHub repo descriptions and re-enable wiki/projects? [y/N] ' read -r confirm if [[ "$confirm" =~ ^[Yy]$ ]]; then for repo in "${REPOS[@]}"; do - IS_ARCHIVED=$(github_api GET "/repos/${GITHUB_USERNAME}/${repo}" 2>/dev/null | jq -r '.archived' || echo "false") - if [[ "$IS_ARCHIVED" != "true" ]]; then - log_info "GitHub repo ${repo} not archived — skipping" + CURRENT_DESC=$(github_api GET "/repos/${GITHUB_USERNAME}/${repo}" 2>/dev/null | jq -r '.description // ""') + if [[ "$CURRENT_DESC" != "[MIRROR]"* ]]; then + log_info "GitHub repo ${repo} not marked as mirror — skipping" continue fi - # Extract original description from the archived description - CURRENT_DESC=$(github_api GET "/repos/${GITHUB_USERNAME}/${repo}" 2>/dev/null | jq -r '.description // ""') + # Extract original description from the mirror description ORIGINAL_DESC="" if [[ "$CURRENT_DESC" == *" — was: "* ]]; then - # Extract everything after "— was: " ORIGINAL_DESC="${CURRENT_DESC##* — was: }" fi - # Un-archive and restore description + # Restore description, homepage, and re-enable wiki/projects RESTORE_PAYLOAD=$(jq -n \ --arg description "$ORIGINAL_DESC" \ - '{archived: false, description: $description}') + '{description: $description, homepage: "", has_wiki: true, has_projects: true}') if github_api PATCH "/repos/${GITHUB_USERNAME}/${repo}" "$RESTORE_PAYLOAD" >/dev/null 2>&1; then - log_success "Un-archived GitHub repo: ${repo}" + log_success "Restored GitHub repo: ${repo}" else - log_error "Failed to un-archive GitHub repo: ${repo}" + log_error "Failed to restore GitHub repo: ${repo}" fi done else diff --git a/phase9_post_check.sh b/phase9_post_check.sh index 387c0aa..1fa177d 100755 --- a/phase9_post_check.sh +++ b/phase9_post_check.sh @@ -38,8 +38,10 @@ for repo in "${REPOS[@]}"; do log_info "--- Checking repo: ${repo} ---" # Check 1: security-scan.yml exists - run_check "security-scan.yml exists in ${repo}" \ - gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}/contents/.gitea/workflows/security-scan.yml" -o /dev/null + check_workflow_exists() { + gitea_api GET "/repos/${GITEA_ORG_NAME}/$1/contents/.gitea/workflows/security-scan.yml" >/dev/null + } + run_check "security-scan.yml exists in ${repo}" check_workflow_exists "$repo" # Check 2: Branch protection includes security checks (if required) if [[ "$SECURITY_FAIL_ON_ERROR" == "true" ]]; then diff --git a/phase9_security.sh b/phase9_security.sh index 96f2720..f23e491 100755 --- a/phase9_security.sh +++ b/phase9_security.sh @@ -71,7 +71,8 @@ for repo in "${REPOS[@]}"; do mkdir -p "${CLONE_DIR}/.gitea/workflows" export SEMGREP_VERSION TRIVY_VERSION GITLEAKS_VERSION PROTECTED_BRANCH render_template "${SCRIPT_DIR}/templates/workflows/security-scan.yml.tpl" \ - "${CLONE_DIR}/.gitea/workflows/security-scan.yml" + "${CLONE_DIR}/.gitea/workflows/security-scan.yml" \ + '${PROTECTED_BRANCH} ${SEMGREP_VERSION} ${TRIVY_VERSION} ${GITLEAKS_VERSION}' # ------------------------------------------------------------------------- # Step 3: Commit and push diff --git a/preflight.sh b/preflight.sh index f90d344..eeb067c 100755 --- a/preflight.sh +++ b/preflight.sh @@ -4,11 +4,22 @@ set -euo pipefail # ============================================================================= # preflight.sh — Validate everything before running migration phases # Installs nothing. Exits 0 only if ALL checks pass. +# +# Usage: +# ./preflight.sh # Run all checks +# ./preflight.sh --skip-port-checks # Skip port-free checks (for --start-from) # ============================================================================= SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" source "${SCRIPT_DIR}/lib/common.sh" +SKIP_PORT_CHECKS=false +for arg in "$@"; do + case "$arg" in + --skip-port-checks) SKIP_PORT_CHECKS=true ;; + esac +done + log_info "=== Preflight Checks ===" PASS_COUNT=0 @@ -204,26 +215,33 @@ fi # Check 13: Port free on Unraid # Uses ss (socket statistics) to check if any process is listening on the port. # The ! negates the grep — we PASS if the port is NOT found in use. +# Skipped when --skip-port-checks is set (e.g. resuming with --start-from +# after phases 1-2 have Gitea already running on these ports). # --------------------------------------------------------------------------- -check_port_unraid() { - local port="${UNRAID_GITEA_PORT:-3000}" - ! ssh_exec UNRAID "ss -tlnp | grep -q ':${port} '" 2>/dev/null -} -check 13 "Port ${UNRAID_GITEA_PORT:-3000} free on Unraid" check_port_unraid -if ! check_port_unraid 2>/dev/null; then - log_error " → Port ${UNRAID_GITEA_PORT:-3000} already in use on Unraid." -fi +if [[ "$SKIP_PORT_CHECKS" == "true" ]]; then + log_info "[13] Port ${UNRAID_GITEA_PORT:-3000} free on Unraid — SKIPPED (--skip-port-checks)" + log_info "[14] Port ${FEDORA_GITEA_PORT:-3000} free on Fedora — SKIPPED (--skip-port-checks)" +else + check_port_unraid() { + local port="${UNRAID_GITEA_PORT:-3000}" + ! ssh_exec UNRAID "ss -tlnp | grep -q ':${port} '" 2>/dev/null + } + check 13 "Port ${UNRAID_GITEA_PORT:-3000} free on Unraid" check_port_unraid + if ! check_port_unraid 2>/dev/null; then + log_error " → Port ${UNRAID_GITEA_PORT:-3000} already in use on Unraid." + fi -# --------------------------------------------------------------------------- -# Check 14: Port free on Fedora -# --------------------------------------------------------------------------- -check_port_fedora() { - local port="${FEDORA_GITEA_PORT:-3000}" - ! ssh_exec FEDORA "ss -tlnp | grep -q ':${port} '" 2>/dev/null -} -check 14 "Port ${FEDORA_GITEA_PORT:-3000} free on Fedora" check_port_fedora -if ! check_port_fedora 2>/dev/null; then - log_error " → Port ${FEDORA_GITEA_PORT:-3000} already in use on Fedora." + # --------------------------------------------------------------------------- + # Check 14: Port free on Fedora + # --------------------------------------------------------------------------- + check_port_fedora() { + local port="${FEDORA_GITEA_PORT:-3000}" + ! ssh_exec FEDORA "ss -tlnp | grep -q ':${port} '" 2>/dev/null + } + check 14 "Port ${FEDORA_GITEA_PORT:-3000} free on Fedora" check_port_fedora + if ! check_port_fedora 2>/dev/null; then + log_error " → Port ${FEDORA_GITEA_PORT:-3000} already in use on Fedora." + fi fi # --------------------------------------------------------------------------- diff --git a/run_all.sh b/run_all.sh index 8ec31db..162c7a9 100755 --- a/run_all.sh +++ b/run_all.sh @@ -69,11 +69,13 @@ record_step() { STEP_RESULTS+=("$result") } -# Run a script, record pass/fail, stop on failure +# Run a script, record pass/fail, stop on failure. +# Usage: run_step "Step Name" "script.sh" [args...] run_step() { local name="$1" script="$2" + shift 2 log_info ">>> Running: ${name}" - if "${SCRIPT_DIR}/${script}"; then + if "${SCRIPT_DIR}/${script}" "$@"; then record_step "$name" "PASS" printf '\n' else @@ -126,9 +128,16 @@ fi # --------------------------------------------------------------------------- # Preflight — always runs (validates .env and infrastructure) # Even with --start-from, preflight ensures the environment is healthy. +# When resuming (--start-from > 1), port-free checks are skipped because +# earlier phases have Gitea already running on those ports. # --------------------------------------------------------------------------- log_info "=== Preflight ===" -run_step "Preflight checks" "preflight.sh" +if [[ "$START_FROM" -gt 1 ]]; then + log_info "Resuming from phase ${START_FROM} — skipping port-free checks" + run_step "Preflight checks" "preflight.sh" --skip-port-checks +else + run_step "Preflight checks" "preflight.sh" +fi # --------------------------------------------------------------------------- # Phases 1-9 — run sequentially, each followed by its post-check diff --git a/teardown_all.sh b/teardown_all.sh index 59c0c2e..672c073 100755 --- a/teardown_all.sh +++ b/teardown_all.sh @@ -105,8 +105,12 @@ for entry in "${TEARDOWNS[@]}"; do log_info ">>> Tearing down Phase ${phase_num}..." if [[ "$AUTO_YES" == "true" ]]; then - # Pipe 'y' to all prompts in the teardown script - if echo "y" | "${SCRIPT_DIR}/${script}"; then + # Feed unlimited 'y' responses via process substitution. + # A pipeline (yes | script) would break under pipefail: when the script + # finishes and closes stdin, `yes` gets SIGPIPE (exit 141), making the + # pipeline report failure even though the teardown succeeded. + # Process substitution avoids this — only the script's exit code matters. + if "${SCRIPT_DIR}/${script}" < <(yes); then PASS=$((PASS + 1)) else log_warn "Phase ${phase_num} teardown had issues (continuing)"