From e2ab5ee108f35f3164fa12e2753bc8893c7f8a41 Mon Sep 17 00:00:00 2001 From: S Date: Thu, 26 Feb 2026 15:21:50 -0600 Subject: [PATCH] =?UTF-8?q?feat:=20add=20Phase=204=20=E2=80=94=20Migrate?= =?UTF-8?q?=20Repos=20+=20Fedora=20mirrors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- phase4_migrate_repos.sh | 163 ++++++++++++++++++++++++++++++++++++++++ phase4_post_check.sh | 92 +++++++++++++++++++++++ phase4_teardown.sh | 51 +++++++++++++ 3 files changed, 306 insertions(+) create mode 100755 phase4_migrate_repos.sh create mode 100755 phase4_post_check.sh create mode 100755 phase4_teardown.sh diff --git a/phase4_migrate_repos.sh b/phase4_migrate_repos.sh new file mode 100755 index 0000000..62780b2 --- /dev/null +++ b/phase4_migrate_repos.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# phase4_migrate_repos.sh — Import repos from GitHub + set up Fedora mirrors +# Depends on: Phase 1 + Phase 2 complete (both Gitea instances running) +# For each repo: +# 1. Import from GitHub → Unraid primary (under org, one-time migration) +# 2. Create pull mirror on Fedora (mirrors Unraid → Fedora on interval) +# Idempotent: skips repos that already exist on either instance. +# ============================================================================= + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib/common.sh" + +load_env +require_vars GITEA_ADMIN_TOKEN GITEA_BACKUP_ADMIN_TOKEN \ + GITEA_INTERNAL_URL GITEA_BACKUP_INTERNAL_URL \ + GITEA_ORG_NAME GITEA_ADMIN_USER GITEA_ADMIN_PASSWORD \ + GITHUB_USERNAME GITHUB_TOKEN \ + REPO_1_NAME REPO_2_NAME REPO_3_NAME \ + MIGRATE_ISSUES MIGRATE_LABELS MIGRATE_MILESTONES MIGRATE_WIKI \ + GITEA_BACKUP_MIRROR_INTERVAL + +phase_header 4 "Migrate Repos + Fedora Mirrors" + +# --------------------------------------------------------------------------- +# Build list of repos from REPO_N_NAME env vars +# Using an array rather than hardcoding 3 loops keeps it flexible. +# --------------------------------------------------------------------------- +REPOS=("$REPO_1_NAME" "$REPO_2_NAME" "$REPO_3_NAME") + +TOTAL=${#REPOS[@]} +SUCCESS=0 +FAILED=0 + +for repo in "${REPOS[@]}"; do + log_info "--- Processing repo: ${repo} ---" + + # ------------------------------------------------------------------------- + # Step A: Import from GitHub to Unraid primary (under org) + # Uses Gitea's migration API which clones the GitHub repo server-side. + # mirror=false because this is a one-time import — we don't want Gitea + # to keep pulling from GitHub (GitHub becomes the push mirror target later). + # ------------------------------------------------------------------------- + log_step "A" "Importing ${repo} from GitHub to Unraid..." + + # Idempotency: check if repo already exists on primary + if gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}" >/dev/null 2>&1; then + log_info "Repo '${GITEA_ORG_NAME}/${repo}' already exists on primary — skipping import" + else + # Build migration payload — JSON with all options from .env + MIGRATE_PAYLOAD=$(jq -n \ + --arg clone_addr "https://github.com/${GITHUB_USERNAME}/${repo}.git" \ + --arg auth_token "$GITHUB_TOKEN" \ + --arg repo_owner "$GITEA_ORG_NAME" \ + --arg repo_name "$repo" \ + --arg service "github" \ + --argjson issues "${MIGRATE_ISSUES}" \ + --argjson labels "${MIGRATE_LABELS}" \ + --argjson milestones "${MIGRATE_MILESTONES}" \ + --argjson wiki "${MIGRATE_WIKI}" \ + '{ + clone_addr: $clone_addr, + auth_token: $auth_token, + repo_owner: $repo_owner, + repo_name: $repo_name, + service: $service, + mirror: false, + issues: $issues, + labels: $labels, + milestones: $milestones, + wiki: $wiki + }') + + if gitea_api POST "/repos/migrate" "$MIGRATE_PAYLOAD" >/dev/null; then + log_success "Migration started for ${repo}" + else + log_error "Failed to start migration for ${repo}" + FAILED=$((FAILED + 1)) + continue + fi + + # Wait for migration to complete — poll until repo has content + # The migration API returns immediately but cloning happens async. + # We check for commits to confirm the repo has actual content. + log_info "Waiting for migration to complete..." + local_elapsed=0 + while [[ $local_elapsed -lt 120 ]]; do + COMMITS=$(gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}/commits?limit=1" 2>/dev/null || echo "[]") + COMMIT_COUNT=$(printf '%s' "$COMMITS" | jq 'length' 2>/dev/null || echo 0) + if [[ "$COMMIT_COUNT" -gt 0 ]]; then + log_success "Repo ${repo} migrated with commits" + break + fi + sleep 3 + local_elapsed=$((local_elapsed + 3)) + done + + if [[ $local_elapsed -ge 120 ]]; then + log_error "Timeout waiting for ${repo} migration to complete" + FAILED=$((FAILED + 1)) + continue + fi + fi + + # ------------------------------------------------------------------------- + # Step B: Create pull mirror on Fedora + # The Fedora instance mirrors the Unraid primary on a schedule. This uses + # Gitea's migration API with mirror=true, which creates a mirror repo that + # periodically pulls from the source URL. + # The mirror authenticates to the primary using admin credentials since + # the primary's repos may be private. + # ------------------------------------------------------------------------- + log_step "B" "Creating pull mirror for ${repo} on Fedora..." + + # Idempotency: check if mirror repo already exists on Fedora + if gitea_backup_api GET "/repos/${GITEA_ADMIN_USER}/${repo}" >/dev/null 2>&1; then + log_info "Mirror '${GITEA_ADMIN_USER}/${repo}' already exists on Fedora — skipping" + else + # The clone URL points to the primary Gitea instance (Unraid) + MIRROR_PAYLOAD=$(jq -n \ + --arg clone_addr "${GITEA_INTERNAL_URL}/${GITEA_ORG_NAME}/${repo}.git" \ + --arg auth_username "$GITEA_ADMIN_USER" \ + --arg auth_password "$GITEA_ADMIN_PASSWORD" \ + --arg repo_owner "$GITEA_ADMIN_USER" \ + --arg repo_name "$repo" \ + --arg mirror_interval "$GITEA_BACKUP_MIRROR_INTERVAL" \ + '{ + clone_addr: $clone_addr, + auth_username: $auth_username, + auth_password: $auth_password, + repo_owner: $repo_owner, + repo_name: $repo_name, + service: "gitea", + mirror: true, + mirror_interval: $mirror_interval + }') + + if gitea_backup_api POST "/repos/migrate" "$MIRROR_PAYLOAD" >/dev/null; then + log_success "Pull mirror created for ${repo} on Fedora" + else + log_error "Failed to create pull mirror for ${repo}" + FAILED=$((FAILED + 1)) + continue + fi + fi + + SUCCESS=$((SUCCESS + 1)) +done + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +printf '\n' +log_info "Results: ${SUCCESS} succeeded, ${FAILED} failed (out of ${TOTAL})" + +if [[ $FAILED -gt 0 ]]; then + log_error "Some repos failed — check logs above" + exit 1 +fi + +log_success "Phase 4 complete — all repos migrated and mirrored" diff --git a/phase4_post_check.sh b/phase4_post_check.sh new file mode 100755 index 0000000..d5130b2 --- /dev/null +++ b/phase4_post_check.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# phase4_post_check.sh — Verify Phase 4 (Repo Migration) succeeded +# Checks for each repo: +# 1. Exists on primary under the org +# 2. Has at least one commit (migration actually imported content) +# 3. Default branch matches GitHub source +# 4. Mirror exists on Fedora under admin user +# 5. Mirror has mirror=true flag set +# Exits 0 only if ALL checks pass. +# ============================================================================= + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib/common.sh" + +load_env +require_vars GITEA_ADMIN_TOKEN GITEA_BACKUP_ADMIN_TOKEN \ + GITEA_INTERNAL_URL GITEA_BACKUP_INTERNAL_URL \ + GITEA_ORG_NAME GITEA_ADMIN_USER \ + GITHUB_USERNAME GITHUB_TOKEN \ + REPO_1_NAME REPO_2_NAME REPO_3_NAME + +log_info "=== Phase 4 Post-Check ===" + +REPOS=("$REPO_1_NAME" "$REPO_2_NAME" "$REPO_3_NAME") +PASS=0 +FAIL=0 + +# Helper: run a check, track results +run_check() { + local description="$1"; shift + if "$@" 2>/dev/null; then + log_success "$description" + PASS=$((PASS + 1)) + else + log_error "FAIL: $description" + FAIL=$((FAIL + 1)) + fi +} + +for repo in "${REPOS[@]}"; do + log_info "--- Checking repo: ${repo} ---" + + # Check 1: Repo exists on primary + run_check "Primary: ${GITEA_ORG_NAME}/${repo} exists" \ + gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}" -o /dev/null + + # Check 2: Repo has commits (migration imported content) + check_commits() { + local commits + commits=$(gitea_api GET "/repos/${GITEA_ORG_NAME}/$1/commits?limit=1") + local count + count=$(printf '%s' "$commits" | jq 'length') + [[ "$count" -gt 0 ]] + } + run_check "Primary: ${repo} has commits" check_commits "$repo" + + # Check 3: Default branch matches GitHub source + check_default_branch() { + local gitea_branch github_branch + gitea_branch=$(gitea_api GET "/repos/${GITEA_ORG_NAME}/$1" | jq -r '.default_branch') + github_branch=$(github_api GET "/repos/${GITHUB_USERNAME}/$1" | jq -r '.default_branch') + [[ "$gitea_branch" == "$github_branch" ]] + } + run_check "Primary: ${repo} default branch matches GitHub" check_default_branch "$repo" + + # Check 4: Mirror exists on Fedora + run_check "Fedora: ${GITEA_ADMIN_USER}/${repo} exists" \ + gitea_backup_api GET "/repos/${GITEA_ADMIN_USER}/${repo}" -o /dev/null + + # Check 5: Mirror has mirror=true + check_mirror_flag() { + local is_mirror + is_mirror=$(gitea_backup_api GET "/repos/${GITEA_ADMIN_USER}/$1" | jq -r '.mirror') + [[ "$is_mirror" == "true" ]] + } + run_check "Fedora: ${repo} has mirror=true" check_mirror_flag "$repo" +done + +# Summary +printf '\n' +log_info "Results: ${PASS} passed, ${FAIL} failed" + +if [[ $FAIL -gt 0 ]]; then + log_error "Phase 4 post-check FAILED" + exit 1 +else + log_success "Phase 4 post-check PASSED — all repos migrated and mirrored" + exit 0 +fi diff --git a/phase4_teardown.sh b/phase4_teardown.sh new file mode 100755 index 0000000..32b81a1 --- /dev/null +++ b/phase4_teardown.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# phase4_teardown.sh — Delete migrated repos from primary + mirrors from Fedora +# Destructive: permanently deletes repositories. +# Prompts for confirmation before each deletion. +# Safe to run if repos have already been deleted (no errors). +# ============================================================================= + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib/common.sh" + +load_env +require_vars GITEA_ADMIN_TOKEN GITEA_BACKUP_ADMIN_TOKEN \ + GITEA_INTERNAL_URL GITEA_BACKUP_INTERNAL_URL \ + GITEA_ORG_NAME GITEA_ADMIN_USER \ + REPO_1_NAME REPO_2_NAME REPO_3_NAME + +log_warn "=== Phase 4 Teardown: Repos + Mirrors ===" + +REPOS=("$REPO_1_NAME" "$REPO_2_NAME" "$REPO_3_NAME") + +printf 'This will DELETE all migrated repos and mirrors. Continue? [y/N] ' +read -r confirm +if [[ ! "$confirm" =~ ^[Yy]$ ]]; then + log_info "Teardown cancelled" + exit 0 +fi + +for repo in "${REPOS[@]}"; do + log_info "--- Tearing down: ${repo} ---" + + # Delete repo from primary (under org) + if gitea_api GET "/repos/${GITEA_ORG_NAME}/${repo}" >/dev/null 2>&1; then + gitea_api DELETE "/repos/${GITEA_ORG_NAME}/${repo}" >/dev/null || true + log_success "Deleted ${GITEA_ORG_NAME}/${repo} from primary" + else + log_info "Repo ${GITEA_ORG_NAME}/${repo} not found on primary — already deleted" + fi + + # Delete mirror from Fedora (under admin user) + if gitea_backup_api GET "/repos/${GITEA_ADMIN_USER}/${repo}" >/dev/null 2>&1; then + gitea_backup_api DELETE "/repos/${GITEA_ADMIN_USER}/${repo}" >/dev/null || true + log_success "Deleted mirror ${GITEA_ADMIN_USER}/${repo} from Fedora" + else + log_info "Mirror ${GITEA_ADMIN_USER}/${repo} not found on Fedora — already deleted" + fi +done + +log_success "Phase 4 teardown complete"