diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7aca41f..7145591 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,23 +1,32 @@ { - "name": "Codespace to bootstrap valet in a Codespace", - //Use base codespace image then pull Valet on postCreateCommand, - "image": "mcr.microsoft.com/vscode/devcontainers/universal:linux", - "remoteUser": "codespace", - "overrideCommand": false, - "mounts": [ - "source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume" - ], - "runArgs": [ - "--cap-add=SYS_PTRACE", - "--security-opt", - "seccomp=unconfined", - "--privileged", - "--init" - ], - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-azuretools.vscode-docker", - ], - "onCreateCommand": 'echo "export GITHUB_ACCESS_TOKEN=$VALET_GHCR_PASSWORD" >> ~/.bashrc', - "postCreateCommand": "sudo bash .devcontainer/setupcodespace.sh ${VALET_GHCR_PASSWORD} ${AZURE_DEVOPS_PROJECT} ${AZURE_DEVOPS_ORGANIZATION} ${AZURE_DEVOPS_ACCESS_TOKEN} ${GITHUB_USER} 'https://github.com/' && gh extension install github/gh-valet || echo 'Could not auto-build. Skipping.' " + "name": "Codespace to perform Valet Labs", + "image": "mcr.microsoft.com/vscode/devcontainers/universal:linux", + "remoteUser": "codespace", + "overrideCommand": false, + "mounts": [ + "source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume" + ], + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--privileged", + "--init" + ], + "remoteEnv": { + "DOCKER_ARGS": "--network=host", + "INSTALLATION_TYPE": "labs" + }, + "customizations": { + "vscode": { + "settings": { + "files.autoSave": "onFocusChange", + "editor.tabSize": 2 + }, + "extensions": [ + "ms-azuretools.vscode-docker" + ] + } + }, + "postCreateCommand": "gh extension install github/gh-valet || echo 'Could not auto-build. Skipping.' " } \ No newline at end of file diff --git a/.devcontainer/setupcodespace.sh b/.devcontainer/setupcodespace.sh deleted file mode 100644 index c98469a..0000000 --- a/.devcontainer/setupcodespace.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -echo "Starting setupcodespace.sh" - -azdoProject="AZURE_DEVOPS_PROJECT=" -azdoOrg="AZURE_DEVOPS_ORGANIZATION=" -azdoInstance="AZURE_DEVOPS_INSTANCE_URL=" -ghAccess="GITHUB_ACCESS_TOKEN=" -azdoAccess="AZURE_DEVOPS_ACCESS_TOKEN=" -ghInstanceUrl="GITHUB_INSTANCE_URL=" - -if [ -z "$1" -o -z "$5" ] -then - echo "Error: Docker Pull Valet not executing because GITHUB_USER and/or VALET_PASSWORD not set" -else - docker login ghcr.io/valet-customers --username $5 --password $1 - docker pull ghcr.io/valet-customers/valet-cli:latest - echo "Success: Docker Pull Valet completed" -fi - -value=`cat valet/.env.local.template` -echo "$value" > valet/.env.local - -if [ -z "$1" -o -z "$2" -o -z "$3" -o -z "$4" -o -z "$6" ] -then - echo "Error: Set envars not set, valid values not passed in. You will have to manually use the valet/.env.local folder" -fi - -echo "Finished setupcodespace.sh" diff --git a/.github/workflows/valetbsazdo.yml b/.github/workflows/valetbsazdo.yml deleted file mode 100644 index bdcda6f..0000000 --- a/.github/workflows/valetbsazdo.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This is a workflow to set up an Azure DevOps instance seeded with pipelines -# You need to Add a secret for an Azure DevOps PAT called AZDOPAT - -name: Valet Bootstrap for Azure DevOps - -on: - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - inputs: - AzDoOrgName: - description: 'Azure DevOps organization name' - required: true - type: string - AzDoUserName: - description: 'Azure DevOps user name' - required: true - type: string - AzDoProjectName: - description: 'Azure DevOps project name' - default: 'ValetBootstrap' - type: string -jobs: - bootstrap: - name: Bootstrap - runs-on: windows-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - name: Run powershell file to bootstrap Azure DevOps - env: - AZDOPATTOUSE: ${{ secrets.AZURE_DEVOPS_ACCESS_TOKEN }} - ROOTFOLDER: ${{ github.workspace }} - shell: pwsh - run: | - ./azure_devops/bootstrap/azDevOps.ps1 -project ${{ github.event.inputs.AzDoProjectName }} -userName ${{ github.event.inputs.AzDoUserName }} -repoName ${{ github.event.inputs.AzDoProjectName }} -orgToUse ${{ github.event.inputs.AzDoOrgName }} -rootFolder $env:ROOTFOLDER -daPassword $env:AZDOPATTOUSE diff --git a/.gitignore b/.gitignore index c05f815..46e2c04 100644 --- a/.gitignore +++ b/.gitignore @@ -1,356 +1,3 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# valet files -.env*local -credentials_file.yaml - -.vscode \ No newline at end of file +.env.local +.cache/ +tmp/ diff --git a/azure_devops/1-configure.md b/azure_devops/1-configure.md new file mode 100644 index 0000000..0f0fc00 --- /dev/null +++ b/azure_devops/1-configure.md @@ -0,0 +1,76 @@ +# Configure credentials for Valet + +In this lab, you will use the `configure` CLI command to set the required credentials and information for Valet to use when working with Azure DevOps and GitHub. + +You will need to complete all of the setup instructions [here](./readme.md#configure-your-codespace) prior to performing this lab. + +## Configuring credentials + +1. Create an Azure DevOps personal access token (PAT). + + __Note__: you can skip this step if you still have the PAT created during the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization). + - Navigate to your existing organization () in your browser. + - In the top right corner of your screen, click `User settings`. + - Click `Personal access tokens`. + - Select `+ New Token` + - Name your token, select the organization where you want to use the token, and set your token to automatically expire after a set number of days. + - Select the following scopes (you may need to `Show more scopes` to reveal all scopes): + - Agents Pool: `Read` + - Build: `Read & Execute` + - Code: `Read & Write` + - Project and Team: `Read, Write, & Manage` + - Release: `Read` + - Service Connections: `Read` + - Task Groups: `Read` + - Variable Groups: `Read` + - Click `Create`. + - Copy the generated API token and save it in a safe location. + +2. Create a GitHub personal access token (PAT): + - Open github.com in a new browser tab. + - In the top right corner of the UI, click your profile photo and then click `Settings`. + - In the left panel, click `Developer Settings`. + - Click `Personal access tokens` and then `Legacy tokens` (if present). + - Click `Generate new token` and then `Generate new legacy token`. You may be required to authenticate with GitHub during this step. + - Select the following scopes: `workflow` and `read:packages`. + - Click `Generate token`. + - Copy the generated PAT and save it in a safe location. + +3. Run the `configure` CLI command: + - Select the `TERMINAL` tab from within the codespace terminal. + - Run the following command: `gh valet configure`. + - Use the down arrow key to highlight `Azure DevOps`, press the spacebar to select, and then press enter to continue. + - At the GitHub Container Registry prompt, enter the GitHub PAT generated in step 2 and press enter. + - At the GitHub PAT prompt, enter the GitHub PAT generated in step 2 and press enter. + - At the GitHub URL prompt, enter the GitHub instance URL or press enter to accept the default value (`https://github.com`). + - At the Azure DevOps token prompt, enter the access token from step 1 and press enter. + - At the Azure DevOps URL prompt, enter your Azure DevOps URL or press enter to accept the default value (`https://dev.azure.com`). + - At the prompt, enter your Azure DevOps organization name. This should be the same organization used in the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization). + - At the prompt, enter your Azure DevOps project name. This should be the same project name used in the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization). + + ![img](https://user-images.githubusercontent.com/18723510/187771230-27c97889-d98a-48f7-af01-c1a9f8df6423.png) + +## Verify your environment + +To verify our environment is configured correctly, run the `update` CLI command. The `update` CLI command will download the latest version of Valet to your codespace. + +1. In the codespace terminal, run the following command: + + ```bash + gh valet update + ``` + +2. You should see a confirmation that you were logged into the GitHub Container Registry and Valet was updated to the latest version. + + ```console + $ gh valet update + Login Succeeded + latest: Pulling from valet-customers/valet-cli + Digest: sha256:a7d00dee8a37e25da59daeed44b1543f476b00fa2c41c47f48deeaf34a215bbb + Status: Image is up to date for ghcr.io/valet-customers/valet-cli:latest + ghcr.io/valet-customers/valet-cli:latest + ``` + +### Next lab + +[Perform an audit of an Azure DevOps project](2-audit.md) diff --git a/azure_devops/2-audit.md b/azure_devops/2-audit.md new file mode 100644 index 0000000..07c6613 --- /dev/null +++ b/azure_devops/2-audit.md @@ -0,0 +1,175 @@ +# Perform an audit of an Azure DevOps project + +In this lab, you will use the `audit` command to get a high-level view of all pipelines in an Azure DevOps organization or project. + +The `audit` command operates by fetching all of the pipelines defined in an Azure DevOps organization or project, converting each to their equivalent GitHub Actions workflow, and writing a report that summarizes how complete and complex of a migration is possible with Valet. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and bootstrap an Azure DevOps project. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). + +## Perform an audit + +You will now perform an audit against the bootstrapped Azure DevOps project. Answer the following questions before running this command: + +1. What is the Azure DevOps organization name that you want to audit? + - __:organization__. This should be the same organization used in the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization) + +2. What is the Azure DevOps project name that you want to audit? + - __:project__. This should be the same project name used in the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization) + +3. Where do you want to store the result? + - __tmp/audit__. This can be any path within the working directory from which Valet commands are executed. + +### Steps + +1. Navigate to the codespace terminal. +2. Run the following command from the root directory: + + ```bash + gh valet audit azure-devops --output-dir tmp/audit + ``` + + __Note__: The Azure DevOps organization and project name can be omitted from the `audit` command because they were persisted in the `.env.local` file in the [configure lab](./1-configure.md). You can optionally provide these arguments on the command line with the `--azure-devops-organization` and `--azure-devops-project` CLI options. + +3. The command will list all the files written to disk in green when the command succeeds. + +## Inspect the output files + +1. Find the `audit_summary.md` file in the file explorer. +2. Right-click the `audit_summary.md` file and select `Open Preview`. +3. This file contains details that summarize what percentage of your pipelines were converted automatically. + +### Review audit summary + +#### Pipelines + +The pipeline summary section contains high level statistics regarding the conversion rate done by Valet: + +```md +## Pipelines + +Total: **5** + +- Successful: **5 (100%)** +- Partially successful: **0 (0%)** +- Unsupported: **0 (0%)** +- Failed: **0 (0%)** + +### Job types + +Supported: **5 (100%)** + +- designer: **2** +- YAML: **3** +``` + +Here are some key terms in the "Pipelines" section in the above example: + +- __Successful__ pipelines had 100% of the pipeline constructs and individual items converted automatically to their GitHub Actions equivalent. +- __Partially successful__ pipelines had all of the pipeline constructs converted, however, there were some individual items (e.g. build tasks or build triggers) that were not converted automatically to their GitHub Actions equivalent. +- __Unsupported__ pipelines are definition types that are not supported by Valet. The following Azure DevOps pipeline types are supported: + - Classic (designer) + - YAML + - Release +- __Failed__ pipelines encountered a fatal error when being converted. This can occur for one of three reasons: + - The pipeline was misconfigured and not valid in Azure DevOps. + - Valet encountered an internal error when converting it. + - There was an unsuccessful network response, often due to invalid credentials, that caused the pipeline to be inaccessible. + +The "Job types" section will summarize which types of pipelines are being used and which are supported or unsupported by Valet. + +#### Build steps + +The build steps summary section presents an overview of the individual build steps that are used across all pipelines and how many were automatically converted by Valet. + +```md +### Build steps + +Total: **14** + +Known: **14 (100%)** + +- script: **4** +- DotNetCoreCLI@2: **2** +- 2c65196a-54fd-4a02-9be8-d9d1837b7c5d@0: **2** +- 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2: **2** +- e213ff0f-5d5c-4791-802d-52ea3e7be1f1@2: **2** +- NodeTool@0: **1** +- checkout: **1** + +Actions: **19** + +- run: **10** +- actions/checkout@v2: **6** +- nuget/setup-nuget@v1: **2** +- actions/setup-node@v2: **1** +``` + +Here are some key terms in the "Build steps" section in the above example: + +- A __known__ build step is a step that was automatically converted to an equivalent action. +- An __unknown__ build step is a step that was not automatically converted to an equivalent action. +- An __unsupported__ build step is a step that is either: + - A step that is fundamentally not supported by GitHub Actions. + - A step that is configured in a way that is incompatible with GitHub Actions. +- An __action__ is a list of the actions that were used in the converted workflows. This is important for the following scenarios: + - Gathering the list of actions to sync to your appliance if you use GitHub Enterprise Server. + - Defining an organization-level allowlist of actions that can be used. This list of actions is a comprehensive list of which actions their security and/or compliance teams will need to review. + +There is an equivalent breakdown of build triggers, environment variables, and other uncategorized items displayed in the audit summary file. + +#### Manual Tasks + +The manual tasks summary section presents an overview of the manual tasks that you will need to perform that Valet is not able to complete automatically. + +```md +### Manual tasks + +Total: **1** + +Self hosted runners: **1** + +- `mechamachine`: **1** +``` + +Here are some key terms in the "Manual tasks" section in the above example: + +- A __secret__ refers to a repository or organization level secret that is used by the converted pipelines. These secrets will need to be created manually in Actions in order for these pipelines to function properly. +- A __self-hosted runner__ refers to a label of a runner that is referenced by a converted pipeline that is not a GitHub-hosted runner. You will need to manually define these runners in order for these pipelines to function properly. + +#### Files + +The final section of the audit report provides a manifest of all of the files that are written to disk during the audit. These files include: + +```md +### Successful + +#### valet-bootstrap/valet-classic-test-import1 + +- [valet-bootstrap/valet-classic-test-import1.yml](valet-bootstrap/valet-classic-test-import1.yml) +- [valet-bootstrap/valet-classic-test-import1.config.json](valet-bootstrap/valet-classic-test-import1.config.json) + +#### valet-bootstrap/valet-classic-test-import2 + +- [valet-bootstrap/valet-classic-test-import2.yml](valet-bootstrap/valet-classic-test-import2.yml) +- [valet-bootstrap/valet-classic-test-import2.config.json](valet-bootstrap/valet-classic-test-import2.config.json) + +#### valet-bootstrap/pipelines/valet-pipeline1 + +- [valet-bootstrap/pipelines/valet-pipeline1.yml](valet-bootstrap/pipelines/valet-pipeline1.yml) +- [valet-bootstrap/pipelines/valet-pipeline1.config.json](valet-bootstrap/pipelines/valet-pipeline1.config.json) +- [valet-bootstrap/pipelines/valet-pipeline1.source.yml](valet-bootstrap/pipelines/valet-pipeline1.source.yml) +``` + +Each pipeline will have a variety of files written that include: + +- The original pipeline as it was defined in Azure DevOps. +- Any network responses used to convert a pipeline. +- The converted workflow. +- Stack traces that can used to troubleshoot a failed pipeline conversion + +### Next lab + +[Forecast potential build runner usage](3-forecast.md) diff --git a/azure_devops/3-forecast.md b/azure_devops/3-forecast.md new file mode 100644 index 0000000..6d0bd7d --- /dev/null +++ b/azure_devops/3-forecast.md @@ -0,0 +1,104 @@ +# Forecast potential build runner usage + +In this lab you will use the `forecast` command to forecast potential GitHub Actions usage by computing metrics from completed pipeline runs in your Azure DevOps project. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and bootstrap an Azure DevOps project. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). + +## Perform a forecast + +Answer the following questions before running the `forecast` command: + +1. What is the Azure DevOps organization name that you want to audit? + - __:organization__. This should be the same organization used in the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization) + +2. What is the Azure DevOps project name that you want to audit? + - __:project__. This should be the same project name used in the setup steps [here](./readme.md#bootstrap-your-azure-devops-organization) + +3. Where do you want to store the results? + - `tmp/forecast` + +### Steps + +1. Navigate to the codespace terminal. +2. Run the following command from the root directory: + + ```bash + gh valet forecast azure-devops --output-dir tmp/forecast + ``` + + __Note__: The Azure DevOps organization and project name can be omitted from the `forecast` command because they were persisted in the `.env.local` file in the [configure lab](./1-configure.md). You can optionally provide these arguments on the command line with the `--azure-devops-organization` and `--azure-devops-project` CLI options. + +3. The command will output a message that says "No jobs found" because no jobs have been executed in your bootstrapped project. + + ![img](https://user-images.githubusercontent.com/18723510/187690315-6312088d-9888-4c55-9bbf-c6f2687fa547.png) + +4. If you inspect the help menu using the `gh valet forecast --help` command, you will see a `--source-file-path` option. You can use this option to perform a `forecast` command using json files that are already present on the filesystem. These labs come bundled with sample json files located [here](./bootstrap/jobs.json). + + ![img](https://user-images.githubusercontent.com/18723510/187692843-623d4bdc-8970-4348-a632-73c8b00a40f8.png) + +5. Run the following `forecast` command while specifying the path to the sample json files: + + ```bash + gh valet forecast azure-devops --output-dir tmp/forecast --source-file-path azure_devops/bootstrap/jobs.json + ``` + +6. The command will list all the files written to disk when the command succeeds. + + ![img](https://user-images.githubusercontent.com/18723510/187694590-9121b997-0c89-4984-bbf2-84f3df2ed882.png) + +## Review the forecast report + +The forecast report, logs, and completed job data will be located within the `tmp/forecast` folder. + +1. Find the `forecast_report.md` file in the file explorer. +2. Right-click the `forecast_report.md` file and select `Open Preview`. +3. This file contains metrics used to forecast potential GitHub Actions usage. + +### Total + +The `Total` section of the forecast report contains high level statistics related to all the jobs completed after the `--start-date` CLI option: + +```md +- Job count: **84** +- Pipeline count: **32** + +- Execution time + + - Total: **82 minutes** + - Median: **0 minutes** + - P90: **2 minutes** + - Min: **0 minutes** + - Max: **4 minutes** + +- Queue time + + - Median: **0 minutes** + - P90: **1 minutes** + - Min: **0 minutes** + - Max: **5 minutes** + +- Concurrent jobs + + - Median: **0** + - P90: **0** + - Min: **0** + - Max: **5** +``` + +Here are some key terms of items defined in the forecast report: + +- The `Job count` is the total number of completed jobs. +- The `Pipeline count` is the number of unique pipelines used. +- `Execution time` describes the amount of time a runner spent on a job. This metric can be used to help plan for the cost of GitHub hosted runners. + - This metric is correlated to how much you should expect to spend in GitHub Actions. This will vary depending on the hardware used for these minutes. You can use the [Actions pricing calculator](https://github.com/pricing/calculator) to estimate a dollar amount. +- `Queue time` metrics describe the amount of time a job spent waiting for a runner to be available to execute it. +- `Concurrent jobs` metrics describe the amount of jobs running at any given time. This metric can be used to define the number of runners a customer should configure. + +Additionally, these metrics are defined for each queue of runners defined in Azure DevOps. This is especially useful if there are a mix of hosted/self-hosted runners or high/low spec machines to see metrics specific to different types of runners. + +## Next steps + +[Perform a dry-run migration of an Azure DevOps pipeline](4-dry-run.md) diff --git a/azure_devops/4-dry-run.md b/azure_devops/4-dry-run.md new file mode 100644 index 0000000..388e50b --- /dev/null +++ b/azure_devops/4-dry-run.md @@ -0,0 +1,93 @@ +# Perform a dry-run migration of an Azure DevOps pipeline + +In this lab you will use the `dry-run` command to convert an Azure DevOps pipeline to its equivalent GitHub Actions workflow. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and bootstrap an Azure DevOps project. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [audit lab](./2-audit.md). + +## Perform a dry run + +You will perform a dry run for a pipeline in the bootstrapped Azure DevOps project. Answer the following questions before running this command: + +1. What is the id of the pipeline to convert? + - __:pipeline_id__. This id can be found by: + - Navigating to the build pipelines in the bootstrapped Azure DevOps project + - Selecting the pipeline with the name "valet-pipeline1" + - Inspecting the URL to locate the pipeline id + +2. Where do you want to store the result? + - __tmp/dry-run__. This can be any path within the working directory from which Valet commands are executed. + +### Steps + +1. Navigate to your codespace terminal. +2. Run the following command from the root directory: + + ```bash + gh valet dry-run azure-devops pipeline --pipeline-id :pipeline_id --output-dir tmp/dry-run + ``` + +3. The command will list all the files written to disk when the command succeeds. +4. View the converted workflow: + - Find `tmp/dry-run` in the file explorer pane in your codespace. + - Click `valet-pipeline1.yml` to open. + +## Inspect the output files + +The files generated from the `dry-run` command represent the equivalent Actions workflow for the given Azure DevOps pipeline. The Azure DevOps pipeline and converted workflow can be seen below: + +
+ Azure DevOps pipeline 👇 + +```yaml +trigger: +- main + +pool: + vmImage: windows-latest + +steps: +- script: echo Hello, I am pipeline 1! + displayName: 'Run a one-line script' + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' +``` + +
+ +
+ Converted workflow 👇 + +```yaml +name: valet-bootstrap/pipelines/valet-pipeline1 +on: + push: + branches: + - main +jobs: + build: + runs-on: windows-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: Run a one-line script + run: echo Hello, I am pipeline 1! + - name: Run a multi-line script + run: |- + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml +``` + +
+ +Despite these two pipelines using different syntax they will function equivalently. + +## Next lab + +[Use custom transformers to customize Valet's behavior](./5-custom-transformers.md) diff --git a/azure_devops/5-custom-transformers.md b/azure_devops/5-custom-transformers.md new file mode 100644 index 0000000..5b72d1f --- /dev/null +++ b/azure_devops/5-custom-transformers.md @@ -0,0 +1,235 @@ +# Using custom transformers to customize Valet's behavior + +In this lab we will build upon the `dry-run` command to override Valet's default behavior and customize the converted workflow using "custom transformers". Custom transformers can be used to: + +1. Convert items that are not automatically converted. +2. Convert items that were automatically converted using different actions. +3. Convert environment variable values differently. +4. Convert references to runners to use a different runner name in Actions. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and bootstrap an Azure DevOps project. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [audit lab](./2-audit.md). +4. Completed the [dry-run lab](./4-dry-run.md). + +## Perform a dry run + +You will perform a dry-run for a pipeline in the bootstrapped Azure DevOps project. Answer the following questions before running this command: + +1. What is the id of the pipeline to convert? + - __:pipeline_id__. This id can be found by: + - Navigating to the build pipelines in the bootstrapped Azure DevOps project + - Selecting the pipeline with the name "valet-custom-transformer-example" + - Inspecting the URL to locate the pipeline id + +2. Where do you want to store the result? + - __tmp/dry-run__. This can be any path within the working directory from which Valet commands are executed. + +### Steps + +1. Navigate to the codespace terminal +2. Run the following command from the root directory: + + ```bash + gh valet dry-run azure-devops pipeline --pipeline-id :pipeline_id --output-dir tmp/dry-run + ``` + +3. The command will list all the files written to disk when the command succeeds. +4. View the converted workflow: + - Find `tmp/dry-run` in the file explorer pane in your codespace. + - Click `valet-custom-transformer-example.yml` to open. + +The converted workflow that is generated can be seen below: + +
+ Converted workflow 👇 + +```yaml +name: valet-bootstrap/pipelines/valet-custom-transformer-example +on: + push: + branches: + - "*" +env: + BUILDCONFIGURATION: Release + BuildParameters_RESTOREBUILDPROJECTS: "**/*.csproj" +jobs: + Job_1: + name: Agent job 1 + runs-on: + - self-hosted + - mechamachine + steps: + - name: checkout + uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - name: Use Node 10.16.3 + uses: actions/setup-node@v2 + with: + node-version: 10.16.3 + - name: Restore + run: dotnet restore ${{ env.BuildParameters_RESTOREBUILDPROJECTS }} + - name: Build + run: dotnet build ${{ env.BuildParameters_RESTOREBUILDPROJECTS }} --configuration ${{ env.BUILDCONFIGURATION }} +``` + +
+ +_Note_: You can refer to the previous [lab](./4-dry-run.md) to learn about the fundamentals of the `dry-run` command. + +## Custom transformers for build steps + +You can use custom transformers to override Valet's default behavior. In this scenario, you may want to override the behavior for converting `DotnetCoreCLI@2` tasks to support parameters that are glob patterns. Answer the following questions before writing a custom transformer: + +1. What is the "identifier" of the step to customize? + - __DotnetCoreCLI@2__ + +2. What is the desired Actions syntax to use instead? + - After some research, you have determined that the uploading test results as an artifact will be suitable: + + ```yaml + - run: shopt -s globstar; for f in ./**/*.csproj; do dotnet build $f --configuration ${{ env.BUILDCONFIGURATION }} ; done + shell: bash + ``` + +Now you can begin to write the custom transformer. Customer transformers use a DSL built on top of Ruby and should be defined in a file with the `.rb` file extension. You can create this file by running the following command in your codespace terminal: + +```bash +code transformers.rb +``` + +Next, you will define a `transform` method for the `DotnetCoreCLI@2` identifier by adding the following code to `transformers.rb`: + +```ruby +transform "DotNetCoreCLI@2" do |item| + projects = item["projects"] + command = item["command"] + run_command = [] + + if projects.include?("$") + command = "build" if command.nil? + run_command << "shopt -s globstar; for f in ./**/*.csproj; do dotnet #{command} $f #{item['arguments']} ; done" + else + run_command << "dotnet #{command} #{item['projects']} #{item['arguments']}" + end + + { + run: run_command.join("\n"), + shell: "bash", + } +end +``` + +This method can use any valid ruby syntax and should return a `Hash` that represents the YAML that should be generated for a given step. Valet will use this method to convert a step with the provided identifier and will use the `item` parameter for the original values configured in Azure DevOps. + +Now you can perform another `dry-run` command and use the `--custom-transformers` CLI option to provide this custom transformer. Run the following command within your codespace terminal: + +```bash +gh valet dry-run azure-devops pipeline --pipeline-id :pipeline_id --output-dir tmp/dry-run --custom-transformers transformers.rb +``` + +Open the workflow that is generated and inspect the contents. Now the `DotnetCoreCLI@2` steps are converted using the customized behavior! + +```diff +- - name: Restore +- run: dotnet restore ${{ env.BuildParameters_RESTOREBUILDPROJECTS }} +- - name: Build +- run: dotnet build ${{ env.BuildParameters_RESTOREBUILDPROJECTS }} --configuration ${{ env.BUILDCONFIGURATION }} ++ - name: Restore ++ run: shopt -s globstar; for f in ./**/*.csproj; do dotnet restore $f ; done ++ shell: bash ++ - name: Build ++ run: shopt -s globstar; for f in ./**/*.csproj; do dotnet build $f --configuration ${{ env.BUILDCONFIGURATION }} ; done ++ shell: bash +``` + +## Custom transformers for environment variables + +You can also use custom transformers to edit the values of environment variables in converted workflows. In this example, you will be updating the `BUILDCONFIGURATION` environment variable to be `Debug` instead of `Release`. + +To do this, add the following code at the top of the `transformers.rb` file. + +```ruby +env "BUILDCONFIGURATION", "Debug" +``` + +In this example, the first parameter to the `env` method is the environment variable name and the second is the updated value. + +Now you can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow, the `DB_ENGINE` environment variable will be set to `mongodb`: + +```diff +env: +- BUILDCONFIGURATION: Release ++ BUILDCONFIGURATION: Debug + BuildParameters_RESTOREBUILDPROJECTS: "**/*.csproj" +``` + +## Custom transformers for runners + +Finally, you can use custom transformers to dictate which runners converted workflows should use. First, answer the following questions: + +1. What is the label of the runner in Azure DevOps to update? + - __mechamachine__ + +2. What is the label of the runner in Actions to use instead? + - __ubuntu-latest__ + +With these questions answered, you can add the following code to the `transformers.rb` file: + +```ruby +runner "mechamachine", "ubuntu-latest" +``` + +In this example, the first parameter to the `runner` method is the Azure DevOps label and the second is the Actions runner label. + +Now you can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow, the `runs-on` statement will use the customized runner label: + +```diff +- runs-on: +- - self-hosted +- - mechamachine ++ runs-on: ubuntu-latest +``` + +At this point, the file contents of `transformers.rb` should match this: + +
+ Custom transformers 👇 + +```ruby +transform "DotNetCoreCLI@2" do |item| + projects = item["projects"] + command = item["command"] + run_command = [] + + if projects.include?("$") + command = "build" if command.nil? + run_command << "shopt -s globstar; for f in ./**/*.csproj; do dotnet #{command} $f #{item['arguments']} ; done" + else + run_command << "dotnet #{command} #{item['projects']} #{item['arguments']}" + end + + { + shell: "bash", + run: run_command.join("\n") + } +end + +env "BUILDCONFIGURATION", "Debug" + +runner "mechamachine", "ubuntu-latest" +``` + +
+ +That's it! At this point you have overridden Valet's default behavior by customizing the conversion of: + +- Build steps +- Environment variables +- Runners + +## Next lab + +[Perform a production migration of an Azure DevOps pipeline](6-migrate.md) diff --git a/azure_devops/6-migrate.md b/azure_devops/6-migrate.md new file mode 100644 index 0000000..551f5af --- /dev/null +++ b/azure_devops/6-migrate.md @@ -0,0 +1,60 @@ +# Perform a production migration of an Azure DevOps pipeline + +In this lab, you will use the `migrate` command to convert an Azure DevOps pipeline and open a pull request with the equivalent Actions workflow. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and bootstrap an Azure DevOps project. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [dry-run lab](./4-dry-run.md). +4. Completed the [custom transformers lab](./5-custom-transformers.md). + +## Performing a migration + +Answer the following questions before running a `migrate` command: + +1. What is the id of the pipeline to convert? + - __:pipeline_id__. This id can be found by: + - Navigating to the build pipelines in the bootstrapped Azure DevOps project + - Selecting the pipeline with the name "valet-pipeline2" + - Inspecting the URL to locate the pipeline id +2. Where do you want to store the logs? + - __tmp/migrate__ +3. What is the URL for the GitHub repository to add the workflow to? + - __this repository__. The URL should follow the pattern with `:owner` and `:repo` replaced with your values. + +### Steps + +1. Run the following `migrate` command in the codespace terminal: + + ```bash + gh valet migrate azure-devops pipeline --pipeline-id :pipeline_id --target-url https://github.com/:owner/:repo --output-dir tmp/migrate + ``` + +2. The command will write the URL to the pull request that was created when the command succeeds. + + ```console + $ gh valet migrate azure-devops pipeline --pipeline-id 8 --target-url https://github.com/ethanis/labs --output-dir tmp/migrate + [2022-09-07 20:25:08] Logs: 'tmp/dry-run/log/valet-20220907-202508.log' + [2022-09-07 20:25:13] Pull request: 'https://github.com/ethanis/labs/pull/42' + ``` + +3. Open the generated pull request in a new browser tab. + +### Inspect the pull request + +The first thing to notice about the pull request is that there is a list of manual steps to complete: + +![img](https://user-images.githubusercontent.com/8703324/189002125-45561312-dd26-42fd-bd38-a596614ee871.png) + +Next, you can inspect the `Files changed` in this pull request to see the converted workflow that is being added. Any additional changes or code reviews that were needed should be done in this pull request. + +Finally, you can merge the pull request once your review has completed. You can then view the workflow running by selecting the `Actions` menu in the top navigation bar in GitHub. + +![img](https://user-images.githubusercontent.com/19557880/185509704-90243ec5-e77f-4baf-a9b2-d9a4d9fda199.png) + +At this point, the migration has completed and you have successfully migrated an Azure DevOps pipeline to Actions! + +### Next lab + +This concludes all labs for migrating Azure DevOps pipelines to Actions with Valet! diff --git a/azure_devops/bootstrap/azDevOps.ps1 b/azure_devops/bootstrap/azDevOps.ps1 deleted file mode 100644 index 8e36704..0000000 --- a/azure_devops/bootstrap/azDevOps.ps1 +++ /dev/null @@ -1,235 +0,0 @@ -param ( - $project = "ValetBootstrap", - $repoName = "ValetBootstrap", - $codeFolder= "code,azure_devops/pipelines", - $rootFolder= ".", - $pipelinesRoot="azure_devops/pipelines", - [String]$orgToUse = "..", - [String]$userName=$(Throw "Azure DevOps Username required."), - [String]$daPassword=$(Throw "Password required.") - ) - -Write-Host "Start Valet Bootstrap" -Write-Host "Azure DevOps Organization to use $orgToUse" -Write-Host "Azure DevOps Project to use $project" -Write-Host "Repo Name $repoName" -Write-Host "Code Folders: $codeFolder" -Write-Host "Root Folders: $rootFolder" -Write-Host "Pipelines Folders: $pipelinesRoot" -Write-Host "Azure DevOps username to authenticate with $userName" -Write-Host "Azure DevOps PAT to authenticate with $daPassword" - -function Get-BasicAuthCreds { - param([string]$Username,[String]$Password) - $AuthString = "{0}:{1}" -f $Username,$Password - $AuthBytes = [System.Text.Encoding]::Ascii.GetBytes($AuthString) - return [Convert]::ToBase64String($AuthBytes) -} - -#Get the creds to use in all GitHub API calls. -$BasicCreds = Get-BasicAuthCreds -Username "$userName" -Password $daPassword - -########################################################################## -#CREATE THE AZURE DEVOPS PROJECT -$createtUrlToUse = [string]::Format("https://dev.azure.com/{0}/_apis/projects?api-version=6.0", $orgToUse) - -$projectToCreate = @{ - "name"= "$project" - "description"= "Project to be used for Valet Demos" - "capabilities"= @{ - "versioncontrol"= @{ - "sourceControlType"= "Git" - } - "processTemplate"= @{ - "templateTypeId"= "6b724908-ef14-45cf-84f8-768b5384da45" - } - } - } - - try{ - $projectResponse = Invoke-RestMethod -Method 'Post' -Uri $createtUrlToUse -Headers @{"Authorization"="Basic $BasicCreds"} -Body ($projectToCreate|ConvertTo-Json) -ContentType "application/json" - $projectResponse | ConvertTo-Json - - Start-Sleep -s 5 - Write-Host "Yes! Project $project was created!" - - } Catch { - Write-Host " " - Write-Host "##########################################################################" - Write-Host "ERROR during project creation: $project" - Write-Host "The API call was: $createtUrlToUse" - Write-Host " " - - if($_.ErrorDetails.Message) { - Write-Host "See error message below:" - Write-Host $_.ErrorDetails.Message - } else { - Write-Host $_ - Write-Host "It is likely that the AZURE_DEVOPS_ACCESS_TOKEN is not set correctly in the worflow OR does not have sufficent permissions" - } - Write-Host " " - Write-Host "##########################################################################" - throw "Exiting. The Project $project was NOT created in Azure DevOps" - } - -########################################################################## -# IMPORT THE REPO -Set-Location $rootFolder - -[array]$folderArray = $codeFolder.Split(",") - -$addTextFile = @{ - "refUpdates"= @( - @{ - "name"= "refs/heads/main" - "oldObjectId"= "0000000000000000000000000000000000000000" - } - ) - "commits"= @( - @{ - "comment"= "Initial commit." - "changes"= @( - foreach ($folder in $folderArray) - { - $codeRootFolder = Join-Path -Path $rootFolder -ChildPath $folder - Write-Host "codeRootFolder: $codeRootFolder" - - $itemPath = $folder.Substring($folder.lastIndexOf('/') + 1) - Write-Host "itemPath: $itemPath" - - $codeFiles = (Get-ChildItem "$codeRootFolder" -Recurse -Attributes !Directory) - foreach ($codeFile in $codeFiles ) { - $daFullName = $codeFile.FullName - $codeBody = (Get-Content "$daFullName" -Raw) - #Write-Host "codeBody: $codeBody" - #Write-Host "folder: $folder" - $relath = $daFullName | Resolve-Path -Relative - #Write-Host "relath: $relath" - $replaceSlash = $folder.Replace("/", "\") - $repoPath = $relath.Substring($relath.lastIndexOf($replaceSlash) + $replaceSlash.Length) - $wholeRepoPath = Join-Path -Path $itemPath -ChildPath $repoPath - $wholeRepoPath = $wholeRepoPath.Replace("\", "/") - #Write-Host "wholeRepoPath: $wholeRepoPath" - #Write-Host "repoPath: $repoPath" - @{ - "changeType"= "add" - "item"= @{ - "path"= "/$wholeRepoPath" - } - "newContent"= @{ - "content"= "$codeBody" - "contentType"= "rawtext" - } - } - } - } - ) - } - ) - } - -try{ - $repoUrlToUse = [string]::Format("https://dev.azure.com/{0}/{2}/_apis/git/repositories/{1}/pushes?api-version=7.1-preview.2", $orgToUse, $repoName, $project) - $pushResponse = Invoke-RestMethod -Method 'Post' -Uri $repoUrlToUse -Headers @{"Authorization"="Basic $BasicCreds"} -Body ($addTextFile|ConvertTo-Json -Depth 5) -ContentType "application/json" - $repoIds = $pushResponse.repository.id - Write-Host "Yes! Repo Was was created! The repoId is $repoIds" -} Catch { - Write-Host " " - Write-Host "##########################################################################" - Write-Host "ERROR during Repo creation:" - Write-Host "The API call was: $repoUrlToUse" - Write-Host " " - - if($_.ErrorDetails.Message) { - Write-Host "See error message below:" - Write-Host $_.ErrorDetails.Message - } else { - Write-Host $_ - } - Write-Host " " - Write-Host "##########################################################################" - throw "Exiting. The Repo was NOT created. It is likely the Project was created in Azure DevOps." -} - -########################################################################## -########################################################################## -# Create YML Pipelines -$plUrlToUse = [string]::Format("https://dev.azure.com/{0}/{1}/_apis/pipelines?api-version=6.0-preview.1", $orgToUse, "$project") - -$files = Get-ChildItem ".\$pipelinesRoot\yml\" -foreach ($f in $files) -{ - $nameOfPl = $f.BaseName - $fileName = $f.Name - Write-Host "basename: $nameOfPl filename: $fileName" - - $pipelineCreateBody = @{ - "folder"= "pipelines" - "name"= "$nameOfPl" - "configuration"= @{ - "type"= "yaml" - "path"= "/pipelines/yml/$fileName" - "repository"= @{ - "id"= "$repoIds" - "name"= "$repoName" - "type"= "azureReposGit" - } - } - } - - try{ - $repoResponse = Invoke-RestMethod -Method 'Post' -Uri $plUrlToUse -Headers @{"Authorization"="Basic $BasicCreds"} -Body ($pipelineCreateBody|ConvertTo-Json) -ContentType "application/json" - } Catch { - Write-Host " " - Write-Host "##########################################################################" - Write-Host "ERROR during Pipeline creation:" - Write-Host "The API call was: $plUrlToUse" - Write-Host " " - - if($_.ErrorDetails.Message) { - Write-Host "See error message below:" - Write-Host $_.ErrorDetails.Message - } else { - Write-Host $_ - } - Write-Host " " - Write-Host "The Pipeline $nameOfPl was NOT created." - Write-Host "##########################################################################" - } -} - -########################################################################## -#CREATE CLASSIC PIPELINEs - -$classicUrlToUse = [string]::Format("https://dev.azure.com/{0}/{1}/_apis/build/definitions?api-version=7.1-preview.7", $orgToUse, "$project") - -$files = Get-ChildItem ".\$pipelinesRoot\classic\" -foreach ($f in $files) -{ - $nameOfPl = $f.BaseName - $fileName = $f.Name - Write-Host "basename: $nameOfPl filename: $fileName" - - $pclassicreateBody = (Get-Content ./$pipelinesRoot/classic/$fileName).Replace("REPOTOREPLACE", "$repoName") - - try{ - Invoke-RestMethod -Method 'Post' -Uri $classicUrlToUse -Headers @{"Authorization"="Basic $BasicCreds"} -Body ($pclassicreateBody) -ContentType "application/json" - } Catch { - Write-Host " " - Write-Host "##########################################################################" - Write-Host "ERROR during Pipeline creation:" - Write-Host "The API call was: $classicUrlToUse" - Write-Host " " - - if($_.ErrorDetails.Message) { - Write-Host "See error message below:" - Write-Host $_.ErrorDetails.Message - } else { - Write-Host $_ - } - Write-Host " " - Write-Host "The Pipeline $nameOfPl was NOT created." - Write-Host "##########################################################################" - } -} -Write-Host "End Valet Bootstrap" diff --git a/azure_devops/bootstrap/azure_devops.tgz b/azure_devops/bootstrap/azure_devops.tgz new file mode 100644 index 0000000..cdc27b2 Binary files /dev/null and b/azure_devops/bootstrap/azure_devops.tgz differ diff --git a/azure_devops/bootstrap/setup b/azure_devops/bootstrap/setup new file mode 100755 index 0000000..7af011a --- /dev/null +++ b/azure_devops/bootstrap/setup @@ -0,0 +1,195 @@ +#!/usr/bin/env ruby +require "date" +require "json" +require "net/http" +require "open3" +require "optparse" +require "pathname" +require "tmpdir" + +options = { + assets_file: "azure_devops.tgz", + pipeline_assets_dir: "pipelines", + root_dir: File.join(ENV["CODESPACE_VSCODE_FOLDER"], "azure_devops/bootstrap"), + forecast_source_file: "jobs.json" +} + +OptionParser.new do |opt| + opt.on('--organization ORGANIZATION') { |o| options[:organization] = o } + opt.on('--project PROJECT') { |o| options[:project] = o } + opt.on('--access-token ACCESS_TOKEN') { |o| options[:access_token] = o } +end.parse! + +[:organization, :project, :access_token].each do |key| + raise OptionParser::MissingArgument, key if options[key].nil? +end + +def post_request(uri, body, access_token) + Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http| + request = Net::HTTP::Post.new(uri) + request.basic_auth "", access_token + request.body = body.to_json + request.content_type = 'application/json' + + response = http.request request + + yield(response) if block_given? + + sleep(5) + + JSON.parse(response.body) + end +end + +def create_project(options) + organization = options[:organization] + project = options[:project] + access_token = options[:access_token] + + puts "Creating project #{project} in organization #{organization}..." + uri = URI("https://dev.azure.com/#{organization}/_apis/projects?api-version=6.0") + + project_to_create = { + name: project, + description: "Project to be used for Valet Labs", + capabilities: { + versioncontrol: { + sourceControlType: "Git" + }, + processTemplate: { + templateTypeId: "6b724908-ef14-45cf-84f8-768b5384da45" + } + } + } + + post_request(uri, project_to_create, access_token) do |response| + raise "Error creating project (#{response.code}:#{response.message})" unless response.code.start_with?("20") + end +end + +def create_repository(options) + organization = options[:organization] + project = options[:project] + # use the default repository created when the project is created + repository = options[:project] + access_token = options[:access_token] + root_dir = options[:root_dir] + assets_file = options[:assets_file] + + tmp_dir = Dir.mktmpdir + puts "Extracting assets..." + _, _, st = Open3.capture3("tar", "-xzf", File.join(root_dir, assets_file), "-C", tmp_dir) + abort unless st.exitstatus == 0 + + uri = URI("https://dev.azure.com/#{organization}/#{project}/_apis/git/repositories/#{repository}/pushes?api-version=7.1-preview.2") + puts "Creating repository #{repository}..." + + root = Pathname.new(root_dir) + repository_to_create = { + refUpdates: [{ + name: "refs/heads/main", + oldObjectId: "0000000000000000000000000000000000000000" + }], + commits: [{ + comment: "Initial commit.", + changes: Dir[File.join(tmp_dir, "**/*")].reject {|f| File.directory?(f) }.map do |entry| + { + changeType: "add", + item: { + path: File.join("/", Pathname.new(entry).relative_path_from(tmp_dir).to_s) + }, + newContent: { + content: File.read(entry), + contentType: "rawText" + } + } + end.flatten + }] + } + + result = post_request(uri, repository_to_create, access_token) do |response| + raise "Error creating repository (#{response.code}:#{response.message})" unless response.code.start_with?("20") + end + + [result, tmp_dir] +end + +def create_yaml_pipelines(options, repository, tmp_dir) + organization = options[:organization] + project = options[:project] + repository_name = repository.dig("repository", "name") + repository_id = repository.dig("repository", "id") + access_token = options[:access_token] + pipeline_assets_dir = options[:pipeline_assets_dir] + + puts "Creating yaml pipelines..." + + uri = URI("https://dev.azure.com/#{organization}/#{project}/_apis/pipelines?api-version=6.0-preview.1") + root = Pathname.new(tmp_dir) + Dir[File.join(tmp_dir, pipeline_assets_dir, "yml", "*")].each do |entry| + puts "Creating pipeline #{entry}..." + + pipeline_to_create = { + folder: "pipelines", + name: File.basename(entry, ".yml"), + configuration: { + type: "yaml", + path: File.join("/", Pathname.new(entry).relative_path_from(root).to_s), + repository: { + id: repository_id, + name: repository_name, + type: "azureReposGit" + } + } + } + + post_request(uri, pipeline_to_create, access_token) do |response| + raise "Error creating pipeline (#{response.code}:#{response.message})" unless response.code.start_with?("20") + end + end +end + +def create_classic_pipelines(options, repository) + organization = options[:organization] + project = options[:project] + repository_name = repository.dig("repository", "name") + repository_id = repository.dig("repository", "id") + access_token = options[:access_token] + root_dir = options[:root_dir] + pipeline_assets_dir = options[:pipeline_assets_dir] + + puts "Creating classic pipelines..." + + uri = URI("https://dev.azure.com/#{organization}/#{project}/_apis/build/definitions?api-version=7.1-preview.7") + root = Pathname.new(root_dir) + Dir[File.join(root_dir, pipeline_assets_dir, "classic", "*")].each do |entry| + puts "Creating pipeline #{entry}..." + + pipeline_to_create = JSON.parse(File.read(entry)) + + pipeline_to_create["repository"]["name"] = repository_name + pipeline_to_create["repository"]["id"] = repository_id + + post_request(uri, pipeline_to_create, access_token) do |response| + raise "Error creating pipeline (#{response.code}:#{response.message})" unless response.code.start_with?("20") + end + end +end + +def update_forecast_source_file(options, tmp_dir) + puts "Updating forecast data" + root_dir = options[:root_dir] + jobs_data_file = options[:forecast_source_file] + jobs_data = File.read(File.join(tmp_dir, jobs_data_file)) + today = Date.today.strftime("%Y-%m-%d") + jobs_data.gsub!(/20[0-2][0-9]-[0-1][0-9]-[0-3][0-9]/, today) + File.write(File.join(root_dir, jobs_data_file), jobs_data) +end + +create_project(options) +repository, tmp_dir = create_repository(options) +create_yaml_pipelines(options, repository, tmp_dir) +create_classic_pipelines(options, repository) +update_forecast_source_file(options, tmp_dir) + +puts "Success!" diff --git a/azure_devops/pipelines/classic/valet-classic1.json b/azure_devops/pipelines/classic/valet-classic1.json deleted file mode 100644 index 1ade862..0000000 --- a/azure_devops/pipelines/classic/valet-classic1.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "options": [ - { - "enabled": true, - "definition": { - "id": "5d58cc01-7c75-450c-be18-a388ddb129ec" - }, - "inputs": { - "branchFilters": "[\"+refs/heads/*\"]", - "additionalFields": "{}" - } - }, - { - "enabled": false, - "definition": { - "id": "a9db38f9-9fdc-478c-b0f9-464221e58316" - }, - "inputs": { - "workItemType": "Task", - "assignToRequestor": "true", - "additionalFields": "{}" - } - } - ], - "variables": { - "BuildConfiguration": { - "value": "release", - "allowOverride": true - }, - "BuildPlatform": { - "value": "any cpu", - "allowOverride": true - }, - "system.debug": { - "value": "false", - "allowOverride": true - } - }, - "properties": {}, - "tags": [], - "_links": [], - "buildNumberFormat": "$(date:yyyyMMdd)$(rev:.r)", - "jobAuthorizationScope": 1, - "jobTimeoutInMinutes": 60, - "jobCancelTimeoutInMinutes": 5, - "process": { - "phases": [ - { - "steps": [ - { - "environment": {}, - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "Use NuGet 4.4.1", - "timeoutInMinutes": 0, - "retryCountOnTaskFailure": 0, - "condition": "succeeded()", - "task": { - "id": "2c65196a-54fd-4a02-9be8-d9d1837b7c5d", - "versionSpec": "0.*", - "definitionType": "task" - }, - "inputs": { - "versionSpec": "4.4.1", - "checkLatest": "false" - } - }, - { - "environment": {}, - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "NuGet restore", - "timeoutInMinutes": 0, - "retryCountOnTaskFailure": 0, - "condition": "succeeded()", - "task": { - "id": "333b11bd-d341-40d9-afcf-b32d5ce6f23b", - "versionSpec": "2.*", - "definitionType": "task" - }, - "inputs": { - "command": "restore", - "solution": "$(Parameters.solution)", - "selectOrConfig": "select", - "feedRestore": "", - "includeNuGetOrg": "true", - "nugetConfigPath": "", - "externalEndpoints": "", - "noCache": "false", - "disableParallelProcessing": "false", - "packagesDirectory": "", - "verbosityRestore": "Detailed", - "searchPatternPush": "$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg", - "nuGetFeedType": "internal", - "feedPublish": "", - "publishPackageMetadata": "true", - "allowPackageConflicts": "false", - "externalEndpoint": "", - "verbosityPush": "Detailed", - "searchPatternPack": "**/*.csproj", - "configurationToPack": "$(BuildConfiguration)", - "outputDir": "$(Build.ArtifactStagingDirectory)", - "versioningScheme": "off", - "includeReferencedProjects": "false", - "versionEnvVar": "", - "requestedMajorVersion": "1", - "requestedMinorVersion": "0", - "requestedPatchVersion": "0", - "packTimezone": "utc", - "includeSymbols": "false", - "toolPackage": "false", - "buildProperties": "", - "basePath": "", - "verbosityPack": "Detailed", - "arguments": "" - } - }, - { - "environment": {}, - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "PowerShell Script", - "timeoutInMinutes": 0, - "retryCountOnTaskFailure": 0, - "condition": "succeeded()", - "task": { - "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", - "versionSpec": "2.*", - "definitionType": "task" - }, - "inputs": { - "targetType": "inline", - "filePath": "", - "arguments": "", - "script": "# Write your PowerShell commands here.\n\nWrite-Host \"Hello World\"\n", - "errorActionPreference": "stop", - "warningPreference": "default", - "informationPreference": "default", - "verbosePreference": "default", - "debugPreference": "default", - "failOnStderr": "false", - "showWarnings": "false", - "ignoreLASTEXITCODE": "false", - "pwsh": "false", - "workingDirectory": "", - "runScriptInSeparateScope": "false" - } - } - ], - "name": "Agent job 1", - "refName": "Job_1", - "condition": "succeeded()", - "target": { - "executionOptions": { - "type": 0 - }, - "allowScriptsAuthAccessOption": false, - "type": 1 - }, - "jobAuthorizationScope": 1 - } - ], - "target": { - "agentSpecification": { - "identifier": "windows-latest" - } - }, - "type": 1 - }, - "repository": { - "properties": { - "cleanOptions": "0", - "labelSources": "0", - "labelSourcesFormat": "$(build.buildNumber)", - "reportBuildStatus": "true", - "gitLfsSupport": "false", - "skipSyncSource": "false", - "checkoutNestedSubmodules": "false", - "fetchDepth": "0" - }, - "id": "", - "type": "TfsGit", - "name": "REPOTOREPLACE", - "url": "", - "defaultBranch": "refs/heads/main", - "clean": "false", - "checkoutSubmodules": false - }, - "processParameters": { - "inputs": [ - { - "aliases": [], - "options": {}, - "properties": {}, - "name": "solution", - "label": "Solution", - "defaultValue": "**\\*.sln", - "required": true, - "type": "filePath", - "helpMarkDown": "The path to the Visual Studio solution file or NuGet packages.config. Wildcards can be used. For example, `**\\\\*.sln` for all sln files in all sub folders." - } - ] - }, - "quality": 1, - "authoredBy": { - "displayName": "David Kalmin", - "url": "https://spsprodcus5.vssps.visualstudio.com/A8ceff4ff-6429-4de3-9c39-fd7f0b1fb05a/_apis/Identities/a886d3a4-5700-6920-a3e4-6dadf46e7614", - "_links": { - "avatar": { - "href": "https://dev.azure.com/microsoft-bootcamp/_apis/GraphProfile/MemberAvatars/aad.YTg4NmQzYTQtNTcwMC03OTIwLWEzZTQtNmRhZGY0NmU3NjE0" - } - }, - "id": "a886d3a4-5700-6920-a3e4-6dadf46e7614", - "uniqueName": "davidkalmin@microsoft.com", - "imageUrl": "https://dev.azure.com/microsoft-bootcamp/_apis/GraphProfile/MemberAvatars/aad.YTg4NmQzYTQtNTcwMC03OTIwLWEzZTQtNmRhZGY0NmU3NjE0", - "descriptor": "aad.YTg4NmQzYTQtNTcwMC03OTIwLWEzZTQtNmRhZGY0NmU3NjE0" - }, - "drafts": [], - "queue": { - "_links": { - "self": { - "href": "https://dev.azure.com/microsoft-bootcamp/_apis/build/Queues/126" - } - }, - "name": "Azure Pipelines", - "url": "https://dev.azure.com/microsoft-bootcamp/_apis/build/Queues/126", - "pool": { - "name": "Azure Pipelines", - "isHosted": true - } - }, - "id": 10, - "name": "valet-classic-test-import1", - "uri": "vstfs:///Build/Definition/10", - "path": "\\", - "type": 2, - "queueStatus": 0, - "revision": 1, - "createdDate": "2022-02-17T19:05:52.500Z", -} diff --git a/azure_devops/pipelines/classic/valet-classic2.json b/azure_devops/pipelines/classic/valet-classic2.json deleted file mode 100644 index fa966c1..0000000 --- a/azure_devops/pipelines/classic/valet-classic2.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "options": [ - { - "enabled": true, - "definition": { - "id": "5d58cc01-7c75-450c-be18-a388ddb129ec" - }, - "inputs": { - "branchFilters": "[\"+refs/heads/*\"]", - "additionalFields": "{}" - } - }, - { - "enabled": false, - "definition": { - "id": "a9db38f9-9fdc-478c-b0f9-464221e58316" - }, - "inputs": { - "workItemType": "Task", - "assignToRequestor": "true", - "additionalFields": "{}" - } - } - ], - "variables": { - "BuildConfiguration": { - "value": "release", - "allowOverride": true - }, - "BuildPlatform": { - "value": "any cpu", - "allowOverride": true - }, - "system.debug": { - "value": "false", - "allowOverride": true - } - }, - "properties": {}, - "tags": [], - "_links": [], - "buildNumberFormat": "$(date:yyyyMMdd)$(rev:.r)", - "jobAuthorizationScope": 1, - "jobTimeoutInMinutes": 60, - "jobCancelTimeoutInMinutes": 5, - "process": { - "phases": [ - { - "steps": [ - { - "environment": {}, - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "Use NuGet 4.4.1", - "timeoutInMinutes": 0, - "retryCountOnTaskFailure": 0, - "condition": "succeeded()", - "task": { - "id": "2c65196a-54fd-4a02-9be8-d9d1837b7c5d", - "versionSpec": "0.*", - "definitionType": "task" - }, - "inputs": { - "versionSpec": "4.4.1", - "checkLatest": "false" - } - }, - { - "environment": {}, - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "NuGet restore", - "timeoutInMinutes": 0, - "retryCountOnTaskFailure": 0, - "condition": "succeeded()", - "task": { - "id": "333b11bd-d341-40d9-afcf-b32d5ce6f23b", - "versionSpec": "2.*", - "definitionType": "task" - }, - "inputs": { - "command": "restore", - "solution": "$(Parameters.solution)", - "selectOrConfig": "select", - "feedRestore": "", - "includeNuGetOrg": "true", - "nugetConfigPath": "", - "externalEndpoints": "", - "noCache": "false", - "disableParallelProcessing": "false", - "packagesDirectory": "", - "verbosityRestore": "Detailed", - "searchPatternPush": "$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg", - "nuGetFeedType": "internal", - "feedPublish": "", - "publishPackageMetadata": "true", - "allowPackageConflicts": "false", - "externalEndpoint": "", - "verbosityPush": "Detailed", - "searchPatternPack": "**/*.csproj", - "configurationToPack": "$(BuildConfiguration)", - "outputDir": "$(Build.ArtifactStagingDirectory)", - "versioningScheme": "off", - "includeReferencedProjects": "false", - "versionEnvVar": "", - "requestedMajorVersion": "1", - "requestedMinorVersion": "0", - "requestedPatchVersion": "0", - "packTimezone": "utc", - "includeSymbols": "false", - "toolPackage": "false", - "buildProperties": "", - "basePath": "", - "verbosityPack": "Detailed", - "arguments": "" - } - }, - { - "environment": {}, - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "PowerShell Script", - "timeoutInMinutes": 0, - "retryCountOnTaskFailure": 0, - "condition": "succeeded()", - "task": { - "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", - "versionSpec": "2.*", - "definitionType": "task" - }, - "inputs": { - "targetType": "inline", - "filePath": "", - "arguments": "", - "script": "# Write your PowerShell commands here.\n\nWrite-Host \"Hello World\"\n", - "errorActionPreference": "stop", - "warningPreference": "default", - "informationPreference": "default", - "verbosePreference": "default", - "debugPreference": "default", - "failOnStderr": "false", - "showWarnings": "false", - "ignoreLASTEXITCODE": "false", - "pwsh": "false", - "workingDirectory": "", - "runScriptInSeparateScope": "false" - } - } - ], - "name": "Agent job 1", - "refName": "Job_1", - "condition": "succeeded()", - "target": { - "executionOptions": { - "type": 0 - }, - "allowScriptsAuthAccessOption": false, - "type": 1 - }, - "jobAuthorizationScope": 1 - } - ], - "target": { - "agentSpecification": { - "identifier": "windows-latest" - } - }, - "type": 1 - }, - "repository": { - "properties": { - "cleanOptions": "0", - "labelSources": "0", - "labelSourcesFormat": "$(build.buildNumber)", - "reportBuildStatus": "true", - "gitLfsSupport": "false", - "skipSyncSource": "false", - "checkoutNestedSubmodules": "false", - "fetchDepth": "0" - }, - "id": "", - "type": "TfsGit", - "name": "REPOTOREPLACE", - "url": "", - "defaultBranch": "refs/heads/main", - "clean": "false", - "checkoutSubmodules": false - }, - "processParameters": { - "inputs": [ - { - "aliases": [], - "options": {}, - "properties": {}, - "name": "solution", - "label": "Solution", - "defaultValue": "**\\*.sln", - "required": true, - "type": "filePath", - "helpMarkDown": "The path to the Visual Studio solution file or NuGet packages.config. Wildcards can be used. For example, `**\\\\*.sln` for all sln files in all sub folders." - } - ] - }, - "quality": 1, - "authoredBy": { - "displayName": "David Kalmin", - "url": "https://spsprodcus5.vssps.visualstudio.com/A8ceff4ff-6429-4de3-9c39-fd7f0b1fb05a/_apis/Identities/a886d3a4-5700-6920-a3e4-6dadf46e7614", - "_links": { - "avatar": { - "href": "https://dev.azure.com/microsoft-bootcamp/_apis/GraphProfile/MemberAvatars/aad.YTg4NmQzYTQtNTcwMC03OTIwLWEzZTQtNmRhZGY0NmU3NjE0" - } - }, - "id": "a886d3a4-5700-6920-a3e4-6dadf46e7614", - "uniqueName": "davidkalmin@microsoft.com", - "imageUrl": "https://dev.azure.com/microsoft-bootcamp/_apis/GraphProfile/MemberAvatars/aad.YTg4NmQzYTQtNTcwMC03OTIwLWEzZTQtNmRhZGY0NmU3NjE0", - "descriptor": "aad.YTg4NmQzYTQtNTcwMC03OTIwLWEzZTQtNmRhZGY0NmU3NjE0" - }, - "drafts": [], - "queue": { - "_links": { - "self": { - "href": "https://dev.azure.com/microsoft-bootcamp/_apis/build/Queues/126" - } - }, - "name": "Azure Pipelines", - "url": "https://dev.azure.com/microsoft-bootcamp/_apis/build/Queues/126", - "pool": { - "name": "Azure Pipelines", - "isHosted": true - } - }, - "id": 10, - "name": "valet-classic-test-import2", - "uri": "vstfs:///Build/Definition/10", - "path": "\\", - "type": 2, - "queueStatus": 0, - "revision": 1, - "createdDate": "2022-02-17T19:05:52.500Z", -} diff --git a/azure_devops/pipelines/readme.md b/azure_devops/pipelines/readme.md deleted file mode 100644 index 40258f9..0000000 --- a/azure_devops/pipelines/readme.md +++ /dev/null @@ -1,10 +0,0 @@ -# pipelines folder -This folder contains Azure DevOps pipelines that gets migrated to the Azure DevOps instance. - -### Note any file added to this folder or sub folders will get migrated to the Azure DevOps instance when the GitHub Action runs to create and bootstrap the Azure DevOps project - -## classic folder -The classic folder contains classic style Azure DevOps pipelines that are pure JSON files. - -## yml folder -The yml folder contains yml Azure DevOps pipelines. diff --git a/azure_devops/pipelines/yml/valet-mapper-example.yml b/azure_devops/pipelines/yml/valet-mapper-example.yml deleted file mode 100644 index 8220c95..0000000 --- a/azure_devops/pipelines/yml/valet-mapper-example.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -variables: -- name: BuildParameters.RESTOREBUILDPROJECTS - value: "**/*.csproj" -- name: BUILDCONFIGURATION - value: Release -name: "$(date:yyyyMMdd)$(rev:.r)" -jobs: -- job: Job_1 - displayName: Agent job 1 - pool: - vmImage: windows-latest - steps: - - checkout: self - - task: NodeTool@0 - displayName: Use Node 10.16.3 - inputs: - versionSpec: 10.16.3 - - task: DotNetCoreCLI@2 - displayName: Restore - inputs: - command: restore - projects: "$(BuildParameters.RESTOREBUILDPROJECTS)" - - task: DotNetCoreCLI@2 - displayName: Build - inputs: - projects: "$(BuildParameters.RESTOREBUILDPROJECTS)" - arguments: "--configuration $(BUILDCONFIGURATION)" - diff --git a/azure_devops/pipelines/yml/valet-pipeline1.yml b/azure_devops/pipelines/yml/valet-pipeline1.yml deleted file mode 100644 index 9c86c9a..0000000 --- a/azure_devops/pipelines/yml/valet-pipeline1.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Bootcap Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: -- main - -pool: - vmImage: windows-latest - -steps: -- script: echo Hello, I am pipeline 1! - displayName: 'Run a one-line script' - -- script: | - echo Add other tasks to build, test, and deploy your project. - echo See https://aka.ms/yaml - displayName: 'Run a multi-line script' diff --git a/azure_devops/pipelines/yml/valet-pipeline2.yml b/azure_devops/pipelines/yml/valet-pipeline2.yml deleted file mode 100644 index 8b801c9..0000000 --- a/azure_devops/pipelines/yml/valet-pipeline2.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Bootcap Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: -- main - -pool: - vmImage: windows-latest - -steps: -- script: echo Hello, I am pipeline 2! - displayName: 'Run a one-line script' - -- script: | - echo Add other tasks to build, test, and deploy your project. - echo See https://aka.ms/yaml - displayName: 'Run a multi-line script' diff --git a/azure_devops/readme.md b/azure_devops/readme.md index a26d655..ca3805e 100644 --- a/azure_devops/readme.md +++ b/azure_devops/readme.md @@ -1,143 +1,111 @@ -# Valet labs for Azure DevOps +# Azure Pipelines to Actions migrations powered by Valet -This lab bootstraps a Valet environment using GitHub Codespaces and enables you to create an Azure DevOps project against which to run the Valet CI/CD migration tool. +These instructions will guide you through configuring the GitHub Codespaces environment that will be used in these labs to demonstrate how to use Valet to migrate Azure DevOps pipelines to GitHub Actions. -- [Use this Repo as a template](#repo-template) -- [Prerequisites](#prerequisites) -- [Codespace secrets](#codespace-secrets) -- [Action secrets](#action-secrets) -- [Azure DevOps project creation](#azure-devops-project-creation) -- [Use Valet with a codespace](#use-valet-with-a-codespace) -- [Labs for Azure DevOps](#labs-for-azure-devops) +These steps **must** be completed prior to starting other labs. -## Repo template +## Create your own repository for these labs -1. Verify you are in your own Repository created from the landing page [Valet Labs](https://github.com/valet-customers/labs). +- Ensure that you have created a repository using [valet-customers/labs](https://github.com/valet-customers/labs) as a template. -## Prerequisites +## Configure your codespace -1. Azure DevOps organization. Please identify or create an Azure DevOps organization to use: [Click to create an Azure DevOps Org](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/create-organization?toc=%2Fazure%2Fdevops%2Fget-started%2Ftoc.json&bc=%2Fazure%2Fdevops%2Fget-started%2Fbreadcrumb%2Ftoc.json&view=azure-devops) - - Note and store the Azure DevOps organization name for later. - - Note and store the username you use for your Azure DevOps organization. It will be an email address. -2. Azure DevOps Project. The default project name for this lab is `ValetBootstrap` There is an Action workflow that will create and populate the Azure DevOps project. No need to create it yourself. Note: The project has to be unique in the Azure DevOps organization. If you already have a project name `ValetBootstrap` please pick a different unique project name. - - Note and store the Azure DevOps project name for later. -3. Create an Azure DevOps personal access token with the following scopes: - - To do so, navigate to Sign in to your organization `https://dev.azure.com/{yourorganization}`. - - Click your `Account management` icon - - Click `User settings`. - - Click `Personal access tokens`. - - Select `+ New Token`. - - Name your token, select the organization where you want to use the token, and then set your token to automatically expire after a set number of days. - - Select the following scopes (Click `Show more scopes` to ensure you see all of the following): - - Agents Pool: `Read` - - Build: `Read & Execute` - - Code: `Read & Write` - - Project and Team: `Read, Write, & Manage` - - Release: `Read` - - Service Connections: `Read` - - Task Groups: `Read` - - Variable Groups: `Read` - - Click `Create`. - - Copy the PAT somewhere safe and temporary. -4. Create a GitHub personal access token. - - To do so, navigate to your GitHub `Settings` - click your profile photo and then click `Settings`. - - Go to `Developer settings`. - - Go to `Personal access tokens` -> `Legacy tokens (if present)` - - Click `Generate new token` -> `Legacy tokens (if present)`. If required, provide your password. - - Select at least these scopes: `read packages` and `workflow`. Optionally, provide a text in the **Note** field and change the expiration. - - Click `Generate token`. - - Copy the PAT somewhere safe and temporary. +1. Start a new codespace. -## Codespace secrets - -Please add the following Codespace secrets. - -- `VALET_GHCR_PASSWORD`: Add `VALET_GHCR_PASSWORD` as the `Name` and the GitHub personal access token created above as the `Value`. -- `AZURE_DEVOPS_ACCESS_TOKEN`: Add `AZURE_DEVOPS_ACCESS_TOKEN` as the `Name` and the Azure DevOps personal access token created above as the value. -- `AZURE_DEVOPS_ORGANIZATION`: Add `AZURE_DEVOPS_ORGANIZATION` as the `Name` and the Azure DevOps organization noted above as the `Value`. -- `AZURE_DEVOPS_PROJECT`: Add `AZURE_DEVOPS_PROJECT` as the `Name` and `ValetBootstrap` as the `Value`. - -Steps to create the Codespace secrets. Complete for secret noted above: - -- Navigate to the `Settings` tab in this repo -- Find `Secrets` and click the down arrow -- Click `Codespaces` -- Click `New repository secret` to create a new secret -- Name the secret as noted above -- Paste in the value noted above -- Click `Add secret` - -## Action secrets - -Please add the following Action secrets. - -- `AZURE_DEVOPS_ACCESS_TOKEN`: Add `AZURE_DEVOPS_ACCESS_TOKEN` as the `Name` and the Azure DevOps personal access token created above as the value. - -Steps to create the Actions secret. Complete for secret noted above: - -- Navigate to the `Settings` tab in this repo -- Find `Secrets` and click the down arrow -- Click `Actions` -- Click `New repository secret` to create a new secret -- Name the secret as noted above -- Paste in the value noted above -- Click `Add secret` - -## Azure DevOps project creation - -Run the Actions workflow: - -- CLick the `Actions` tab -- Select the `Valet Bootstrap for Azure DevOps` action -- Click `Run workflow` -- Input the Azure DevOps organization name identified above -- Input the Azure DevOps user name of the user that created the Azure DevOps PAT above. This will be an email address -- Accept the default Azure DevOps project name or change it to one of your preference - - NOTE: The default project name is `ValetBootstrap`. This must be unique in your Azure DevOps organization. If not, please choose a unique name. -- Click `Run workflow` -- Verify the workflow completed successfully. If the workflow did not run successfully you will see a detailed error message. - -### Example ### - -![runaction](https://user-images.githubusercontent.com/26442605/167679930-9bdf6f4f-2e94-4145-aed3-8ee3e8e91d90.png) - -## Use Valet with a codespace - -1. Start the codespace - - Navigate to the `< > Code` tab within your GitHub repository. - - Click the `Code` with button down arrow above repository on the repository's landing page. + - Click the `Code` button on your repository's landing page. - Click the `Codespaces` tab. - - Click `Create codespaces on main` to create the codespace. If you are in another branch then the `main` branch, the codespace will button will have the current branch specified. - - Wait a couple minutes, then verify that the codespace starts up. Once it is fully booted up, the termininal should be present. -2. Verify Valet CLI is installed and working. More information on the [GitHub Valet CLI extension](https://github.com/github/gh-valet) - - Verify Valet CLI is installed and working. - - Run `gh valet version` in the Visual Studio Code terminal and verify the output looks like the below image. Note the valet version will be different than below as the latest version gets pulled down. - - If `gh valet version` did not produce a similar image with a version please follow these instructions [Troubleshoot GH Valet extension](#troubleshoot-gh-valet-extension). - - Start using Valet by following along with the [Labs for Azure DevOps](#labs-for-azure-devops). + - Click `Create codespaces on main` to create the codespace. + - After the codespace has initialized there will be a terminal present. -### Example ### +2. Verify the Valet CLI is installed and working. More information on the Valet extension for the official GitHub CLI can be found [here](https://github.com/github/gh-valet). -![gh-valet-version](https://user-images.githubusercontent.com/26442605/170106559-e69e669f-a1f6-4c2c-8998-3f089b899704.png) + - Run the following command in the codespace terminal: + + ```bash + gh valet version + ``` + + - Verify the output is similar to below. + + ```console + $ gh valet version + gh version 2.14.3 (2022-07-26) + gh valet github/gh-valet v0.1.12 + valet-cli unknown + ``` + + - If `gh valet version` did not produce similar output, please refer to the troubleshooting [guide](#troubleshoot-the-valet-cli). + +## Bootstrap your Azure DevOps organization + +1. Create an Azure DevOps personal access token (PAT): + + - Navigate to your existing organization () in your browser. + - In the top right corner of the screen, click `User settings`. + - Click `Personal access tokens`. + - Select `+ New Token` + - Name your token, select the organization where you want to use the token, and set your token to automatically expire after a set number of days. + - Select the following scopes (you may need to `Show more scopes` to reveal all scopes): + - Agents Pool: `Read` + - Build: `Read & Execute` + - Code: `Read & Write` + - Project and Team: `Read, Write, & Manage` + - Release: `Read` + - Service Connections: `Read` + - Task Groups: `Read` + - Variable Groups: `Read` + - Click `Create`. + - Copy the generated API token and save it in a safe location. + +2. Execute the Azure DevOps setup script that will create a new Azure DevOps project in your organization to be used in the following labs. This script should only be run once. + + - Run the following command from the codespace terminal, replacing the values accordingly: + - `:organization`: the name of your existing Azure DevOps organization + - `:project`: the name of the project to be created in your Azure DevOps organization + - `:access_token`: the PAT created in step 1 above + + ```bash + ./azure_devops/bootstrap/setup --organization :organization --project :project --access-token :access-token + ``` + +3. Open the newly created Azure DevOps project in your browser () + + - Once authenticated, you will see an Azure DevOps project with a few predefined pipelines. ## Labs for Azure DevOps -Perform the following labs to test-drive Valet +Perform the following labs to learn how to migrate Azure DevOps pipelines to GitHub Actions using Valet: -- [Audit Azure DevOps pipelines using the Valet audit command](valet-audit-lab.md) -- [Dry run the migration of an Azure DevOps pipeline to GitHub Actions](valet-dry-run-lab.md) -- [Migrate an Azure DevOps pipeline to GitHub Actions](valet-migrate-lab.md) -- [Migrate an Azure DevOps pipeline to GitHub Actions with a custom transformer](valet-migrate-custom-lab.md) -- [Forecast: Valet forecast lab](valet-forecast-lab.md) +1. [Configure credentials for Valet](1-configure.md) +2. [Perform an audit of an Azure DevOps project](2-audit.md) +3. [Forecast potential build runner usage](3-forecast.md) +4. [Perform a dry-run migration of an Azure DevOps pipeline](4-dry-run.md) +5. [Use custom transformers to customize Valet's behavior](5-custom-transformers.md) +6. [Perform a production migration of a Azure DevOps pipeline](6-migrate.md) -## Troubleshoot GH Valet extension +## Troubleshoot the Valet CLI -Manually Install the GitHub CLI Valet extension. More information on the [GitHub Valet CLI extension](https://github.com/github/gh-valet) +The CLI extension for Valet can be manually installed by following these steps: -- Verify you are in the Visual Studio Code terminal -- Run this command to install the GitHub Valet extension -- `gh extension install github/gh-valet` -- Verify the result of the install is: `✓ Installed extension github/gh-valet` -- If you get a similiar error to the following, click the link to authorize the token - ![linktolcickauth](https://user-images.githubusercontent.com/26442605/169588015-9414404f-82b6-4d0f-89d4-5f0e6941b029.png) - - Restart Codespace after clicking the link -- Verify Valet CLI is installed and working +- Verify you are in the codespace terminal +- Run this command from within the codespace terminal: + + ```bash + gh extension install github/gh-valet + ``` + +- Verify the result of the install contains: + + ```console + $ gh extension install github/gh-valet + ✓ Installed extension github/gh-valet + ``` + +- If you get an error similar to the image below, click the link in the terminal output to authorize the token. + - Restart the codespace after clicking the link. + ![img](https://user-images.githubusercontent.com/26442605/169588015-9414404f-82b6-4d0f-89d4-5f0e6941b029.png) +- Verify Valet CLI extension is installed and working by running the following command from the codespace terminal: + + ```bash + gh valet version + ``` diff --git a/azure_devops/valet-audit-lab.md b/azure_devops/valet-audit-lab.md deleted file mode 100644 index 40aa2dc..0000000 --- a/azure_devops/valet-audit-lab.md +++ /dev/null @@ -1,52 +0,0 @@ -# Audit Azure DevOps pipelines using the Valet audit command -In this lab, you will use Valet to `audit` an Azure DevOps organization. The `audit` command can be used to scan a CI server and output a summary of the current pipelines. This summary can then be used to plan the timeline and effort required to migrate to GitHub Actions. - -- [Prerequisites](#prerequisites) -- [Perform an audit](#perform-an-audit) -- [View audit output](#view-audit-output) -- [Review the pipelines](#review-the-pipelines) -- [Next Lab](#next-lab) - -## Prerequisites - -1. Follow all steps [here](../azure_devops#readme) to set up your environment -2. Create or start a codespace in this repository (if not started) - -## Perform an audit -You will use the codespace preconfigured in this repository to perform the audit. - -1. Navigate to the codespace Visual Studio Code terminal -2. Verify you are in the `valet` directory -3. Now, from the `valet` folder in your repository, run Valet to verify your Azure DevOps configuration: - -``` -gh valet audit azure-devops --output-dir audit -``` -### Example -![valet-audit-1](https://user-images.githubusercontent.com/26442605/169615028-696dad13-ff83-41a7-b475-0ab8c0bbcd65.png) - -4. Valet displays green log files to indicate a successful audit - -### Example -![valet-audit-2](https://user-images.githubusercontent.com/26442605/169615218-a8a3199d-a436-4d70-8c1e-17a61b089eb6.png) - -## View audit output -The audit summary, logs, Azure DevOps yml, and GitHub yml should all be located in the `valet` folder. - -1. Under the `valet` folder find the `audit_summary.md` -2. Right-click the `audit_summary.md` file and select `Open Preview` -3. The file contains details about your current pipelines and what can be migrated 100% automatically vs. what will need some manual intervention or aren't supported by GitHub Actions. -4. Review the file. - -### Example -![valet-audit-3](https://user-images.githubusercontent.com/26442605/169615428-26f7a962-2064-46d0-8206-ea930109b252.png) - -## Review the pipelines -The `audit` command grabs the yml, classic, and release pipelines from Azure DevOps and converts them to GitHub Actions. - -### Example -View the source yml and the proposed GitHub yml -![valet-audit-4](https://user-images.githubusercontent.com/26442605/169615630-8d700081-c631-4b2a-ab1c-e52503f7838f.png) - -### Next Lab -[Dry run the migration of an Azure DevOps pipeline to GitHub Actions](valet-dry-run-lab.md) diff --git a/azure_devops/valet-dry-run-lab.md b/azure_devops/valet-dry-run-lab.md deleted file mode 100644 index 36ae851..0000000 --- a/azure_devops/valet-dry-run-lab.md +++ /dev/null @@ -1,52 +0,0 @@ -# Dry run the migration of an Azure DevOps pipeline to GitHub Actions -In this lab, you will use the `valet dry-run` command to convert an Azure DevOps pipeline to it's equivalent GitHub Actions workflow and write the workflow to your local filesystem. - -- [Prerequisites](#prerequisites) -- [Identify the Azure DevOps pipeline ID to use](#identify-the-azure-devops-pipeline-id-to-use) -- [Perform a dry run](#perform-a-dry-run) -- [View dry-run output](#view-dry-run-output) -- [Next Lab](#next-lab) - -## Prerequisites - -1. Follow all steps [here](../azure_devops#readme) to set up your environment -2. Create or start a codespace in this repository (if not started) -3. You have completed the [Valet audit lab](valet-audit-lab.md). - -## Identify the Azure DevOps pipeline ID to use -You will need a pipeline ID to perform the dry run -1. Go to the `valet/ValetBootstrap/pipelines` folder -2. Open the `valet/ValetBootstrap/pipelines/valet-pipeline1.config.json` file -3. Look for the `$.web.href` link -4. At the end of the link is the pipeline ID. Copy or note the ID. - -### Example -![valet-dr-5](https://user-images.githubusercontent.com/26442605/169616920-7c407adc-0c4e-4104-9046-d1d2ecdd6edf.png) - -## Perform a dry run -You will use the codespace preconfigured in this repository to perform the dry run. - -1. Navigate to the codespace Visual Studio Code terminal -2. Verify you are in the `valet` directory - -``` -gh valet dry-run azure-devops pipeline --pipeline-id PIPELINE-ID --output-dir dry-run -``` -3. Now, from the `valet` folder in your repository, run `gh valet dry-run` to see the output: - -### Example -![valet-dr-1](https://user-images.githubusercontent.com/26442605/169616149-46c2d743-47fe-4061-a48a-7f21442624cb.png) - -4. Valet will create a folder called `dry-runs` under the `valet` folder that shows what will be migrated. - -### Example -![valet-dr-2](https://user-images.githubusercontent.com/26442605/169616265-176a19fd-3071-44fc-bff7-866a172afc57.png) - -## View dry-run output -The dry-run output will show you the GitHub Actions yml that will be migrated to GitHub. - -### Example -![valet-dr-3](https://user-images.githubusercontent.com/26442605/169616486-fd5512fa-0761-45fe-a252-5b2ef0926a04.png) - -### Next Lab -[Migrate an Azure DevOps pipeline to GitHub Actions](valet-migrate-lab.md) diff --git a/azure_devops/valet-forecast-lab.md b/azure_devops/valet-forecast-lab.md deleted file mode 100644 index fd7bb86..0000000 --- a/azure_devops/valet-forecast-lab.md +++ /dev/null @@ -1,3 +0,0 @@ -# Forecast Azure DevOps to GitHub using Valet forecast command - -### Coming soon diff --git a/azure_devops/valet-migrate-custom-lab.md b/azure_devops/valet-migrate-custom-lab.md deleted file mode 100644 index eb7ca67..0000000 --- a/azure_devops/valet-migrate-custom-lab.md +++ /dev/null @@ -1,87 +0,0 @@ -# Migrate an Azure DevOps pipeline to GitHub Actions with a custom transformer -In this lab, you will create a custom plugin that transforms some of the existing migration mapping and replaces it with your own mapping. Then use the `migrate` subcommand to migrate the pipeline. The `migrate` subcommand can be used to convert a pipeline to its GitHub Actions equivalent and then create a pull request with the contents. - -- [Prerequisites](#prerequisites) -- [Identify the Azure DevOps pipeline ID to use](#identify-the-azure-devops-pipeline-id-to-use) -- [Create a custom transformer](#create-a-custom-transformer) -- [Migrate with a custom transformer](#migrate-with-a-custom-transformer) -- [View the pull request](#view-the-pull-request) - -## Prerequisites - -1. Follow all steps [here](../azure_devops#readme) to set up your environment -2. Create or start a codespace in this repository (if not started) -3. Complete the [Valet audit lab](valet-audit-lab.md). -4. Complete the [Valet migrate lab](valet-migrate-lab.md). - -## Identify the Azure DevOps pipeline ID to use -You will need the `valet-mapper-example` Azure DevOps pipeline ID to perform the migration -1. Go to the `valet/ValetBootstrap/pipelines` folder -2. Open the `valet/ValetBootstrap/pipelines/valet-mapper-example.config.json` file -3. Look for the `web - href` link -4. At the end of the link is the pipeline ID. Copy or note the ID. - -### Example -![mapperprops](https://user-images.githubusercontent.com/26442605/175090567-525b97a7-60d2-41b7-9dcd-d559ca1c5bd7.png) - -## Create a custom transformer - -To create a transformer, you need to create a Ruby file that looks as follows: -``` ruby -transform "azuredevopstaskname" do |item| - # your ruby code here that produces output -end -``` - -We start by creating a new folder called `plugin` under the `valet` folder in your repository. In there create a file called `DotNetCoreCLI.rb`. - -Next change the function name to match the Azure DevOps task name `DotNetCoreCLI@2`. -The way you find this name is by clicking the **view yaml** button at a task in the pipeline: - -This results in the following code: -``` ruby -transform "DotNetCoreCLI@2" do |item| - # your ruby code here that produces output -end -``` -The parameter item is a collection of items than contain the properties of the original task that was retrieved from Azure DevOps. -In this case we can see in the yaml that the properties that are set are `command` and `projects`. - -Add the following code to the ruby file: -``` Ruby -transform "DotNetCoreCLI@2" do |item| - projects = item["projects"] - command = item["command"] - run_command = [] - - if projects.include?("$") - command = "build" if command.nil? - run_command << "shopt -s globstar; for f in ./**/*.csproj; do dotnet #{command} $f #{item['arguments']} ; done" - else - run_command << "dotnet #{command} #{item['projects']} #{item['arguments']}" - end - - { - shell: "bash", - run: run_command.join("\n") - } -end -``` - -## Migrate with a custom transformer - -Run the `gh valet migrate` command from the `valet` directory with the transformer again and pass it the custom plugin. Look at the result and see if it results in a successful build. - -``` -gh valet migrate azure-devops pipeline --target-url https://github.com/GITHUB-ORG/GITHUB-REPO --pipeline-id PIPELINE-ID --custom-transformers plugin/DotNetCoreCLI.rb --output-dir migrate -``` - -### Example -![valet-cm-1](https://user-images.githubusercontent.com/26442605/169618556-7c79b34b-6d4c-48d5-98e5-7f8d771117a5.png) - -## View the pull request -The migrate output will show you the pull request on GitHub. Note here that the checks on the pull request all passed! - -### Example -![mapper-ex3](https://user-images.githubusercontent.com/26442605/161117488-93e38847-3034-4f04-a768-e74e16dba4ae.png) - diff --git a/azure_devops/valet-migrate-lab.md b/azure_devops/valet-migrate-lab.md deleted file mode 100644 index 3fa1029..0000000 --- a/azure_devops/valet-migrate-lab.md +++ /dev/null @@ -1,59 +0,0 @@ -# Migrate an Azure DevOps pipeline to GitHub Actions -In this lab, you will use the Valet `migrate` command to migrate an Azure DevOps pipeline to GitHub Actions. The `migrate` subcommand can be used to convert a pipeline to its GitHub Actions equivalent and then create a pull request with the contents. - -- [Prerequisites](#prerequisites) -- [Identify the Azure DevOps pipeline ID to use](#identify-the-azure-devops-pipeline-id-to-use) -- [Perform a migration](#perform-a-migration) -- [View the pull request](#view-the-pull-request) -- [Next Lab](#next-lab) - -## Prerequisites - -1. Follow all steps [here](../azure_devops#readme) to set up your environment -2. Create or start a codespace in this repository (if not started) -3. Completed the [Valet audit lab](valet-audit-lab.md). - -## Identify the Azure DevOps pipeline ID to use -You will need a pipeline ID to perform the migration -1. Go to the `valet/ValetBootstrap/pipelines` folder -2. Open the `valet/ValetBootstrap/pipelines/valet-pipeline1.config.json` file -3. Look for the `web - href` link -4. At the end of the link is the pipeline ID. Copy or note the ID. - -### Example -![Screen Shot 2022-05-10 at 8 50 06 AM](https://user-images.githubusercontent.com/26442605/167670536-b46aa383-74bd-4e22-a782-0de5d0ce64a5.png) - - -## Perform a migration -You will use the codespace preconfigured in this repository to perform the migration. - -1. Navigate to the codespace Visual Studio Code terminal -2. Verify you are in the valet directory -3. Copy the following command and replace: - - `GITHUB-ORG` with the name of your organization. - - `GITHUB-REPO` with the name of your repository. - - `PIPELINE-ID` with your pipeline ID. -4. Run `gh valet migrate` from the `valet` directory to migrate the pipeline to GitHub Actions.. -``` -gh valet migrate azure-devops pipeline --target-url https://github.com/GITHUB-ORG/GITHUB-REPO --pipeline-id PIPELINE-ID --output-dir migrate -``` - -### Example -![valet-migrate-1](https://user-images.githubusercontent.com/26442605/169617557-289cee54-0116-4d13-8e6f-a9d0508259e1.png) - - -5. Valet will create a pull request directly to your GitHub repository. -6. Click the green pull request link in the output of the migrate command. See below. - -### Example -![valet-migrate-5](https://user-images.githubusercontent.com/26442605/169617699-ce0c0720-8830-46ed-811d-c2fe1ccf06ea.png) - - -## View the pull request -The migrate output will show you the pull request on GitHub. - -### Example -![migrate-pr](https://user-images.githubusercontent.com/26442605/161110724-f39d9cb9-1992-44c5-bea5-da2fcebb074c.png) - -### Next Lab -[Migrate an Azure DevOps pipeline to GitHub Actions with a custom transformer](valet-migrate-custom-lab.md) diff --git a/code/Bootcamp.sln b/code/Bootcamp.sln deleted file mode 100644 index 099b74f..0000000 --- a/code/Bootcamp.sln +++ /dev/null @@ -1,56 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.30114.105 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{B26D43AA-4A35-4035-9E99-48EF9A3E64DD}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Attendee", "src\Attendee\Attendee.csproj", "{2804EC63-670C-4970-85E4-2A63C9327FF8}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{348A52EC-7046-4D1A-88DB-55B025C2BB68}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AttendeeTest", "test\AttendeeTest\AttendeeTest.csproj", "{DED76823-F195-46D4-8509-5692E3431D53}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Debug|x64.ActiveCfg = Debug|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Debug|x64.Build.0 = Debug|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Debug|x86.ActiveCfg = Debug|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Debug|x86.Build.0 = Debug|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Release|Any CPU.Build.0 = Release|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Release|x64.ActiveCfg = Release|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Release|x64.Build.0 = Release|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Release|x86.ActiveCfg = Release|Any CPU - {2804EC63-670C-4970-85E4-2A63C9327FF8}.Release|x86.Build.0 = Release|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Debug|x64.ActiveCfg = Debug|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Debug|x64.Build.0 = Debug|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Debug|x86.ActiveCfg = Debug|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Debug|x86.Build.0 = Debug|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Release|Any CPU.Build.0 = Release|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Release|x64.ActiveCfg = Release|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Release|x64.Build.0 = Release|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Release|x86.ActiveCfg = Release|Any CPU - {DED76823-F195-46D4-8509-5692E3431D53}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {2804EC63-670C-4970-85E4-2A63C9327FF8} = {B26D43AA-4A35-4035-9E99-48EF9A3E64DD} - {DED76823-F195-46D4-8509-5692E3431D53} = {348A52EC-7046-4D1A-88DB-55B025C2BB68} - EndGlobalSection -EndGlobal diff --git a/code/readme.md b/code/readme.md deleted file mode 100644 index d58f7e7..0000000 --- a/code/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -# code folder -This folder contains a dotnet core application used for building puproses with the CI/CD selected. - -### Note any file added to this folder or sub folders will get migrated to the Azure DevOps instance when the GitHub Action runs to create and bootstrap the Azure DevOps project - diff --git a/code/src/Attendee/Attendee.cs b/code/src/Attendee/Attendee.cs deleted file mode 100644 index 37e4c52..0000000 --- a/code/src/Attendee/Attendee.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; - -namespace Attendees -{ - public class Attendee - { - public bool AddAttendee(string added) - { - if (added == "exists") - return true; - - return false; - } - } -} diff --git a/code/src/Attendee/Attendee.csproj b/code/src/Attendee/Attendee.csproj deleted file mode 100644 index 563e6f9..0000000 --- a/code/src/Attendee/Attendee.csproj +++ /dev/null @@ -1,7 +0,0 @@ - - - - net5.0 - - - diff --git a/code/test/AttendeeTest/AttendeeExists.cs b/code/test/AttendeeTest/AttendeeExists.cs deleted file mode 100644 index 70e3b6d..0000000 --- a/code/test/AttendeeTest/AttendeeExists.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using Xunit; -using Xunit.Extensions; -using Attendees; - -namespace AttendeeTest -{ - public class AttendeeTest - { - [Fact] - public void AttendeeExistsReturnTrue() - { - Attendee attendee = new Attendee(); - bool doesExist = attendee.AddAttendee("doesnotexist"); - Assert.False(doesExist, "The attendee does not exist"); - } - } -} diff --git a/code/test/AttendeeTest/AttendeeTest.csproj b/code/test/AttendeeTest/AttendeeTest.csproj deleted file mode 100644 index 8b41439..0000000 --- a/code/test/AttendeeTest/AttendeeTest.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - net5.0 - - false - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - diff --git a/gitlab/1-configure.md b/gitlab/1-configure.md index 6b4aa0f..6a00435 100644 --- a/gitlab/1-configure.md +++ b/gitlab/1-configure.md @@ -6,7 +6,8 @@ You will need to complete all of the setup instructions [here](./readme.md#confi ## Configuring credentials -1. Run the setup script in the codespace's terminal to ensure the GitLab server is ready: +1. Run the setup script in the codespace terminal to ensure the GitLab server is ready: + ```bash ./gitlab/bootstrap/setup.sh ``` @@ -16,40 +17,40 @@ You will need to complete all of the setup instructions [here](./readme.md#confi - In the `PORTS` tab find the row for port 80. - Hover over the address under the `Local Address` column and click the globe to "open in browser". -3. Create a GitLab personal access token: +3. Create a GitLab personal access token (PAT): - Authenticate with the GitLab server using the following credentials: - Username: `root` - Password: `valet-labs!` - - Follow the GitLab [instructions](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) to generate a personal access token. + - Follow the GitLab [instructions](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) to generate a PAT. - Ensure the token has the `read_api` scope. - - Copy the generated token and save in a safe location. + - Copy the generated token and save it in a safe location. -4. Create a GitHub Personal Access Token (PAT): +4. Create a GitHub personal access token (PAT): - Open github.com in a new browser tab. - - Click your profile photo in the top right of the UI and click `Settings`. - - Click on `Developer Settings` in the left hand panel. - - Click `Personal Access Tokens` and then `Legacy tokens` (if present). - - Click `Generate new token` and then `Legacy tokens`. You may be required to authenticate with GitHub during this step. - - Select the following scopes: `read:packages` and `workflow`. + - In the top right corner of the UI, click your profile photo and click `Settings`. + - In the left panel, click `Developer Settings`. + - Click `Personal access tokens` and then `Legacy tokens` (if present). + - Click `Generate new token` and then `Generate new legacy token`. You may be required to authenticate with GitHub during this step. + - Select the following scopes: `workflow` and `read:packages`. - Click `Generate token`. - - Copy the generated PAT and save in a safe location. + - Copy the generated PAT and save it in a safe location. -4. Run the `configure` CLI command: +5. Run the `configure` CLI command: - Select the `TERMINAL` tab from within the codespace terminal window. - Run the following command: `gh valet configure`. - - Using the down arrow key to highlight `GitLab CI`, press the spacebar to select, and then hit enter to continue. - - At the prompt enter your GitHub Username and press enter. - - At the GitHub Container Registry prompt enter the GitHub PAT generated in step 4 and press enter. - - At the GitHub PAT prompt enter the GitHub PAT generated in step 4 and press enter. - - At the GitHub url prompt enter the GitHub instance url or hit enter to accept the default value (`https://github.com`). - - At the GitLab CI token prompt enter the GitLab CI access token from step 3 and press enter. - - At the GitLab CI url prompt enter `http://localhost` and press enter. + - Use the down arrow key to highlight `GitLab CI`, press the spacebar to select, and then press enter to continue. + - At the prompt, enter your GitHub username and press enter. + - At the GitHub Container Registry prompt, enter the GitHub PAT generated in step 4 and press enter. + - At the GitHub PAT prompt, enter the GitHub PAT generated in step 4 and press enter. + - At the GitHub URL prompt, enter the GitHub instance URL or press enter to accept the default value (`https://github.com`). + - At the GitLab CI token prompt, enter the GitLab CI access token from step 3 and press enter. + - At the GitLab CI URL prompt, enter `http://localhost` and press enter. ![img](https://user-images.githubusercontent.com/18723510/183990474-d0b2559c-d2bf-40d9-ac43-19af53e45329.png) ## Verify your environment -To verify our environment is configured correctly, we are going to run the `update` CLI command. The `update` CLI command will download the latest version of Valet to your codespace. +To verify your environment is configured correctly, run the `update` CLI command. The `update` CLI command will download the latest version of Valet to your codespace. 1. In the codespace terminal run the following command: @@ -59,7 +60,8 @@ To verify our environment is configured correctly, we are going to run the `upda 2. You should see a confirmation that you were logged into the GitHub Container Registry and Valet was updated to the latest version. - ```bash + ```console + $ gh valet update Login Succeeded latest: Pulling from valet-customers/valet-cli Digest: sha256:a7d00dee8a37e25da59daeed44b1543f476b00fa2c41c47f48deeaf34a215bbb diff --git a/gitlab/2-audit.md b/gitlab/2-audit.md index 09c1cac..614e88a 100644 --- a/gitlab/2-audit.md +++ b/gitlab/2-audit.md @@ -6,18 +6,18 @@ The `audit` command operates by fetching all of the pipelines defined in a GitLa ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a GitLab server. +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a GitLab server. 2. Completed the [configure lab](./1-configure.md#configuring-credentials). ## Perform an audit -We will be performing an audit against your preconfigured GitLab server. We will need to answer the following questions before running this command: +You will be performing an audit against your preconfigured GitLab server. Answer the following questions before running this command: -1. What namespace (e.g. group) do we want to audit? - - __valet__. In this example we will be auditing the `valet` group. In the future, you could add additional groups and subgroups to the audit command. +1. What namespace (e.g. group) do you want to audit? + - __valet__. In this example you will be auditing the `valet` group. In the future, you could add additional groups and subgroups to the audit command. -2. Where do we want to store the result? - - __./tmp/audit__. This can be any path within the working directory that Valet commands are executed from. +2. Where do you want to store the result? + - __tmp/audit__. This can be any path within the working directory from which Valet commands are executed. ### Steps @@ -34,7 +34,7 @@ We will be performing an audit against your preconfigured GitLab server. We will 1. Find the `audit_summary.md` file in the file explorer. 2. Right-click the `audit_summary.md` file and select `Open Preview`. -3. This file contains details that summarizes what percentage of your pipelines were converted automatically. +3. This file contains details that summarize what percentage of your pipelines were converted automatically. ### Review audit summary @@ -66,8 +66,9 @@ Unsupported: **1 (9%)** Here are some key terms in the “Pipelines” section in the above example: - __Successful__ pipelines had 100% of the pipeline constructs and individual items converted automatically to their GitHub Actions equivalent. -- __Partially successful__ pipelines had all of the pipeline constructs converted, however, there were some "Auto DevOps" pipelines. -- __Failed pipelines__ encountered a fatal error when being converted. This can occur for one of three reasons: +- __Partially successful__ pipelines had all of the pipeline constructs converted, however, there were some individual items (e.g. build tasks or build triggers) that were not converted automatically to their GitHub Actions equivalent. +- __Unsupported__ pipelines are definition types that are not supported by Valet. Auto DevOps pipelines are not supported. +- __Failed__ pipelines encountered a fatal error when being converted. This can occur for one of three reasons: - The pipeline was misconfigured and not valid in GitLab. - Valet encountered an internal error when converting it. - There was an unsuccessful network response, often due to invalid credentials, that caused the pipeline to be inaccessible. @@ -175,8 +176,8 @@ Each pipeline will have a variety of files written that include: - The original pipeline as it was defined in GitLab. - Any network responses used to convert a pipeline. - The converted workflow. -- Stack traces that can used to troubleshoot a failed pipeline conversion +- Stack traces that can be used to troubleshoot a failed pipeline conversion ### Next lab -[Perform a dry-run of a GitLab pipeline](3-dry-run.md) +[Forecast potential build runner usage](3-forecast.md) diff --git a/gitlab/6-forecast.md b/gitlab/3-forecast.md similarity index 67% rename from gitlab/6-forecast.md rename to gitlab/3-forecast.md index bb5294b..172ad98 100644 --- a/gitlab/6-forecast.md +++ b/gitlab/3-forecast.md @@ -4,27 +4,27 @@ In this lab you will use the `forecast` command to forecast potential GitHub Act ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a GitLab server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a GitLab server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). ## Perform a forecast -We will need to answer the following questions before running the `forecast` command: +Answer the following questions before running the `forecast` command: -1. What namespace do we want to run the forecast for? +1. What namespace do you want to run the forecast for? - **valet** -2. What is the date we want to start forecasting from? +2. What is the date you want to start forecasting from? - **2022-08-02**. This date is needed as it is prior to when the data was seeded in GitLab for these labs. This value defaults to the date one week ago, however, you should use a start date that will show a representative view of typical usage. -3. Where do we want to store the results? - - **./tmp/forecast_reports** +3. Where do you want to store the results? + - **tmp/forecast** ### Steps -1. Navigate to the codespace terminal +1. Navigate to your codespace terminal 2. Run the following command from the root directory: ```bash - gh valet forecast gitlab --output-dir ./tmp/forecast_reports --namespace valet --start-date 2022-08-02 + gh valet forecast gitlab --output-dir tmp/forecast --namespace valet --start-date 2022-08-02 ``` 3. The command will list all the files written to disk when the command succeeds. @@ -33,7 +33,7 @@ We will need to answer the following questions before running the `forecast` com ## Review the forecast report -The forecast report, logs, and completed job data will be located within the `tmp/forecast_reports` folder. +The forecast report, logs, and completed job data will be located within the `tmp/forecast` folder. 1. Find the `forecast_report.md` file in the file explorer. 2. Right-click the `forecast_report.md` file and select `Open Preview`. @@ -74,8 +74,8 @@ Here are some key terms of items defined in the forecast report: - The `job count` is the total number of completed jobs. - The `pipeline count` is the number of unique pipelines used. -- `Execution time` describes the amount of time a runner spent on a job. This metric can be used to help plan for the cost of GitHub hosted runners. - - This metric is correlated to how much you should expect to spend in GitHub Actions. This will vary depending on the hardware used for these minutes and the [Actions pricing calculator](https://github.com/pricing/calculator) should be used to estimate a dollar amount. +- `Execution time` describes the amount of time a runner spent on a job. This metric can be used to help plan for the cost of GitHub-hosted runners. + - This metric is correlated to how much you should expect to spend in GitHub Actions. This will vary depending on the hardware used for these minutes. You can use the [Actions pricing calculator](https://github.com/pricing/calculator) to estimate a dollar amount. - `Queue time` metrics describe the amount of time a job spent waiting for a runner to be available to execute it. - `Concurrent jobs` metrics describe the amount of jobs running at any given time. This metric can be used to define the number of runners a customer should configure. @@ -83,20 +83,20 @@ Additionally, these metrics are defined for each queue of runners defined in Git ## Forecasting multiple providers -We can examine the available options for the `forecast` command by running `gh valet forecast --help`. When you do this you will see the `--source-file-path` option: +You can examine the available options for the `forecast` command by running `gh valet forecast --help`. When you do this you will see the `--source-file-path` option: ![img](https://user-images.githubusercontent.com/19557880/186263140-f02c6cab-7979-417c-bdfe-b9590e9c5597.png) -The `--source-file-path` CLI option can be used to combine data from multiple reports into a single report. This becomes useful if you use multiple CI/CD providers and wanted to get a holistic view of the runner usage. This works by using the `.json` files generated by `forecast` commands as space-delimited values for the `--source-file-path` CLI option. Optionally, this value could be a glob pattern to dynamically specify the list of files (e.g. `**/*.json`). +You can use the `--source-file-path` CLI option to combine data from multiple reports into a single report. This becomes useful if you use multiple CI/CD providers and want to get a holistic view of the runner usage. This works by using the `.json` files generated by `forecast` commands as space-delimited values for the `--source-file-path` CLI option. Optionally, this value could be a glob pattern to dynamically specify the list of files (e.g. `**/*.json`). Run the following command from within the codespace terminal: ```bash -gh valet forecast --source-file-path tmp/**/jobs/*.json -o tmp/combined-forecast +gh valet forecast --source-file-path tmp/**/jobs/*.json --output-dir tmp/forecast-combined ``` You can now inspect the output of the command to see a forecast report using all of the files matching the `tmp/**/jobs/*.json` pattern. ## Next steps -This concludes all labs for migrating GitLab pipelines to Actions with Valet! +[Perform a dry-run migration of a GitLab pipeline](4-dry-run.md) diff --git a/gitlab/3-dry-run.md b/gitlab/4-dry-run.md similarity index 84% rename from gitlab/3-dry-run.md rename to gitlab/4-dry-run.md index cf66935..169e925 100644 --- a/gitlab/3-dry-run.md +++ b/gitlab/4-dry-run.md @@ -1,33 +1,33 @@ -# Perform a dry-run of a GitLab pipeline +# Perform a dry-run migration of a GitLab pipeline In this lab you will use the `dry-run` command to convert a GitLab pipeline to its equivalent GitHub Actions workflow. ## Prerequisites 1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a GitLab server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). +2. Completed the [configure lab](./1-configure.md#configuring-credentials). 3. Completed the [audit lab](./2-audit.md). ## Perform a dry run -We will be performing a dry-run against a pipeline in your preconfigured GitLab server. We will need to answer the following questions before running this command: +You will be performing a dry run against a pipeline in your preconfigured GitLab server. Answer the following questions before running this command: -1. What is the project we want to convert? +1. What project do you want to convert? - __basic-pipeline-example__ 2. What is the namespace for that project? - __Valet__ -3. Where do we want to store the result? - - __./tmp/dry-run-lab__. This can be any path within the working directory that Valet commands are executed from. +3. Where do you want to store the result? + - __tmp/dry-run__. This can be any path within the working directory from which Valet commands are executed. ### Steps -1. Navigate to the codespace terminal +1. Navigate to your codespace terminal 2. Run the following command from the root directory: ```bash - gh valet dry-run gitlab --output-dir ./tmp/dry-run-lab --namespace valet --project basic-pipeline-example + gh valet dry-run gitlab --output-dir tmp/dry-run --namespace valet --project basic-pipeline-example ``` 3. The command will list all the files written to disk when the command succeeds. @@ -35,9 +35,9 @@ We will be performing a dry-run against a pipeline in your preconfigured GitLab ![img](https://user-images.githubusercontent.com/18723510/184173635-aec28d1c-8c61-4dcf-a743-f86cbdc836c5.png) 4. View the converted workflow: - - Find `./tmp/dry-run/valet` in the file explorer pane in codespaces. + - Find `tmp/dry-run/valet` in the file explorer pane in your codespace. - Click `basic-pipeline-example.yml` to open. - + ## Inspect the output files The files generated from the `dry-run` command represent the equivalent Actions workflow for the given GitLab pipeline. The GitLab pipeline and converted workflow can be seen below: @@ -98,7 +98,7 @@ deploy_b:
Converted workflow 👇 - + ```yaml name: valet/basic-pipeline-example on: @@ -196,13 +196,14 @@ jobs: - run: echo "test stage complete. It will start at about the same time as deploy_a." - run: sleep 400 ``` +
-Despite these 2 pipelines using different syntax they will function equivantly. +Despite these two pipelines using different syntax they will function equivalently. -## Perform a dry-run of a pipeline using `include`'d files +## Perform a dry-run migration of a pipeline using `include`'d files -The previous example demonstrated a basic pipeline that mapped exactly to concepts in GitHub Actions. In this section, we will perform a dry-run of the `included-files-example` pipeline that uses the `include` statement in GitLab: +The previous example demonstrated a basic pipeline that mapped exactly to concepts in GitHub Actions. In this section, you will perform a dry run of the `included-files-example` pipeline that uses the `include` statement in GitLab: ```yaml include: @@ -247,8 +248,8 @@ jobs: It's important to note that Valet converted this into a single workflow without templates. This is because of fundamental differences in how GitLab templates and GitHub Actions templates (i.e. Reusable Workflows and Composite Actions) function in regards to job ordering. Unfortunately, elements of reusability will be sacrificed in order for the converted pipelines to function the same. It is likely that the output of Valet could be refactored to use [reusable workflow](https://docs.github.com/en/actions/using-workflows/reusing-workflows) at a later date. -As an added challenge, try constructing and running the `dry-run` command yourself. Hint, you should just have to change the project name. +As an added challenge, try constructing and running the `dry-run` command yourself. Hint, you should only have to change the project name. ## Next lab -[Use custom transformers to customize Valet's behavior](./4-custom-transformers.md) +[Use custom transformers to customize Valet's behavior](./5-custom-transformers.md) diff --git a/gitlab/4-custom-transformers.md b/gitlab/5-custom-transformers.md similarity index 60% rename from gitlab/4-custom-transformers.md rename to gitlab/5-custom-transformers.md index 25e0fbb..77a5fca 100644 --- a/gitlab/4-custom-transformers.md +++ b/gitlab/5-custom-transformers.md @@ -1,6 +1,6 @@ # Using custom transformers to customize Valet's behavior -In this lab we will build upon the `dry-run` command to override Valet's default behavior and customize the converted workflow using "custom transformers". Custom transformers can be used to: +In this lab you will build upon the `dry-run` command to override Valet's default behavior and customize the converted workflow using "custom transformers". Custom transformers can be used to: 1. Convert items that are not automatically converted. 2. Convert items that were automatically converted using different actions. @@ -9,16 +9,16 @@ In this lab we will build upon the `dry-run` command to override Valet's default ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start you GitLab server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). -3. Completed the [dry-run lab](./3-dry-run.md). +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start you GitLab server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [dry-run lab](./4-dry-run.md). ## Perform a dry-run -We will be performing a `dry-run` command to inspect the workflow that is converted by default. Run the following command within the codespace terminal: +You will be performing a `dry-run` command to inspect the workflow that is converted by default. Run the following command within the codespace terminal: ```bash -gh valet dry-run gitlab --output-dir tmp --namespace valet --project terraform-example +gh valet dry-run gitlab --output-dir tmp/dry-run --namespace valet --project terraform-example ``` The converted workflow that is generated by the above command can be seen below: @@ -53,17 +53,17 @@ jobs: -_Note_: You can refer to the previous [lab](./3-dry-run.md) to learn about the fundamentals of the `dry-run` command. +_Note_: You can refer to the previous [lab](./4-dry-run.md) to learn about the fundamentals of the `dry-run` command. ## Custom transformers for an unknown step -The converted workflow above contains an `artifacts.terraform` step was not automatically converted. We will need to answer the following questions before writing a custom transformer: +The converted workflow above contains an `artifacts.terraform` step that was not automatically converted. Answer the following questions before writing a custom transformer: 1. What is the "identifier" of the step to customize? - __artifacts.terraform__ 2. What is the desired Actions syntax to use instead? - - After some research, we have determined that the following bash script will provide similar functionality: + - After some research, you have determined that the following bash script will provide similar functionality: ```yaml - uses: actions/upload-artifact@v3 @@ -71,13 +71,13 @@ The converted workflow above contains an `artifacts.terraform` step was not auto path: VALUE_FROM_GITLAB ``` -Now we can begin to write the custom transformer. Customer transformers use a DSL built on top of Ruby and should be defined in a file with the `.rb` file extension. You can create this file by running the following command in your codespace terminal: +Now you can begin to write the custom transformer. Custom transformers use a DSL built on top of Ruby and should be defined in a file with the `.rb` file extension. You can create this file by running the following command in your codespace terminal: ```bash code transformers.rb ``` -Next, we will define a `transform` method for the `artifacts.terraform` identifier by adding the following code to `transformers.rb`: +Next, you will define a `transform` method for the `artifacts.terraform` identifier by adding the following code to `transformers.rb`: ```ruby transform "artifacts.terraform" do |item| @@ -92,10 +92,10 @@ end This method can use any valid ruby syntax and should return a `Hash` that represents the YAML that should be generated for a given step. Valet will use this method to convert a step with the provided identifier and will use the `item` parameter for the original values configured in GitLab. -Now, we can perform another `dry-run` command and use the `--custom-transformers` CLI option to provide this custom transformer. Run the following command within your codespace terminal: +Now you can perform another `dry-run` command and use the `--custom-transformers` CLI option to provide this custom transformer. Run the following command within your codespace terminal: ```bash -gh valet dry-run gitlab --output-dir tmp --namespace valet --project terraform-example --custom-transformers transformers.rb +gh valet dry-run gitlab --output-dir tmp/dry-run --namespace valet --project terraform-example --custom-transformers transformers.rb ``` The converted workflow that is generated by the above command will now use the custom logic for the `artifacts.terraform` step. @@ -107,7 +107,7 @@ The converted workflow that is generated by the above command will now use the c + path: "$PLAN_JSON" ``` -_Note_: If you were unsure what the data structure of `item` was then you could use the following code in the custom transformer to print `item` to the console: +_Note_: If you were unsure what the data structure of `item` was, you could use the following code in the custom transformer to print `item` to the console: ```ruby transform "artifacts.terraform" do |item| @@ -117,9 +117,9 @@ end ## Custom transformers for environment variables -We can also use custom transformers to edit the values of environment variables in converted workflows. In our example, we will be updating the `PLAN_JSON` environment variable to be `custom_plan.json` instead of `plan.json`. +You can also use custom transformers to edit the values of environment variables in converted workflows. In this example, you will update the `PLAN_JSON` environment variable to be `custom_plan.json` instead of `plan.json`. -To do this, add the following code to the `transformers.rb` file. +To do this, add the following code at the top of the `transformers.rb` file. ```ruby env "PLAN_JSON", "custom_plan.json" @@ -127,7 +127,7 @@ env "PLAN_JSON", "custom_plan.json" In this example, the first parameter to the `env` method is the environment variable name and the second is the updated value. -Now, we can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `PLAN_JSON` environment variable will be set to `custom_plan.json`: +Now you can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `PLAN_JSON` environment variable will be set to `custom_plan.json`: ```diff env: @@ -136,7 +136,7 @@ Now, we can perform another `dry-run` command with the `--custom-transformers` C + PLAN_JSON: custom_plan.json ``` -At this point of the lab the file contents of `transformers.rb` should match this: +At this point, the file contents of `transformers.rb` should match this:
Custom transformers 👇 @@ -156,11 +156,11 @@ At this point of the lab the file contents of `transformers.rb` should match thi
-Thats it! Congratulations you have overridden Valet's default behavior by customizing the conversion of: +That's it! Congratulations, you have overridden Valet's default behavior by customizing the conversion of: - Unknown steps - Environment variables ## Next lab -[Perform a production migration of a GitLab pipeline](5-migrate.md) \ No newline at end of file +[Perform a production migration of a GitLab pipeline](6-migrate.md) diff --git a/gitlab/5-migrate.md b/gitlab/6-migrate.md similarity index 55% rename from gitlab/5-migrate.md rename to gitlab/6-migrate.md index b58f953..3dcd734 100644 --- a/gitlab/5-migrate.md +++ b/gitlab/6-migrate.md @@ -4,21 +4,21 @@ In this lab, you will use the `migrate` command to convert a GitLab pipeline and ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a GitLab server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). -3. Completed the [dry-run lab](./3-dry-run.md). -4. Completed the [custom transformers lab](./4-custom-transformers.md). +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a GitLab server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [dry-run lab](./4-dry-run.md). +4. Completed the [custom transformers lab](./5-custom-transformers.md). ## Performing a migration -We need to answer the following questions before running a `migrate` command: +Answer the following questions before running a `migrate` command: -1. What is the project we want to migrate? +1. What project do you want to migrate? - __rails-example__ 2. What is the namespace for that project? - __Valet__ -3. Where do we want to store the logs? - - __./tmp/migrate__ +3. Where do you want to store the logs? + - __tmp/migrate__ 4. What is the URL for the GitHub repository to add the workflow to? - __this repository__. The URL should should follow the pattern with `:owner` and `:repo` replaced with your values. @@ -27,7 +27,7 @@ We need to answer the following questions before running a `migrate` command: 1. Run the following `migrate` command in the codespace terminal: ```bash - gh valet migrate gitlab --target-url https://github.com/:owner/:repo --output-dir ./tmp/migrate --namespace valet --project rails-example + gh valet migrate gitlab --target-url https://github.com/:owner/:repo --output-dir tmp/migrate --namespace valet --project rails-example ``` 2. The command will write the URL to the pull request that was created when the command succeeds. @@ -38,16 +38,16 @@ We need to answer the following questions before running a `migrate` command: ### Inspect the pull request -The first thing we should notice about the PR is that there is a list of manual steps for us to complete. +The first thing to notice about the pull request is that there is a list of manual steps to complete. -Next, you can inspect the "Files changed" in this PR and see the converted workflow that is being added. Any additional changes or code reviews that were needed should be done in this PR. +Next, you can inspect the "Files changed" in this pull request and see the converted workflow that is being added. Any additional changes or code reviews that were needed should be done in this pull request. + +Finally, you can merge the pull request once your review has completed. You can then view the workflow running by selecting the "Actions" menu in the top navigation bar in GitHub. -Finally, you can merge the PR once your review has completed. We can then view the workflow running by selecting the "Actions" menu in the top navigation bar in GitHub. - ![img](https://user-images.githubusercontent.com/18723510/184960870-590b1a28-422f-4350-9ec0-0423bf7ad445.png) At this point, the migration has completed and you have successfully migrated a GitLab pipeline to Actions! ### Next Lab -[Forecast potential build runner usage](6-forecast.md) +This concludes all labs for migrating GitLab pipelines to Actions with Valet! diff --git a/gitlab/bootstrap/setup.sh b/gitlab/bootstrap/setup.sh index 021ce64..27006ba 100755 --- a/gitlab/bootstrap/setup.sh +++ b/gitlab/bootstrap/setup.sh @@ -33,10 +33,6 @@ else docker exec -it gitlab update-permissions &> /dev/null fi -# allow valet to talk to GitLab by removing network isolation between containers -export DOCKER_ARGS="--network=host" -`grep -q "export DOCKER_ARGS=" ~/.bashrc || echo 'export DOCKER_ARGS="--network=host"' >> ~/.bashrc` - echo -e "Waiting for GitLab to be ready. This might take a while \U23F0" until $(curl --output /dev/null --silent --head --fail http://localhost); do printf '.' diff --git a/gitlab/readme.md b/gitlab/readme.md index 042e378..ad3d2a0 100644 --- a/gitlab/readme.md +++ b/gitlab/readme.md @@ -1,59 +1,61 @@ # GitLab to Actions migrations powered by Valet -The instructions below will guide you through configuring a GitHub Codespace environment that will be used in subsequent labs that demonstrate how to use Valet to migrate GitLab pipelines to GitHub Actions. +These instructions will guide you through configuring the GitHub Codespaces environment that will be used in these labs that demonstrate how to use Valet to migrate GitLab pipelines to GitHub Actions. These steps **must** be completed prior to starting other labs. ## Create your own repository for these labs -1. Ensure that you have created a repository using the [valet-customers/labs](https://github.com/valet-customers/labs) as a template. +- Ensure that you have created a repository using [valet-customers/labs](https://github.com/valet-customers/labs) as a template. -## Configure your Codespace +## Configure your codespace -1. Start the codespace +1. Start a new codespace -- Click the `Code` button down arrow above the repository on the repository's landing page. -- Click the `Codespaces` tab. -- Click `Create codespaces on main` to create the codespace. -- After the Codespace has initialized there will be a terminal present. + - Click the `Code` button on your repository's landing page. + - Click the `Codespaces` tab. + - Click `Create codespaces on main` to create the codespace. + - After the codespace has initialized there will be a terminal present. 2. Verify the Valet CLI is installed and working. More information on the Valet extension for the official GitHub CLI can be found [here](https://github.com/github/gh-valet). -- Run the following command in the codespace's terminal: + - Run the following command in the codespace terminal: - ```bash - gh valet version - ``` + ```bash + gh valet version + ``` -- Verify the output is similar to below. - - ```bash - gh version 2.14.3 (2022-07-26) - gh valet github/gh-valet v0.1.12 - valet-cli unknown - ``` + - Verify the output is similar to below. - - If `gh valet version` did not produce similar output then please follow the troubleshooting [guide](#troubleshoot-the-valet-cli). + ```console + $ gh valet version + gh version 2.14.3 (2022-07-26) + gh valet github/gh-valet v0.1.12 + valet-cli unknown + ``` + + - If `gh valet version` did not produce similar output, please refer to the troubleshooting [guide](#troubleshoot-the-valet-cli). ## Bootstrap a GitLab server -1. Execute the GitLab setup script that will start a container with GitLab running inside of it. The script should be executed when starting a new Codespace or restarting an existing one. +1. Execute the GitLab setup script that will start a container with GitLab running inside of it. The script should be executed when starting a new codespace or restarting an existing one. -- Run the following command from the codespace's terminal: - - ```bash - ./gitlab/bootstrap/setup.sh - ``` + - Run the following command from the codespace terminal: -- After some time, a pop-up box should appear with a link to the URL for your GitLab server. - - You can also access the URL by going to the `Ports` tab in your terminal. Right click on the URL listed under the `Local Address` and click the `Open in Browser` tab. + ```bash + ./gitlab/bootstrap/setup.sh + ``` + + - After some time, a pop-up box should appear with a link to the URL for your GitLab server. + + - You can also access the URL by going to the `Ports` tab in your terminal. Right-click the URL listed under the `Local Address` and click the `Open in Browser` tab. 2. Open the GitLab server in your browser and use the following credentials to authenticate: - - Username: `root` - - Password: `valet-labs!` + - Username: `root` + - Password: `valet-labs!` -- Once authenticated, you should see a GitLab server with a few predefined pipelines in the `valet` group. +3. Once authenticated, you should see a GitLab server with a few predefined pipelines in the `valet` group. ## Labs for GitLab @@ -61,17 +63,17 @@ Perform the following labs to learn more about Actions migrations with Valet: 1. [Configure credentials for Valet](1-configure.md) 2. [Perform an audit on GitLab pipelines](2-audit.md) -3. [Perform a dry-run of a GitLab pipeline](3-dry-run.md) -4. [Use custom transformers to customize Valet's behavior](4-custom-transformers.md) -5. [Perform a production migration of a GitLab pipeline](5-migrate.md) -6. [Forecast potential build runner usage](6-forecast.md) +3. [Forecast potential build runner usage](3-forecast.md) +4. [Perform a dry-run migration of a GitLab pipeline](4-dry-run.md) +5. [Use custom transformers to customize Valet's behavior](5-custom-transformers.md) +6. [Perform a production migration of a GitLab pipeline](6-migrate.md) ## Troubleshoot the Valet CLI The CLI extension for Valet can be manually installed by following these steps: - Verify you are in the codespace terminal -- Run this command from within the codespace's terminal: +- Run this command from within the codespace terminal: ```bash gh extension install github/gh-valet @@ -79,14 +81,15 @@ The CLI extension for Valet can be manually installed by following these steps: - Verify the result of the install contains: - ```bash + ```console + $ gh extension install github/gh-valet ✓ Installed extension github/gh-valet ``` -- If you get an error similar to the image below, then click the link in the terminal output to authorize the token. +- If you get an error similar to the image below, click the link in the terminal output to authorize the token. - Restart the codespace after clicking the link. ![img](https://user-images.githubusercontent.com/26442605/169588015-9414404f-82b6-4d0f-89d4-5f0e6941b029.png) -- Verify Valet CLI extension is installed and working by running the following command from the codespace's terminal: +- Verify Valet CLI extension is installed and working by running the following command from the codespace terminal: ```bash gh valet version diff --git a/jenkins/1-configure.md b/jenkins/1-configure.md index 4b519be..8128140 100644 --- a/jenkins/1-configure.md +++ b/jenkins/1-configure.md @@ -7,44 +7,44 @@ You will need to complete all of the setup instructions [here](./readme.md#confi ## Configuring credentials 1. Open the Jenkins server in a new browser tab: - 1. Click the `PORTS` tab in the codespace terminal window. - 2. In the `PORTS` tab find the row for port 8080. - 3. Hover over the address under the `Local Address` column and click the globe to "open in browser". + - Click the `PORTS` tab in the codespace terminal window. + - In the `PORTS` tab, find the row for port 8080. + - Hover over the address under the `Local Address` column and click the globe to "open in browser". 2. Create a Jenkins API token: - 1. Click the `admin` button in the top right menu bar. - 2. Click on the `Configure` gear located on the left hand panel. - 3. Click the `Add new Token` button in the `API token` section and click `Generate`. - 4. Copy the generated API token and save in a safe location. + - In the top right menu bar, click the `admin` button. + - In the left panel, click the `Configure` gear. + - In the `API token` section, click the `Add new Token` button and click `Generate`. + - Copy the generated API token and save it in a safe location. ![img](https://user-images.githubusercontent.com/19557880/184041667-d06cb7f2-a885-474e-b728-7567314aeaf3.png) -3. Create a GitHub Personal Access Token (PAT): - 1. Open github.com in a new browser tab. - 2. Click your profile photo in the top right of the UI and click `Settings`. - 3. Click on `Developer Settings` in the left hand panel. - 4. Click `Personal Access Tokens` and then `Legacy tokens` (if present). - 5. Click `Generate new token` and then `Legacy tokens`. You may be required to authenticate with GitHub during this step. - 6. Select the following scopes: `read:packages` and `workflow`. - 7. Click `Generate token`. - 8. Copy the generated PAT and save in a safe location. +3. Create a GitHub personal access token (PAT): + - Open github.com in a new browser tab. + - In the top right corner of the UI, click your profile photo and click `Settings`. + - In the left panel, click `Developer Settings`. + - Click `Personal access tokens` and then `Legacy tokens` (if present). + - Click `Generate new token` and then `Generate new legacy token`. You may be required to authenticate with GitHub during this step. + - Select the following scopes: `workflow` and `read:packages`. + - Click `Generate token`. + - Copy the generated PAT and save it in a safe location. 4. Run the `configure` CLI command: - Select the `TERMINAL` tab from within the codespace terminal window. - Run the following command: `gh valet configure`. - - Using the down arrow key to highlight `Jenkins`, press the spacebar to select, and then hit enter to continue. - - At the prompt enter your GitHub Username and press enter. - - At the GitHub Container Registry prompt enter the GitHub PAT generated in step 3 and press enter. - - At the GitHub PAT prompt enter the GitHub PAT generated in step 3 and press enter. - - At the GitHub url prompt enter the GitHub instance url or hit enter to accept the default value (`https://github.com`). - - At the Jenkins token prompt enter the Jenkins access token from step 2 and press enter. - - At the Jenkins url prompt enter `http://localhost:8080/` and press enter. - - At the Personal access token to fetch source code in GitHub prompt hit enter to accept the default value. + - Use the down arrow key to highlight `Jenkins`, press the spacebar to select, and then press enter to continue. + - At the prompt, enter your GitHub username and press enter. + - At the GitHub Container Registry prompt, enter the GitHub PAT generated in step 3 and press enter. + - At the GitHub PAT prompt, enter the GitHub PAT generated in step 3 and press enter. + - At the GitHub URL prompt, enter the GitHub instance URL or press enter to accept the default value (`https://github.com`). + - At the Jenkins token prompt, enter the Jenkins access token from step 2 and press enter. + - At the Jenkins URL prompt, enter `http://localhost:8080/` and press enter. + - At the Personal access token to fetch source code in GitHub prompt, press enter to accept the default value. ![img](https://user-images.githubusercontent.com/19557880/184041328-ce54ea22-b0cd-4c84-b02c-10ad7b09ad89.png) ## Verify your environment -To verify our environment is configured correctly, we are going to run the `update` CLI command. The `update` CLI command will download the latest version of Valet to your codespace. +To verify your environment is configured correctly, run the `update` CLI command. The `update` CLI command will download the latest version of Valet to your codespace. 1. In the codespace terminal run the following command: @@ -54,7 +54,8 @@ To verify our environment is configured correctly, we are going to run the `upda 2. You should see a confirmation that you were logged into the GitHub Container Registry and Valet was updated to the latest version. - ```bash + ```console + $ gh valet update Login Succeeded latest: Pulling from valet-customers/valet-cli Digest: sha256:a7d00dee8a37e25da59daeed44b1543f476b00fa2c41c47f48deeaf34a215bbb diff --git a/jenkins/2-audit.md b/jenkins/2-audit.md index e7673b7..f637503 100644 --- a/jenkins/2-audit.md +++ b/jenkins/2-audit.md @@ -6,18 +6,18 @@ The `audit` command operates by fetching all of the pipelines defined in a Jenki ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a Jenkins server. +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a Jenkins server. 2. Completed the [configure lab](./1-configure.md#configuring-credentials). ## Perform an audit -We will be performing an audit against your preconfigured Jenkins server. We will need to answer the following questions before running this command: +You will be performing an audit against your preconfigured Jenkins server. Answer the following questions before running this command: -1. Do we want to audit the entire Jenkins instance or just a single folder? - - In this example we will be auditing the entire Jenkins instance, but in the future if you wanted to configure a specific folder to be audited add the `-f ` flag to the audit command. +1. Do you want to audit the entire Jenkins instance or just a single folder? + - In this example you will audit the entire Jenkins instance, but in the future if you wanted to configure a specific folder to be audited add the `-f ` flag to the `audit` command. -2. Where do we want to store the result? - - __./tmp/audit__. This can be any path within the working directory that Valet commands are executed from. +2. Where do you want to store the result? + - __tmp/audit__. This can be any path within the working directory from which Valet commands are executed. ### Steps @@ -30,7 +30,30 @@ We will be performing an audit against your preconfigured Jenkins server. We wil 3. The command will list all the files written to disk in green when the command succeeds. - ![img](https://user-images.githubusercontent.com/19557880/184682347-b19760fa-36a6-423e-a445-bb30eda5ac59.png) + ```console + $ gh valet audit jenkins --output-dir tmp/audit + [2022-08-20 22:08:20] Logs: 'tmp/audit/log/valet-20220916-015817.log' + [2022-08-20 22:08:20] Auditing 'http://localhost:8080/' + [2022-08-20 22:08:20] Output file(s):==========================================| + [2022-08-20 22:08:20] tmp/audit/demo_pipeline.yml + [2022-08-20 22:08:20] tmp/audit/demo_pipeline.config.json + [2022-08-20 22:08:20] tmp/audit/demo_pipeline.jenkinsfile + [2022-08-20 22:08:20] tmp/audit/groovy_script.error.txt + [2022-08-20 22:08:20] tmp/audit/groovy_script.config.json + [2022-08-20 22:08:20] tmp/audit/monas_dev_work/monas_freestyle.yml + [2022-08-20 22:08:20] tmp/audit/monas_dev_work/monas_freestyle.config.json + [2022-08-20 22:08:20] tmp/audit/monas_dev_work/monas_pipeline.yml + [2022-08-20 22:08:20] tmp/audit/monas_dev_work/monas_pipeline.config.json + [2022-08-20 22:08:20] tmp/audit/monas_dev_work/monas_pipeline.jenkinsfile + [2022-08-20 22:08:20] tmp/audit/test_freestyle_project.yml + [2022-08-20 22:08:20] tmp/audit/test_freestyle_project.config.json + [2022-08-20 22:08:20] tmp/audit/test_mutlibranch_pipeline.config.json + [2022-08-20 22:08:20] tmp/audit/test_pipeline.yml + [2022-08-20 22:08:20] tmp/audit/test_pipeline.config.json + [2022-08-20 22:08:20] tmp/audit/test_pipeline.jenkinsfile + [2022-08-20 22:08:20] tmp/audit/workflow_usage.csv + [2022-08-20 22:08:20] tmp/audit/audit_summary.md + ``` ## Inspect the output files @@ -46,7 +69,28 @@ The audit summary, logs, config files, jenkinsfiles, and transformed workflows w The pipeline summary section contains high level statistics regarding the conversion rate done by Valet: - ![img](https://user-images.githubusercontent.com/19557880/184683664-81985baf-5c03-4765-a067-f4023416e3ea.png) +```md +## Pipelines + +Total: **7** + +- Successful: **3 (42%)** +- Partially successful: **3 (42%)** +- Unsupported: **1 (14%)** +- Failed: **0 (0%)** + +### Job types + +Supported: **6 (85%)** + +- flow-definition: **3** +- project: **2** +- org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject: **1** + +Unsupported: **1 (14%)** + +- scripted: **1** +``` Here are some key terms in the “Pipelines” section in the above example: @@ -56,7 +100,7 @@ Here are some key terms in the “Pipelines” section in the above example: - Flow Definition - Project (declarative Jenkinsfile pipelines) - Multibranch Project -- __Failed pipelines__ encountered a fatal error when being converted. This can occur for one of three reasons: +- __Failed__ pipelines encountered a fatal error when being converted. This can occur for one of three reasons: - The pipeline was misconfigured and not valid in Jenkins. - Valet encountered an internal error when converting it. - There was an unsuccessful network response, often due to invalid credentials, that caused the pipeline to be inaccessible. @@ -67,7 +111,35 @@ The "Job types" section will summarize which types of pipelines are being used a The build steps summary section presents an overview of the individual build steps that are used across all pipelines and how many were automatically converted by Valet. - ![img](https://user-images.githubusercontent.com/19557880/184684062-69ab0bde-5e32-45f8-a7dd-ed4655872975.png) +```md +### Build steps + +Total: **17** + +Known: **13 (76%)** + +- echo: **6** +- hudson.tasks.Shell: **3** +- junit: **2** +- archiveArtifacts: **1** +- sh: **1** + +Unknown: **3 (17%)** + +- sleep: **2** +- hudson.plugins.git.GitPublisher: **1** + +Unsupported: **1 (5%)** + +- hudson.tasks.Mailer: **1** + +Actions: **22** + +- run: **10** +- actions/checkout@v2: **9** +- EnricoMi/publish-unit-test-result-action@v1.7: **2** +- actions/upload-artifact@v2: **1** +``` Here are some key terms in the "Build steps" section in the above example: @@ -86,7 +158,21 @@ There is an equivalent breakdown of build triggers, environment variables, and o The manual tasks summary section presents an overview of the manual tasks that you will need to perform that Valet is not able to complete automatically. - ![img](https://user-images.githubusercontent.com/19557880/184684249-9accfd94-c2df-4891-af56-dcff66beb557.png) +```md +### Manual tasks + +Total: **9** + +Secrets: **2** + +- `${{ secrets.SECRET_TEST_EXPRESSION_VAR }}`: **1** +- `${{ secrets.EXPRESSION_FIRST_VAR }}`: **1** + +Self hosted runners: **7** + +- `TeamARunner`: **6** +- `DemoRunner`: **1** +``` Here are some key terms in the “Manual tasks” section in the above example: @@ -97,7 +183,50 @@ Here are some key terms in the “Manual tasks” section in the above example: The final section of the audit report provides a manifest of all of the files that are written to disk during the audit. These files include: - ![img](https://user-images.githubusercontent.com/19557880/184684416-b3db774e-4ab8-46e0-91ad-e503632df5cb.png) +```md +### Successful + +#### demo_pipeline + +- [demo_pipeline.yml](demo_pipeline.yml) +- [demo_pipeline.config.json](demo_pipeline.config.json) +- [demo_pipeline.jenkinsfile](demo_pipeline.jenkinsfile) + +#### monas_dev_work/monas_freestyle + +- [monas_dev_work/monas_freestyle.yml](monas_dev_work/monas_freestyle.yml) +- [monas_dev_work/monas_freestyle.config.json](monas_dev_work/monas_freestyle.config.json) + +#### test_mutlibranch_pipeline + +- [test_mutlibranch_pipeline.config.json](test_mutlibranch_pipeline.config.json) + +### Partially successful + +#### monas_dev_work/monas_pipeline + +- [monas_dev_work/monas_pipeline.yml](monas_dev_work/monas_pipeline.yml) +- [monas_dev_work/monas_pipeline.config.json](monas_dev_work/monas_pipeline.config.json) +- [monas_dev_work/monas_pipeline.jenkinsfile](monas_dev_work/monas_pipeline.jenkinsfile) + +#### test_freestyle_project + +- [test_freestyle_project.yml](test_freestyle_project.yml) +- [test_freestyle_project.config.json](test_freestyle_project.config.json) + +#### test_pipeline + +- [test_pipeline.yml](test_pipeline.yml) +- [test_pipeline.config.json](test_pipeline.config.json) +- [test_pipeline.jenkinsfile](test_pipeline.jenkinsfile) + +### Failed + +#### groovy_script + +- [groovy_script.error.txt](groovy_script.error.txt) +- [groovy_script.config.json](groovy_script.config.json) +``` Each pipeline will have a variety of files written that include: @@ -108,4 +237,4 @@ Each pipeline will have a variety of files written that include: ## Next lab -[Perform a dry-run of a Jenkins pipeline](3-dry-run.md) +[Forecast potential build runner usage](3-forecast.md) diff --git a/jenkins/3-forecast.md b/jenkins/3-forecast.md new file mode 100644 index 0000000..7196536 --- /dev/null +++ b/jenkins/3-forecast.md @@ -0,0 +1,127 @@ +# Forecast potential build runner usage + +In this lab you will use the `forecast` command to forecast potential GitHub Actions usage by computing metrics from completed pipeline runs in your Jenkins server. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a Jenkins server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). + +## Perform a forecast + +Answer the following questions before running the `forecast` command: + +1. Do you want to forecast the entire Jenkins server or a single folder? + - You will be forecasting the entire Jenkins server but you could optionally limit this by using the `-f ` CLI option. + +2. What is the date you want to start forecasting from? + - __2022-08-02__. This date is needed as it is prior to when the data was seeded in Jenkins for these labs. This value defaults to the date one week ago, however, you should use a start date that will show a representative view of typical usage. + +3. Where do you want to store the results? + - `tmp/forecast` + +### Steps + +1. Navigate to your codespace terminal +2. Run the following command from the root directory: + + ```bash + gh valet forecast jenkins --output-dir tmp/forecast --start-date 2022-08-02 + ``` + +3. The command will list all the files written to disk when the command succeeds. + + ```console + $ gh valet forecast jenkins --output-dir tmp/forecast --start-date 2022-08-02 + [2022-08-20 22:08:20] Logs: 'tmp/forecast/log/valet-20220916-021004.log' + [2022-08-20 22:08:20] Forecasting 'http://localhost:8080/' + [2022-08-20 22:08:20] Output file(s): + [2022-08-20 22:08:20] tmp/forecast/jobs/09-16-2022-02-10_jobs_0.json + [2022-08-20 22:08:20] tmp/forecast/forecast_report.md + ``` + +## Review the forecast report + +The forecast report, logs, and completed job data will be located within the `tmp/forecast` folder. + +1. Find the `forecast_report.md` file in the file explorer. +2. Right-click the `forecast_report.md` file and select `Open Preview`. +3. This file contains metrics used to forecast potential GitHub Actions usage. + +### Total + +The "Total" section of the forecast report contains high level statistics related to all the jobs completed after the `--start-date` CLI option: + +```md +- Job count: **73** +- Pipeline count: **6** + +- Execution time + + - Total: **27,057 minutes** + - Median: **2 minutes** + - P90: **19 minutes** + - Min: **0 minutes** + - Max: **15,625 minutes** + +- Queue time + + - Median: **0 minutes** + - P90: **0 minutes** + - Min: **0 minutes** + - Max: **0 minutes** + +- Concurrent jobs + + - Median: **1** + - P90: **3** + - Min: **0** + - Max: **29** +``` + +Here are some key terms of items defined in the forecast report: + +- The `job count` is the total number of completed jobs. +- The `pipeline count` is the number of unique pipelines used. +- `Execution time` describes the amount of time a runner spent on a job. This metric can be used to help plan for the cost of GitHub-hosted runners. + - This metric is correlated to how much you should expect to spend in GitHub Actions. This will vary depending on the hardware used for these minutes. You can use the [Actions pricing calculator](https://github.com/pricing/calculator) to estimate a dollar amount. +- `Queue time` metrics describe the amount of time a job spent waiting for a runner to be available to execute it. +- `Concurrent jobs` metrics describe the amount of jobs running at any given time. This metric can be used to define the number of runners a customer should configure. + +Additionally, these metrics are defined for each queue of runners defined in Jenkins. This is especially useful if there are a mix of hosted/self-hosted runners or high/low spec machines to see metrics specific to different types of runners. + +## Forecasting multiple providers + +You can examine the available options for the `forecast` command by running `gh valet forecast jenkins --help`. When you do this you will see the `--source-file-path` option: + +```console +$ gh valet forecast jenkins --help +Options: + -u, --jenkins-instance-url The URL of the Jenkins CI instance. + -n, --jenkins-username Username for the Jenkins instance. + -t, --jenkins-access-token Access token for the Jenkins instance. + -f, --folders Folders to forecast in the instance + --source-file-path The file path(s) to existing jobs data. + -o, --output-dir (REQUIRED) The location for any output files. + --start-date The start date of the forecast analysis in YYYY-MM-DD format. [default: 9/9/2022 2:14:15 AM] + --time-slice The time slice in seconds to use for computing concurrency metrics. [default: 60] + --credentials-file The file containing the credentials to use. + --no-telemetry Boolean value to disallow telemetry. + --no-ssl-verify Disable ssl certificate verification. + --no-http-cache Disable caching of http responses. + -?, -h, --help Show help and usage information +``` + +You can use the `--source-file-path` CLI option to combine data from multiple reports into a single report. This becomes useful if you use multiple CI/CD providers and wanted to get a holistic view of the runner usage. This works by using the `.json` files generated by `forecast` commands as space-delimited values for the `--source-file-path` CLI option. Optionally, this value could be a glob pattern to dynamically specify the list of files (e.g. `**/*.json`). + +Run the following command from within the codespace terminal: + +```bash +gh valet forecast --source-file-path tmp/**/jobs/*.json --output-dir tmp/forecast-combined +``` + +You can now inspect the output of the command to see a forecast report using all of the files matching the `tmp/**/jobs/*.json` pattern. + +## Next steps + +[Perform a dry-run migration of a Jenkins pipeline](4-dry-run.md) diff --git a/jenkins/3-dry-run.md b/jenkins/4-dry-run.md similarity index 60% rename from jenkins/3-dry-run.md rename to jenkins/4-dry-run.md index d022e59..e24f7cd 100644 --- a/jenkins/3-dry-run.md +++ b/jenkins/4-dry-run.md @@ -1,41 +1,46 @@ -# Perform a dry-run of a Jenkins pipeline +# Perform a dry-run migration of a Jenkins pipeline In this lab you will use the `dry-run` command to convert a Jenkins pipeline to its equivalent GitHub Actions workflow. ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a Jenkins server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a Jenkins server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). 3. Completed the [audit lab](./2-audit.md). -## Perform a dry-run +## Perform a dry run -We will be performing a dry-run against a pipeline in your preconfigured Jenkins server. We will need to answer the following questions before running this command: +You will be performing a dry run against a pipeline in your preconfigured Jenkins server. Answer the following questions before running this command: -1. What is the name of the pipeline we want to convert? +1. What is the name of the pipeline you want to convert? - __test_pipeline__ -2. What is the URL of the pipeline we want to convert? +2. What is the URL of the pipeline you want to convert? - ____ -3. Where do we want to store the result? - - __./tmp/dry-run-lab__. This can be any path within the working directory that Valet commands are executed from. +3. Where do you want to store the result? + - __tmp/dry-run__. This can be any path within the working directory from which Valet commands are executed. ### Steps -1. Navigate to the codespace terminal +1. Navigate to your codespace terminal 2. Run the following command from the root directory: ```bash - gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline -o .tmp/jenkins/dry-run + gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline --output-dir tmp/dry-run ``` 3. The command will list all the files written to disk when the command succeeds. - ![img](https://user-images.githubusercontent.com/19557880/184935603-5c2d4dfe-66ef-4cb1-9398-e96954ca72e3.png) + ```console + $ gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline --output-dir tmp/dry-run + [2022-08-20 22:08:20] Logs: 'tmp/dry-run/log/valet-20220916-022338.log' + [2022-08-20 22:08:20] Output file(s): + [2022-08-20 22:08:20] tmp/dry-run/test_pipeline.yml + ``` 4. View the converted workflow: - - Find `./tmp/dry-run` in the file explorer pane in codespaces. + - Find `tmp/dry-run` in the file explorer pane in your codespace. - Click `test_pipeline.yml` to open ## Inspect the output files @@ -66,7 +71,7 @@ pipeline { } stage('test') { steps{ - junit '**/target/*.xml' + junit '**/target/*.xml' } } } @@ -123,7 +128,7 @@ jobs: -These 2 pipelines function equivantly despite using different syntax. In this case, the pipeline conversion was “partially successful” (i.e. there were item(s) not automatically converted) and the unconverted item was placed as comment in the location the Jenkins pipeline used it. For example: +These two pipelines function equivalently despite using different syntax. In this case, the pipeline conversion was “partially successful” (i.e. there were item(s) not automatically converted) and the unconverted item was placed as comment in the location the Jenkins pipeline used it. For example: ```diff - sleep 80 @@ -135,10 +140,10 @@ These 2 pipelines function equivantly despite using different syntax. In this ca + # value: 80 ``` -In the next lab, we'll learn how to override Valet's default behavior and customize the converted workflow that is generate. +In the next lab, you'll learn how to override Valet's default behavior and customize the converted workflow that is generated. -Try running the `dry-run` command for different pipelines in the Jenkins server. As a hint, you just have to change the `--source-url` CLI option. +Try running the `dry-run` command for different pipelines in the Jenkins server. As a hint, you only have to change the `--source-url` CLI option. ## Next lab -[Use custom transformers to customize Valet's behavior](4-custom-transformers.md) +[Use custom transformers to customize Valet's behavior](5-custom-transformers.md) diff --git a/jenkins/4-custom-transformers.md b/jenkins/5-custom-transformers.md similarity index 60% rename from jenkins/4-custom-transformers.md rename to jenkins/5-custom-transformers.md index b9d28ba..913fab4 100644 --- a/jenkins/4-custom-transformers.md +++ b/jenkins/5-custom-transformers.md @@ -1,6 +1,6 @@ # Use custom transformers to customize Valet's behavior -In this lab we will build upon the `dry-run` command to override Valet's default behavior and customize the converted workflow using "custom transformers". Custom transformers can be used to: +In this lab you will build upon the `dry-run` command to override Valet's default behavior and customize the converted workflow using "custom transformers". Custom transformers can be used to: 1. Convert items that are not automatically converted. 2. Convert items that were automatically converted using different actions. @@ -9,16 +9,16 @@ In this lab we will build upon the `dry-run` command to override Valet's default ## Prerequisites -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a Jenkins server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). -3. Completed the [dry-run lab](./3-dry-run.md). +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a Jenkins server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [dry-run lab](./4-dry-run.md). ## Perform a dry-run -We will be performing a `dry-run` command to inspect the workflow that is converted by default. Run the following command within the codespace terminal: +You will be performing a `dry-run` command to inspect the workflow that is converted by default. Run the following command within the codespace terminal: ```bash -gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline -o tmp/jenkins/dry-run +gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline --output-dir tmp/dry-run ``` The converted workflow that is generated by the above command can be seen below: @@ -71,17 +71,17 @@ jobs: -_Note_: You can refer to the previous [lab](./3-dry-run.md) to learn about the fundamentals of the `dry-run` command. +_Note_: You can refer to the previous [lab](./4-dry-run.md) to learn about the fundamentals of the `dry-run` command. ## Custom transformers for an unknown step -The converted workflow above contains a `sleep` step was not automatically converted. We will need to answer the following questions before writing a custom transformer: +The converted workflow above contains a `sleep` step was not automatically converted. Answer the following questions before writing a custom transformer: 1. What is the "identifier" of the step to customize? - __sleep__. The identifier will be the key of a key value pair within the step of a Jenkinsfile. 2. What is the desired Actions syntax to use instead? - - After some research, we have determined that the following bash script will provide similar functionality: + - After some research, you have determined that the following bash script will provide similar functionality: ```yaml - name: Sleep for 80 seconds @@ -89,13 +89,13 @@ The converted workflow above contains a `sleep` step was not automatically conve shell: bash ``` -Now we can begin to write the custom transformer. Customer transformers use a DSL built on top of Ruby and should be defined in a file with the `.rb` file extension. You can create this file by running the following command in your codespace terminal: +Now you can begin to write the custom transformer. Custom transformers use a DSL built on top of Ruby and should be defined in a file with the `.rb` file extension. You can create this file by running the following command in your codespace terminal: ```bash code transformers.rb ``` -Next, we will define a `transform` method for the `sleep` identifier by adding the following code to `transformers.rb`: +Next, you will define a `transform` method for the `sleep` identifier by adding the following code to `transformers.rb`: ```ruby transform "sleep" do |item| @@ -111,13 +111,13 @@ end This method can use any valid ruby syntax and should return a `Hash` that represents the YAML that should be generated for a given step. Valet will use this method to convert a step with the provided identifier and will use the `item` parameter for the original values configured in Jenkins. -Now, we can perform another `dry-run` command and use the `--custom-transformers` CLI option to provide this custom transformer. Run the following command within your codespace terminal: +Now you can perform another `dry-run` command and use the `--custom-transformers` CLI option to provide this custom transformer. Run the following command within your codespace terminal: ```bash -gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline -o tmp/jenkins/dry-run --custom-transformers transformers.rb +gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline --output-dir tmp/dry-run --custom-transformers transformers.rb ``` -Open the workflow that is generated and inspect the contents. Now, the `sleep` step is converted and uses the customized behavior! +Open the workflow that is generated and inspect the contents. Now the `sleep` step is converted and uses the customized behavior! ```diff - # # This item has no matching transformer @@ -133,13 +133,13 @@ Open the workflow that is generated and inspect the contents. Now, the `sleep` s ## Custom transformers for a known step -We can also override Valet's default behavior. In this scenario, we may not desire to use the third-party action for publishing junit test results that is used by default. Again, we will need to answer the following questions before writing a custom transformer: +You can also override Valet's default behavior. In this scenario, you may not want to use the third-party action for publishing junit test results that is used by default. Again, answer the following questions before writing a custom transformer: 1. What is the "identifier" of the step to customize? - __junit__ 2. What is the desired Actions syntax to use instead? - - After some research, we have determined that the uploading test results as an artifact will be suitable: + - After some research, you have determined that uploading test results as an artifact will be suitable: ```yaml - uses: actions/upload-artifact@v3 @@ -148,9 +148,9 @@ We can also override Valet's default behavior. In this scenario, we may not desi path: path/to/artifact/world.txt ``` -We will build this custom transformer similar to the previous custom transformer, however, we first need to inspect the `item` keyword to programmatically use the file path to junit's test results in the `actions/upload-artifact@v3` step. +You will build this custom transformer similar to the previous custom transformer, however, you first need to inspect the `item` keyword to programmatically use the file path to junit's test results in the `actions/upload-artifact@v3` step. -To do this, we will print `item` to the console. You can achieve this by adding the following custom transformer to `transformers.rb`: +To do this, you will print `item` to the console. You can achieve this by adding the following custom transformer to `transformers.rb`: ```ruby transform "junit" do |item| @@ -160,9 +160,15 @@ end Now, we can perform another `dry-run` command with the `--custom-transformers` CLI option. The output of the `dry-run` command should look similar to this: -![img](https://user-images.githubusercontent.com/19557880/186782050-65ece0c4-52a3-4f88-818f-0f860c50c2b7.png) +```console +$ gh valet dry-run jenkins --source-url http://localhost:8080/job/test_pipeline --output-dir tmp/dry-run --custom-transformers transformers.rb +[2022-08-20 22:08:20] Logs: 'tmp/dry-run/log/valet-20220916-022628.log' +This is the item: {"name"=>"junit", "arguments"=>[{"key"=>"testResults", "value"=>{"isLiteral"=>true, "value"=>"**/target/*.xml"}}]} +[2022-08-20 22:08:20] Output file(s): +[2022-08-20 22:08:20] tmp/dry-run/test_pipeline.yml +``` -Now that we know the data structure of `item` we can access the file path programmatically by editing the custom transformer to following: +Now that you know the data structure of `item`, you can access the file path programmatically by editing the custom transformer to the following: ```ruby transform "junit" do |item| @@ -181,7 +187,7 @@ end _Note_: `transformers.rb` should contain a `transform` method for both `sleep` and `junit`. -Now, we can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `EnricoMi/publish-unit-test-result-action@v1.7` action will have been replaced with the customized steps. +Now you can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `EnricoMi/publish-unit-test-result-action@v1.7` action will have been replaced with the customized steps. ```diff - - name: Publish test results @@ -197,9 +203,9 @@ Now, we can perform another `dry-run` command with the `--custom-transformers` C ## Custom transformers for environment variables -We can also use custom transformers to edit the values of environment variables in converted workflows. In our example, we will be updating the `DB_ENGINE` environment variable to be `mongodb` instead of `sqlite`. +You can also use custom transformers to edit the values of environment variables in converted workflows. In this example, you will be updating the `DB_ENGINE` environment variable to be `mongodb` instead of `sqlite`. -To do this, add the following code to the `transformers.rb` file. +To do this, add the following code at the top of the `transformers.rb` file. ```ruby env "DB_ENGINE", "mongodb" @@ -207,7 +213,7 @@ env "DB_ENGINE", "mongodb" In this example, the first parameter to the `env` method is the environment variable name and the second is the updated value. -Now, we can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `DB_ENGINE` environment variable will be set to `mongodb`: +Now you can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `DB_ENGINE` environment variable will be set to `mongodb`: ```diff env: @@ -218,7 +224,7 @@ env: ## Custom transformers for runners -Finally, we can use custom transformers to dictate which runners converted workflows should use. To do this we will need to answer the following questions: +Finally, you can use custom transformers to dictate which runners the converted workflows should use. To do this, answer the following questions: 1. What is label of the runner in Jenkins to update? - __TeamARunner__ @@ -226,7 +232,7 @@ Finally, we can use custom transformers to dictate which runners converted workf 2. What is the label of the runner in Actions to use instead? - __ubuntu-latest__ -With these questions answered, we can add the following code to the `transformers.rb` file: +With these questions answered, you can add the following code to the `transformers.rb` file: ```ruby runner "TeamARunner", "ubuntu-latest" @@ -234,7 +240,7 @@ runner "TeamARunner", "ubuntu-latest" In this example, the first parameter to the `runner` method is the Jenkins label and the second is the Actions runner label. -Now, we can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `runs-on` statement will use the customized runner label: +Now you can perform another `dry-run` command with the `--custom-transformers` CLI option. When you open the converted workflow the `runs-on` statement will use the customized runner label: ```diff runs-on: @@ -243,7 +249,7 @@ runs-on: + - ubuntu-latest ``` -At this point of the lab the file contents of `transformers.rb` should match this: +At this point the file contents of `transformers.rb` should match this:
Custom transformers 👇 @@ -281,7 +287,7 @@ At this point of the lab the file contents of `transformers.rb` should match thi
-Thats it! Congratulations you have overridden Valet's default behavior by customizing the conversion of: +That's it! Congratulations, you have overridden Valet's default behavior by customizing the conversion of: - Unknown steps - Known steps @@ -290,4 +296,4 @@ Thats it! Congratulations you have overridden Valet's default behavior by custom ## Next lab -[Perform a production migration of a Jenkins pipeline](5-migrate.md) +[Perform a production migration of a Jenkins pipeline](6-migrate.md) diff --git a/jenkins/5-migrate.md b/jenkins/5-migrate.md deleted file mode 100644 index a8ebba2..0000000 --- a/jenkins/5-migrate.md +++ /dev/null @@ -1,55 +0,0 @@ -# Perform a production migration of a Jenkins pipeline - -In this lab, you will use the `migrate` command to convert a Jenkins pipeline and open a pull request with the equivalent Actions workflow. - -## Prerequisites - -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a Jenkins server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). -3. Completed the [dry-run lab](./3-dry-run.md). -4. Completed the [custom transformers lab](./4-custom-transformers.md). - -## Performing a migration - -We need to answer the following questions before running a `migrate` command: - -1. What is the source URL of the pipeline we want to convert? - - ____ -2. Where do we want to store the result? - - __./tmp/migrate__ -3. What is the URL for the GitHub repository to add the workflow to? - - __this repository__. The URL should should follow the pattern with `:owner` and `:repo` replaced with your values. - -### Steps - -1. Run the following `migrate` command in the codespace terminal: - - ```bash - gh valet migrate jenkins --target-urlhttps://github.com/:owner/:repo --output-dir ./tmp/migrate --source-url http://localhost:8080/job/monas_dev_work/job/monas_freestyle - ``` - -2. The command will write the URL to the pull request that was created when the command succeeds. - - ![img](https://user-images.githubusercontent.com/19557880/185509412-ab64d92d-2a56-4d5a-bbb4-35a41a2ca48c.png) - -3. Open the generated pull request in a new browser tab. - -### Inspect the pull request - -The first thing we should notice about the PR is that there is a list of manual steps for us to complete: - -![img](https://user-images.githubusercontent.com/19557880/186784161-b7882ac4-ac99-4462-b69f-f49b9202527b.png) - -Next, let's review the workflow we are adding by clicking on `Files changed` tab. This is where you would double check everything looks good. If it didn't you could push commits with the required changes, prior to merging. - -Next, you can inspect the "Files changed" in this PR and see the converted workflow that is being added. Any additional changes or code reviews that were needed should be done in this PR. - -Finally, you can merge the PR once your review has completed. We can then view the workflow running by selecting the "Actions" menu in the top navigation bar in GitHub. - -![img](https://user-images.githubusercontent.com/19557880/185509704-90243ec5-e77f-4baf-a9b2-d9a4d9fda199.png) - -At this point, the migration has completed and you have successfully migrated a Jenkins pipeline to Actions! - -### Next lab - -[Forecast potential build runner usage](6-forecast.md) diff --git a/jenkins/6-forecast.md b/jenkins/6-forecast.md deleted file mode 100644 index deeb6b0..0000000 --- a/jenkins/6-forecast.md +++ /dev/null @@ -1,104 +0,0 @@ -# Forecast potential build runner usage - -In this lab you will use the `forecast` command to forecast potential GitHub Actions usage by computing metrics from completed pipeline runs in your Jenkins server. - -## Prerequisites - -1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your Codespace environment and start a Jenkins server. -2. Completed the [configure lab](./1-configure-lab.md#configuring-credentials). - -## Perform a forecast - -We will need to answer the following questions before running the `forecast` command: - -1. Do we want to forecast the entire Jenkins server or a single folder? - - We will be forecasting the entire Jenkins server but we could optionally limit this by using the `-f ` CLI option. - -2. What is the date we want to start forecasting from? - - __2022-08-02__. This date is needed as it is prior to when the data was seeded in Jenkins for these labs. This value defaults to the date one week ago, however, you should use a start date that will show a representative view of typical usage. - -3. Where do we want to store the results? - - `./tmp/forecast_reports` - -### Steps - -1. Navigate to the codespace terminal -2. Run the following command from the root directory: - - ```bash - gh valet forecast jenkins --output-dir ./tmp/forecast_reports --start-date 2022-08-02 - ``` - -3. The command will list all the files written to disk when the command succeeds. - - ![img](https://user-images.githubusercontent.com/19557880/186223037-18556c82-5a29-4434-bc17-4b906d704967.png) - -## Review the forecast report - -The forecast report, logs, and completed job data will be located within the `tmp/forecast_reports` folder. - -1. Find the `forecast_report.md` file in the file explorer. -2. Right-click the `forecast_report.md` file and select `Open Preview`. -3. This file contains metrics used to forecast potential GitHub Actions usage. - -### Total - -The "Total" section of the forecast report contains high level statistics related to all the jobs completed after the `--start-date` CLI option: - -```md -- Job count: __73__ -- Pipeline count: __6__ - -- Execution time - - - Total: __27,057 minutes__ - - Median: __2 minutes__ - - P90: __19 minutes__ - - Min: __0 minutes__ - - Max: __15,625 minutes__ - -- Queue time - - - Median: __0 minutes__ - - P90: __0 minutes__ - - Min: __0 minutes__ - - Max: __0 minutes__ - -- Concurrent jobs - - - Median: __1__ - - P90: __3__ - - Min: __0__ - - Max: __29__ -``` - -Here are some key terms of items defined in the forecast report: - -- The `job count` is the total number of completed jobs. -- The `pipeline count` is the number of unique pipelines used. -- `Execution time` describes the amount of time a runner spent on a job. This metric can be used to help plan for the cost of GitHub hosted runners. - - This metric is correlated to how much you should expect to spend in GitHub Actions. This will vary depending on the hardware used for these minutes and the [Actions pricing calculator](https://github.com/pricing/calculator) should be used to estimate a dollar amount. -- `Queue time` metrics describe the amount of time a job spent waiting for a runner to be available to execute it. -- `Concurrent jobs` metrics describe the amount of jobs running at any given time. This metric can be used to define the number of runners a customer should configure. - -Additionally, these metrics are defined for each queue of runners defined in Jenkins. This is especially useful if there are a mix of hosted/self-hosted runners or high/low spec machines to see metrics specific to different types of runners. - -## Forecasting multiple providers - -We can examine the available options for the `forecast` command by running `gh valet forecast --help`. When you do this you will see the `--source-file-path` option: - -![img](https://user-images.githubusercontent.com/19557880/186263140-f02c6cab-7979-417c-bdfe-b9590e9c5597.png) - -The `--source-file-path` CLI option can be used to combine data from multiple reports into a single report. This becomes useful if you use multiple CI/CD providers and wanted to get a holistic view of the runner usage. This works by using the `.json` files generated by `forecast` commands as space-delimited values for the `--source-file-path` CLI option. Optionally, this value could be a glob pattern to dynamically specify the list of files (e.g. `**/*.json`). - -Run the following command from within the codespace terminal: - -```bash -gh valet forecast --source-file-path tmp/**/jobs/*.json -o tmp/combined-forecast -``` - -You can now inspect the output of the command to see a forecast report using all of the files matching the `tmp/**/jobs/*.json` pattern. - -## Next steps - -This concludes all labs for migrating Jenkins pipelines to Actions with Valet! \ No newline at end of file diff --git a/jenkins/6-migrate.md b/jenkins/6-migrate.md new file mode 100644 index 0000000..773b929 --- /dev/null +++ b/jenkins/6-migrate.md @@ -0,0 +1,59 @@ +# Perform a production migration of a Jenkins pipeline + +In this lab, you will use the `migrate` command to convert a Jenkins pipeline and open a pull request with the equivalent Actions workflow. + +## Prerequisites + +1. Followed the steps [here](./readme.md#configure-your-codespace) to set up your GitHub Codespaces environment and start a Jenkins server. +2. Completed the [configure lab](./1-configure.md#configuring-credentials). +3. Completed the [dry-run lab](./4-dry-run.md). +4. Completed the [custom transformers lab](./5-custom-transformers.md). + +## Performing a migration + +Answer the following questions before running a `migrate` command: + +1. What is the source URL of the pipeline you want to convert? + - ____ +2. Where do you want to store the logs? + - __tmp/migrate__ +3. What is the URL for the GitHub repository to add the workflow to? + - __this repository__. The URL should follow the pattern with `:owner` and `:repo` replaced with your values. + +### Steps + +1. Run the following `migrate` command in your codespace terminal: + + ```bash + gh valet migrate jenkins --target-url https://github.com/:owner/:repo --output-dir tmp/migrate --source-url http://localhost:8080/job/monas_dev_work/job/monas_freestyle + ``` + +2. The command will write the URL to the pull request that was created when the command succeeds. + + ```console + $ gh valet migrate jenkins --target-url https://github.com/:owner/:repo --output-dir tmp/migrate --source-url http://localhost:8080/job/monas_dev_work/job/monas_freestyle + [2022-08-20 22:08:20] Logs: 'tmp/migrate/log/valet-20220916-014033.log' + [2022-08-20 22:08:20] Pull request: 'https://github.com/:owner/:repo/pull/1' + ``` + +3. Open the generated pull request in a new browser tab. + +### Inspect the pull request + +The first thing we should notice about the pull request is that there is a list of manual steps to complete: + +![img](https://user-images.githubusercontent.com/19557880/186784161-b7882ac4-ac99-4462-b69f-f49b9202527b.png) + +Next, review the workflow you are adding by clicking on the `Files changed` tab. This is where you double check that everything looks good. If it didn't, you could push commits with the required changes, prior to merging. + +Next, you can inspect the "Files changed" in this pull request and see the converted workflow that is being added. Any additional changes or code reviews that were needed should be done in this pull request. + +Finally, you can merge the pull request once your review has completed. You can then view the workflow running by selecting the "Actions" menu in the top navigation bar in GitHub. + +![img](https://user-images.githubusercontent.com/19557880/185509704-90243ec5-e77f-4baf-a9b2-d9a4d9fda199.png) + +At this point, the migration has completed and you have successfully migrated a Jenkins pipeline to Actions! + +### Next lab + +This concludes all labs for migrating Jenkins pipelines to Actions with Valet! diff --git a/jenkins/bootstrap/setup.sh b/jenkins/bootstrap/setup.sh index 5947fa4..e843745 100755 --- a/jenkins/bootstrap/setup.sh +++ b/jenkins/bootstrap/setup.sh @@ -18,10 +18,6 @@ else docker run -d --name jenkins -p 8080:8080 --env JENKINS_ADMIN_ID=$username --env JENKINS_ADMIN_PASSWORD=$password jenkins:$container_name fi -# allow valet to talk to Jenkins by removing network isolation between containers -export DOCKER_ARGS="--network=host" -grep -q "export DOCKER_ARGS=" ~/.bashrc || echo 'export DOCKER_ARGS="--network=host"' >> ~/.bashrc - echo -e "\nWaiting for Jenkins to start..." while ! curl -s http://localhost:8080/ > /dev/null; do printf "." diff --git a/jenkins/readme.md b/jenkins/readme.md index 0466eb9..aa4ab40 100644 --- a/jenkins/readme.md +++ b/jenkins/readme.md @@ -1,6 +1,6 @@ # Jenkins to Actions migrations powered by Valet -The instructions below will guide you through configuring a GitHub Codespace environment that will be used in subsequent labs that demonstrate how to use Valet to migrate Jenkins pipelines to GitHub Actions. +These instructions will guide you through configuring a GitHub Codespaces environment that will be used in subsequent labs that demonstrate how to use Valet to migrate Jenkins pipelines to GitHub Actions. These steps **must** be completed prior to starting other labs. @@ -8,52 +8,54 @@ These steps **must** be completed prior to starting other labs. 1. Ensure that you have created a repository using the [valet-customers/labs](https://github.com/valet-customers/labs) as a template. -## Configure your Codespace +## Configure your codespace -1. Start a new Codespace. +1. Start a new codespace. -- Click the `Code` with button down arrow above repository on the repository's landing page. -- Click the `Codespaces` tab -- Click `Create codespaces on main` to create the codespace. -- After the Codespace has initialized there will be a terminal present. + - Click the `Code` button on your repository's landing page. + - Click the `Codespaces` tab. + - Click `Create codespaces on main` to create the codespace. + - After the codespace has initialized there will be a terminal present. 2. Verify the Valet CLI is installed and working. More information on the Valet extension for the official GitHub CLI can be found [here](https://github.com/github/gh-valet). -- Run the following command in the codespace's terminal: + - Run the following command in the codespace's terminal: - ```bash - gh valet version - ``` + ```bash + gh valet version + ``` -- Verify the output is similar to below. - - ```bash - gh version 2.14.3 (2022-07-26) - gh valet github/gh-valet v0.1.12 - valet-cli unknown - ``` + - Verify the output is similar to below. - - If `gh valet version` did not produce similar output then please follow the troubleshooting [guide](#troubleshoot-the-valet-cli). + ```console + $ gh valet version + gh version 2.14.3 (2022-07-26) + gh valet github/gh-valet v0.1.12 + valet-cli unknown + ``` + + - If `gh valet version` did not produce similar output, refer to the troubleshooting [guide](#troubleshoot-the-valet-cli). ## Bootstrap a Jenkins server - 1. Execute the Jenkins setup script that will start a container with a Jenkins server running inside of it. This script should be executed when starting a new Codespace or restarting an existing one. +1. Execute the Jenkins setup script that will start a container with a Jenkins server running inside of it. This script should be executed when starting a new codespace or restarting an existing one. -- Run the following command from the codespace's terminal to start a Jenkins server: + - Run the following command from the codespace's terminal to start a Jenkins server: - ```bash - ./jenkins/bootstrap/setup.sh - ``` + ```bash + ./jenkins/bootstrap/setup.sh + ``` -- After some time, a pop-up box should appear with a link to the URL for your Jenkins server. - - You can also access the URL by going to the `Ports` tab in your terminal. Right click on the URL listed under the `Local Address` and click the `Open in Browser` tab. + - After some time, a pop-up box should appear with a link to the URL for your Jenkins server. + + - You can also access the URL by going to the `Ports` tab in your terminal. Right-click the URL listed under the `Local Address` and click the `Open in Browser` tab. 2. Open the Jenkins server in your browser and use the following credentials to authenticate: - - Username: `admin` - - Password: `password` + - Username: `admin` + - Password: `password` -- Once authenticated, you should see a Jenkins server with a few predefined pipelines. +3. Once authenticated, you should see a Jenkins server with a few predefined pipelines. ## Labs for Jenkins @@ -61,17 +63,17 @@ Perform the following labs to learn more about Actions migrations with Valet: 1. [Configure credentials for Valet](1-configure.md) 2. [Perform an audit of a Jenkins server](2-audit.md) -3. [Perform a dry-run of a Jenkins pipeline](3-dry-run.md) -4. [Use custom transformers to customize Valet's behavior](4-custom-transformers.md) -5. [Perform a production migration of a Jenkins pipeline](5-migrate.md) -6. [Forecast potential build runner usage](6-forecast.md) +3. [Forecast potential build runner usage](3-forecast.md) +4. [Perform a dry-run migration of a Jenkins pipeline](4-dry-run.md) +5. [Use custom transformers to customize Valet's behavior](5-custom-transformers.md) +6. [Perform a production migration of a Jenkins pipeline](6-migrate.md) ## Troubleshoot the Valet CLI The CLI extension for Valet can be manually installed by following these steps: - Verify you are in the codespace terminal -- Run this command from within the codespace's terminal: +- Run this command from within the codespace terminal: ```bash gh extension install github/gh-valet @@ -79,14 +81,15 @@ The CLI extension for Valet can be manually installed by following these steps: - Verify the result of the install contains: - ```bash + ```console + $ gh extension install github/gh-valet âś“ Installed extension github/gh-valet ``` - If you get an error similar to the image below, then click the link in the terminal output to authorize the token. - Restart the codespace after clicking the link. ![img](https://user-images.githubusercontent.com/26442605/169588015-9414404f-82b6-4d0f-89d4-5f0e6941b029.png) -- Verify Valet CLI extension is installed and working by running the following command from the codespace's terminal: +- Verify Valet CLI extension is installed and working by running the following command from the codespace terminal: ```bash gh valet version @@ -96,10 +99,10 @@ The CLI extension for Valet can be manually installed by following these steps: Follow these steps if the Jenkins server does not start correctly after running the setup script: -1. Navigate to the `Docker` tab on the left side of the codespace. +1. On the left side of the codespace, navigate to the `Docker` tab. 2. Under the `Containers` tab you should see a docker container named `jenkins:valet` listed with a green play button â–¶ -- If you see the `jenkins:valet` container, but it has a red stopped symbol next to it â–˘, right click on the container and click on `start`, the container should begin running again. -- If the container does not start even after trying to manually start it, right click on the `jenkins:valet` container and click the `remove` button. Then, attempt to start the Jenkins server again by following the steps [here](#bootstrap-a-jenkins-server). +- If you see the `jenkins:valet` container, but it has a red stopped symbol next to it â–˘, right-click the container and click `start`. The container should begin running again. +- If the container does not start even after trying to start it manually, right-click the `jenkins:valet` container and click the `remove` button. Then, attempt to start the Jenkins server again by following the steps [here](#bootstrap-a-jenkins-server). ![img](https://user-images.githubusercontent.com/19557880/183770210-c0386616-656e-4fe9-9324-b410ad62c406.png) diff --git a/readme.md b/readme.md index 0317e10..f2c1371 100644 --- a/readme.md +++ b/readme.md @@ -1,15 +1,14 @@ # GitHub Actions migrations powered by Valet -Valet helps facilitate the migration of Azure DevOps, CircleCI, GitLab CI, Jenkins, and Travis CI pipelines to GitHub Actions. This repository contains learning paths that teach you how to use Valet and approach migrations to Actions. +Valet helps facilitate the migration of Azure DevOps, CircleCI, GitLab CI, Jenkins, and Travis CI pipelines to GitHub Actions. This repository contains learning paths that teach you how to use Valet and how to approach migrations to Actions. > Valet is currently private and you must be onboarded prior to using it. Please reach out to [GitHub Sales](https://github.com/enterprise/contact) to inquire about being granted access. To get started: 1. Use the `valet-customers/labs` repository as a template to [generate](https://github.com/valet-customers/labs/generate) a new GitHub repository. -2. Select which learning path to begin. There are currently learning paths for: - - [Azure DevOps to GitHub Actions migrations](/azure_devops/readme.md) - - [GitLab CI to GitHub Actions migrations](/gitlab/readme.md) - - [Jenkins to GitHub Actions migrations](/jenkins/readme.md) -3. Follow the instructions in each learning path to bootstrap a CI/CD environment. -4. Follow the instructions to complete each learning path. +2. Select which learning path to begin with. There are currently learning paths for: + - [Migrations from Azure DevOps to GitHub Actions](/azure_devops/readme.md) + - [Migrations from GitLab CI to GitHub Actions](/gitlab/readme.md) + - [Migrations from Jenkins to GitHub Actions](/jenkins/readme.md) +3. Each learning path describes how to configure your codespace, bootstrap a CI/CD environment, and troubleshoot the Valet CLI. diff --git a/valet/.env.local.template b/valet/.env.local.template deleted file mode 100644 index ce4ca6f..0000000 --- a/valet/.env.local.template +++ /dev/null @@ -1,7 +0,0 @@ -GITHUB_ACCESS_TOKEN= -GITHUB_INSTANCE_URL= - -AZURE_DEVOPS_PROJECT= -AZURE_DEVOPS_ORGANIZATION= -AZURE_DEVOPS_INSTANCE_URL= -AZURE_DEVOPS_ACCESS_TOKEN=