feature: initial commit for all new image changes

2024-11-13 17:16:01 -07:00
parent de6aa9e70e
commit fcf3d0ac96
1042 changed files with 45086 additions and 1 deletions
@@ -0,0 +1,181 @@
+# Ideally we would use GitHub Actions for this, but since we use self-hosted machines to run image builds
+# we need the following features to use GitHub Actions for Images CI:
+# - https://github.community/t5/GitHub-Actions/Make-secrets-available-to-builds-of-forks/m-p/30678#M508
+# - https://github.community/t5/GitHub-Actions/GitHub-Actions-Manual-Trigger-Approvals/td-p/31504
+# - https://github.community/t5/GitHub-Actions/Protecting-github-workflows/td-p/30290
+
+parameters:
+  - name: job_id
+    type: string
+    default: 'generate_image'
+
+  - name: image_type
+    type: string
+
+  - name: image_template_name
+    type: string
+
+  - name: image_readme_name
+    type: string
+
+  - name: agent_pool
+    type: object
+    default:
+      name: 'ci-agent-pool'
+
+  - name: variable_group_name
+    type: string
+    default: 'Image Generation Variables'
+
+  - name: create_release
+    type: boolean
+    default: true
+
+  - name: repository_ref
+    type: string
+    default: 'self'
+
+jobs:
+- job: ${{ parameters.job_id }}
+  displayName: Image Generation (${{ parameters.image_type }})
+  timeoutInMinutes: 600
+  cancelTimeoutInMinutes: 30
+  pool: ${{ parameters.agent_pool }}
+  variables:
+  - group: ${{ parameters.variable_group_name }}
+
+  steps:
+  - checkout: ${{ parameters.repository_ref }}
+    clean: true
+    fetchDepth: 0
+    fetchTags: false
+
+  - task: PowerShell@2
+    displayName: 'Download custom repository'
+    condition: and(ne(variables['CUSTOM_REPOSITORY_URL'], ''), ne(variables['CUSTOM_REPOSITORY_BRANCH'], ''))
+    inputs:
+      targetType: 'filePath'
+      filePath: ./images.CI/download-repo.ps1
+      arguments: -RepoUrl $(CUSTOM_REPOSITORY_URL) `
+                 -RepoBranch $(CUSTOM_REPOSITORY_BRANCH)
+
+  - task: AzureCLI@2
+    displayName: 'Set variables'
+    inputs:
+      azureSubscription: 'spn-hosted-runners'
+      scriptType: 'pscore'
+      scriptLocation: 'inlineScript'
+      inlineScript: |
+        $ImageType = "${{ parameters.image_type }}"
+        $TemplateDirectoryName = if ($ImageType.StartsWith("ubuntu")) { "ubuntu/templates" } else { "windows/templates" }
+        $TemplateDirectoryPath = Join-Path "images" $TemplateDirectoryName | Resolve-Path
+
+        $TemplateFileName = "${{ parameters.image_template_name }}"
+        $TemplatePath = Join-Path $TemplateDirectoryPath $TemplateFileName
+        Write-Host "##vso[task.setvariable variable=TemplateDirectoryPath;]$TemplateDirectoryPath"
+        Write-Host "##vso[task.setvariable variable=TemplatePath;]$TemplatePath"
+
+        $ManagedImageName = "${{ parameters.image_type }}-$(Build.BuildId)"
+        Write-Host "##vso[task.setvariable variable=ManagedImageName;]$ManagedImageName"
+
+        $TempResourceGroupName = "packer-temp-$ManagedImageName"
+        Write-Host "##vso[task.setvariable variable=TempResourceGroupName;]$TempResourceGroupName"
+
+        $clientSecret = $(az keyvault secret show --name "spnhostedrunners" --vault-name "gh-imagegeneration" --query value -o tsv)
+        Write-Host "##vso[task.setvariable variable=ClientSecret;issecret=true]$clientSecret"
+
+  - task: PowerShell@2
+    displayName: 'Build VM'
+    inputs:
+      targetType: filePath
+      filePath: ./images.CI/linux-and-win/build-image.ps1
+      arguments: -ClientId $(CLIENT_ID) `
+                 -ClientSecret "$(ClientSecret)" `
+                 -TemplatePath $(TemplatePath) `
+                 -ImageName "$(ManagedImageName)" `
+                 -ImageResourceGroupName $(AZURE_RESOURCE_GROUP) `
+                 -TempResourceGroupName "$(TempResourceGroupName)" `
+                 -SubscriptionId $(AZURE_SUBSCRIPTION) `
+                 -TenantId $(AZURE_TENANT) `
+                 -Location $(AZURE_LOCATION) `
+                 -VirtualNetworkName $(BUILD_AGENT_VNET_NAME) `
+                 -VirtualNetworkRG $(BUILD_AGENT_VNET_RESOURCE_GROUP) `
+                 -VirtualNetworkSubnet $(BUILD_AGENT_SUBNET_NAME)
+
+    env:
+      PACKER_LOG: 1
+      PACKER_LOG_PATH: "$(Agent.TempDirectory)/packer-log.txt"
+
+  - task: PowerShell@2
+    displayName: 'Copy image artifacts to the separate directory'
+    inputs:
+      targetType: 'inline'
+      script: |
+        $ImageType = "${{ parameters.image_type }}"
+        $rootDirectoryName = if ($ImageType.StartsWith("ubuntu")) { "ubuntu" } else { "windows" }
+        $rootDirectoryPath = Join-Path "images" $rootDirectoryName | Resolve-Path
+
+        $readmePath = Join-Path $rootDirectoryPath "${{ parameters.image_readme_name }}"
+        $softwareReportPath = Join-Path $rootDirectoryPath "software-report.json"
+
+        Copy-Item -Path $readmePath -Destination "$(Build.ArtifactStagingDirectory)/"
+        if (Test-Path $softwareReportPath) {
+            Copy-Item -Path $softwareReportPath -Destination "$(Build.ArtifactStagingDirectory)/"
+        }
+
+  - task: PowerShell@2
+    displayName: 'Print markdown software report'
+    inputs:
+      targetType: 'inline'
+      script: |
+        Get-Content -Path "$(Build.ArtifactStagingDirectory)/${{ parameters.image_readme_name }}"
+
+  - task: PowerShell@2
+    displayName: 'Print json software report'
+    inputs:
+      targetType: 'inline'
+      script: |
+        $softwareReportPath = "$(Build.ArtifactStagingDirectory)/software-report.json"
+        if (Test-Path $softwareReportPath) {
+            Get-Content -Path $softwareReportPath
+        }
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      ArtifactName: 'Built_VM_Artifacts'
+    displayName: Publish Artifacts
+
+  - task: PowerShell@2
+    displayName: 'Print provisioners duration'
+    inputs:
+      targetType: 'filePath'
+      filePath: ./images.CI/measure-provisioners-duration.ps1
+      arguments: -PackerLogPath "$(Agent.TempDirectory)/packer-log.txt" `
+                        -PrefixToPathTrim "$(TemplateDirectoryPath)" `
+                        -PrintTopNLongest 25
+
+  - ${{ if eq(parameters.create_release, true) }}:
+    - task: PowerShell@2
+      displayName: 'Create release for VM deployment'
+      inputs:
+        targetType: filePath
+        filePath: ./images.CI/linux-and-win/create-release.ps1
+        arguments: -BuildId $(Build.BuildId) `
+                   -Organization $(RELEASE_TARGET_ORGANIZATION) `
+                   -DefinitionId $(RELEASE_TARGET_DEFINITION_ID) `
+                   -Project $(RELEASE_TARGET_PROJECT) `
+                   -ImageType "${{ parameters.image_type }}" `
+                   -ManagedImageName "$(ManagedImageName)" `
+                   -AccessToken $(RELEASE_TARGET_TOKEN)
+
+  - task: PowerShell@2
+    displayName: 'Clean up resources'
+    condition: always()
+    inputs:
+      targetType: filePath
+      filePath: ./images.CI/linux-and-win/cleanup.ps1
+      arguments: -TempResourceGroupName "$(TempResourceGroupName)" `
+                 -SubscriptionId $(AZURE_SUBSCRIPTION) `
+                 -ClientId $(CLIENT_ID) `
+                 -ClientSecret "$(ClientSecret)" `
+                 -TenantId $(AZURE_TENANT)
@@ -0,0 +1,21 @@
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_type: ubuntu2004
+    image_readme_name: Ubuntu2004-Readme.md
+    image_template_name: ubuntu-20.04.pkr.hcl
@@ -0,0 +1,21 @@
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_type: ubuntu2204
+    image_readme_name: Ubuntu2204-Readme.md
+    image_template_name: ubuntu-22.04.pkr.hcl
@@ -0,0 +1,21 @@
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_type: ubuntu2404
+    image_readme_name: Ubuntu2404-Readme.md
+    image_template_name: ubuntu-24.04.pkr.hcl
@@ -0,0 +1,21 @@
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_type: windows2019
+    image_readme_name: Windows2019-Readme.md
+    image_template_name: windows-2019.pkr.hcl
@@ -0,0 +1,21 @@
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily
+  branches:
+    include:
+    - main
+  always: true
+
+trigger: none
+pr:
+  autoCancel: true
+  branches:
+    include:
+    - main
+
+jobs:
+- template: image-generation.yml
+  parameters:
+    image_type: windows2022
+    image_readme_name: Windows2022-Readme.md
+    image_template_name: windows-2022.pkr.hcl
@@ -0,0 +1,66 @@
+param(
+    [String] [Parameter (Mandatory=$true)] $TemplatePath,
+    [String] [Parameter (Mandatory=$true)] $ClientId,
+    [String] [Parameter (Mandatory=$true)] $ClientSecret,
+    [String] [Parameter (Mandatory=$true)] $Location,
+    [String] [Parameter (Mandatory=$true)] $ImageName,
+    [String] [Parameter (Mandatory=$true)] $ImageResourceGroupName,
+    [String] [Parameter (Mandatory=$true)] $TempResourceGroupName,
+    [String] [Parameter (Mandatory=$true)] $SubscriptionId,
+    [String] [Parameter (Mandatory=$true)] $TenantId,
+    [String] [Parameter (Mandatory=$false)] $VirtualNetworkName,
+    [String] [Parameter (Mandatory=$false)] $VirtualNetworkRG,
+    [String] [Parameter (Mandatory=$false)] $VirtualNetworkSubnet,
+    [String] [Parameter (Mandatory=$false)] $AllowedInboundIpAddresses = "[]"
+)
+
+if (-not (Test-Path $TemplatePath))
+{
+    Write-Error "'-TemplatePath' parameter is not valid. You have to specify correct Template Path"
+    exit 1
+}
+
+$ImageTemplateName = [io.path]::GetFileName($TemplatePath).Split(".")[0]
+$InstallPassword = [System.GUID]::NewGuid().ToString().ToUpper()
+
+$SensitiveData = @(
+    'OSType',
+    'StorageAccountLocation',
+    'OSDiskUri',
+    'OSDiskUriReadOnlySas',
+    'TemplateUri',
+    'TemplateUriReadOnlySas',
+    ':  ->'
+)
+
+Write-Host "Show Packer Version"
+packer --version
+
+Write-Host "Download packer plugins"
+packer init $TemplatePath
+
+Write-Host "Validate packer template"
+packer validate -syntax-only $TemplatePath
+
+Write-Host "Build $ImageTemplateName VM"
+packer build    -var "client_id=$ClientId" `
+                -var "client_secret=$ClientSecret" `
+                -var "install_password=$InstallPassword" `
+                -var "location=$Location" `
+                -var "managed_image_name=$ImageName" `
+                -var "managed_image_resource_group_name=$ImageResourceGroupName" `
+                -var "subscription_id=$SubscriptionId" `
+                -var "temp_resource_group_name=$TempResourceGroupName" `
+                -var "tenant_id=$TenantId" `
+                -var "virtual_network_name=$VirtualNetworkName" `
+                -var "virtual_network_resource_group_name=$VirtualNetworkRG" `
+                -var "virtual_network_subnet_name=$VirtualNetworkSubnet" `
+                -var "allowed_inbound_ip_addresses=$($AllowedInboundIpAddresses)" `
+                -color=false `
+                $TemplatePath `
+        | Where-Object {
+            #Filter sensitive data from Packer logs
+            $currentString = $_
+            $sensitiveString = $SensitiveData | Where-Object { $currentString -match $_ }
+            $sensitiveString -eq $null
+        }
@@ -0,0 +1,19 @@
+param(
+    [Parameter (Mandatory=$true)] [string] $TempResourceGroupName,
+    [Parameter (Mandatory=$true)] [string] $SubscriptionId,
+    [Parameter (Mandatory=$true)] [string] $ClientId,
+    [Parameter (Mandatory=$true)] [string] $ClientSecret,
+    [Parameter (Mandatory=$true)] [string] $TenantId
+)
+
+az login --service-principal --username $ClientId --password=$ClientSecret --tenant $TenantId | Out-Null
+az account set --subscription $SubscriptionId | Out-Null
+
+$groupExist = az group exists --name $TempResourceGroupName
+if ($groupExist -eq "true") {
+    Write-Host "Found a match, deleting temporary files"
+    az group delete --name $TempResourceGroupName --yes | Out-Null
+    Write-Host "Temporary group was deleted successfully"
+} else {
+    Write-Host "No temporary groups found"
+}
@@ -0,0 +1,36 @@
+param(
+    [Parameter (Mandatory)] [UInt32] $BuildId,
+    [Parameter (Mandatory)] [string] $Organization,
+    [Parameter (Mandatory)] [string] $Project,
+    [Parameter (Mandatory)] [string] $ImageType,
+    [Parameter (Mandatory)] [string] $ManagedImageName,
+    [Parameter (Mandatory)] [string] $DefinitionId,
+    [Parameter (Mandatory)] [string] $AccessToken
+)
+
+$Body = @{
+    definitionId = $DefinitionId
+    variables = @{
+      ImageBuildId = @{
+        value = $BuildId
+      }
+      ImageType = @{
+        value = $ImageType
+      }
+      ManagedImageName = @{
+        value = $ManagedImageName
+      }
+    }
+    isDraft = "false"
+} | ConvertTo-Json -Depth 3
+
+$URL = "https://vsrm.dev.azure.com/$Organization/$Project/_apis/release/releases?api-version=5.1"
+$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}"))
+$headers = @{
+    Authorization = "Basic ${base64AuthInfo}"
+}
+
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -bor [Net.SecurityProtocolType]::Tls13
+$NewRelease = Invoke-RestMethod $URL -Body $Body -Method "POST" -Headers $headers -ContentType "application/json"
+
+Write-Host "Created release: $($NewRelease._links.web.href)"