Merge branch 'main' into refactor/formatting

help/CreateImageAndAzureResources.md

@@ -12,6 +12,7 @@ After successful image generation, a snapshot of the temporary VM will be conver
 - `packer` - Can be downloaded from https://www.packer.io/downloads
 - `PowerShell 5.0 or higher` or `PSCore` for linux distributes.
 - `Azure CLI ` - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest
+- `Azure Powershell module` - https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-4.6.1
 
 ### Azure DevOps self-hosted pool requirements
 To connect to a temporary VM packer use WinRM or SSH connections on public IP interfaces.
@@ -31,7 +32,7 @@ Download `packer` from https://www.packer.io/downloads, or install it via Chocol
 choco install packer
 ```
 
-Install Azure CLI - https://docs.microsoft.com/ru-ru/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli.
+Install Azure CLI - https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-windows?view=azure-cli-latest&tabs=azure-cli.
 ```
 Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi; Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet'; rm .\AzureCLI.msi
 ```

helpers/GenerateResourcesAndImage.ps1

@@ -110,12 +110,12 @@ Function GenerateResourcesAndImage {
     $ServicePrincipalClientSecret = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper();
     $InstallPassword = $env:UserName + [System.GUID]::NewGuid().ToString().ToUpper();
 
-    Login-AzureRmAccount
+    Connect-AzAccount
-    Set-AzureRmContext -SubscriptionId $SubscriptionId
+    Set-AzContext -SubscriptionId $SubscriptionId
 
     $alreadyExists = $true;
     try {
-        Get-AzureRmResourceGroup -Name $ResourceGroupName
+        Get-AzResourceGroup -Name $ResourceGroupName
         Write-Verbose "Resource group was found, will delete and recreate it."
     }
     catch {
@@ -126,8 +126,8 @@ Function GenerateResourcesAndImage {
     if ($alreadyExists) {
         if($Force -eq $true) {
             # Cleanup the resource group if it already exitsted before
-            Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force
+            Remove-AzResourceGroup -Name $ResourceGroupName -Force
-            New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation
+            New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation
         } else {
             $title = "Delete Resource Group"
             $message = "The resource group you specified already exists. Do you want to clean it up?"
@@ -146,13 +146,13 @@ Function GenerateResourcesAndImage {
 
             switch ($result)
             {
-                0 { Remove-AzureRmResourceGroup -Name $ResourceGroupName -Force; New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation }
+                0 { Remove-AzResourceGroup -Name $ResourceGroupName -Force; New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation }
                 1 { <# Do nothing #> }
                 2 { exit }
             }
         }
     } else {
-        New-AzureRmResourceGroup -Name $ResourceGroupName -Location $AzureLocation
+        New-AzResourceGroup -Name $ResourceGroupName -Location $AzureLocation
     }
 
     # This script should follow the recommended naming conventions for azure resources
@@ -164,19 +164,19 @@ Function GenerateResourcesAndImage {
     $storageAccountName = $storageAccountName.Replace("-", "").Replace("_", "").Replace("(", "").Replace(")", "").ToLower()
     $storageAccountName += "001"
 
-    New-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS"
+    New-AzStorageAccount -ResourceGroupName $ResourceGroupName -AccountName $storageAccountName -Location $AzureLocation -SkuName "Standard_LRS"
 
     $spDisplayName = [System.GUID]::NewGuid().ToString().ToUpper()
-    $sp = New-AzureRmADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force)
+    $sp = New-AzADServicePrincipal -DisplayName $spDisplayName -Password (ConvertTo-SecureString $ServicePrincipalClientSecret -AsPlainText -Force)
 
     $spAppId = $sp.ApplicationId
     $spClientId = $sp.ApplicationId
     $spObjectId = $sp.Id
     Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup
 
-    New-AzureRmRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId
+    New-AzRoleAssignment -RoleDefinitionName Contributor -ServicePrincipalName $spAppId
     Start-Sleep -Seconds $SecondsToWaitForServicePrincipalSetup
-    $sub = Get-AzureRmSubscription -SubscriptionId $SubscriptionId
+    $sub = Get-AzSubscription -SubscriptionId $SubscriptionId
     $tenantId = $sub.TenantId
     # "", "Note this variable-setting script for running Packer with these Azure resources in the future:", "==============================================================================================", "`$spClientId = `"$spClientId`"", "`$ServicePrincipalClientSecret = `"$ServicePrincipalClientSecret`"", "`$SubscriptionId = `"$SubscriptionId`"", "`$tenantId = `"$tenantId`"", "`$spObjectId = `"$spObjectId`"", "`$AzureLocation = `"$AzureLocation`"", "`$ResourceGroupName = `"$ResourceGroupName`"", "`$storageAccountName = `"$storageAccountName`"", "`$install_password = `"$install_password`"", ""
 

images.CI/macos/azure-pipelines/image-generation.yml

@@ -7,6 +7,8 @@ jobs:
   variables:
   - group: Mac-Cloud Image Generation
   - group: Mac-Cloud Image Generation Key Vault
+  - name: VirtualMachineName
+    value: $(Build.BuildNumber).$(System.JobAttempt)
 
   steps:
   - checkout: self
@@ -56,7 +58,7 @@ jobs:
         -var="output_folder=$(output-folder)" `
         -var="vm_username=$(vm-username)" `
         -var="vm_password=$(vm-password)" `
-        -var="build_id=$(Build.BuildNumber)" `
+        -var="build_id=${{ variables.VirtualMachineName }}" `
         -var="baseimage_name=${{ parameters.base_image_name }}" `
         -var="github_feed_token=$(github-feed-token)" `
         -var="xcode_install_user=$(xcode-installation-user)" `
@@ -83,7 +85,7 @@ jobs:
       ls $(Common.TestResultsDirectory)
 
       echo "Put VM name to 'VM_Done_Name' file"
-      echo "$(Build.BuildNumber)" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name"
+      echo "${{ variables.VirtualMachineName }}" > "$(Build.ArtifactStagingDirectory)/VM_Done_Name"
     displayName: Prepare artifact
 
   - bash: |

images/linux/scripts/installers/post-deployment.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+################################################################################
+##  File: post-deployment.sh
+##  Desc: Post deployment actions
+################################################################################
+
+# set chmod -R 777 /opt
+if [[ -d "/opt" ]]; then
+    echo "chmod -R 777 /opt"
+    chmod -R 777 /opt
+fi
+
+# remove installer and helper folders
+rm -rf $HELPER_SCRIPT_FOLDER
+rm -rf $INSTALLER_SCRIPT_FOLDER
+chmod 755 $IMAGE_FOLDER

images/linux/ubuntu1604.json

@@ -305,10 +305,13 @@
         },
         {
             "type": "shell",
-            "inline": [
+            "scripts":[
-                "rm -rf {{user `helper_script_folder`}}",
+                "{{template_dir}}/scripts/installers/post-deployment.sh"
-                "rm -rf {{user `installer_script_folder`}}",
+            ],
-                "chmod 755 {{user `image_folder`}}"
+            "environment_vars":[
+                "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "IMAGE_FOLDER={{user `image_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu1804.json

@@ -309,10 +309,13 @@
         },
         {
             "type": "shell",
-            "inline": [
+            "scripts":[
-                "rm -rf {{user `helper_script_folder`}}",
+                "{{template_dir}}/scripts/installers/post-deployment.sh"
-                "rm -rf {{user `installer_script_folder`}}",
+            ],
-                "chmod 755 {{user `image_folder`}}"
+            "environment_vars":[
+                "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "IMAGE_FOLDER={{user `image_folder`}}"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu2004.json

@@ -311,22 +311,25 @@
         },
         {
             "type": "shell",
-            "scripts": [
+            "scripts":[
+                "{{template_dir}}/scripts/installers/post-deployment.sh"
+            ],
+            "environment_vars":[
+                "HELPER_SCRIPT_FOLDER={{user `helper_script_folder`}}",
+                "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
+                "IMAGE_FOLDER={{user `image_folder`}}"
+            ],
+            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
+        },
+        {
+            "type": "shell",
+            "scripts":[
                 "{{template_dir}}/scripts/installers/validate-disk-space.sh"
             ],
             "environment_vars": [
                 "RUN_VALIDATION={{user `run_validation_diskspace`}}"
             ]
         },
-        {
-            "type": "shell",
-            "inline": [
-                "rm -rf {{user `helper_script_folder`}}",
-                "rm -rf {{user `installer_script_folder`}}",
-                "chmod 755 {{user `image_folder`}}"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
         {
             "type": "file",
             "source": "{{template_dir}}/config/ubuntu2004.conf",

images/macos/provision/configuration/configure-machine.sh

@@ -19,3 +19,12 @@ sudo rm -f /var/vm/sleepimage
 
 # Change screen resolution to the maximum supported for 4Mb video memory
 sudo "/Library/Application Support/VMware Tools/vmware-resolutionSet" 1176 885
+
+# https://developer.apple.com/support/expiration/
+# Enterprise iOS Distribution Certificates generated between February 7 and September 1st, 2020 will expire on February 7, 2023.
+# Rotate the certificate before expiration to ensure your apps are installed and signed with an active certificate.
+# Confirm that the correct intermediate certificate is installed by verifying the expiration date is set to 2030.
+# sudo security delete-certificate -Z FF6797793A3CD798DC5B2ABEF56F73EDC9F83A64 /Library/Keychains/System.keychain
+curl https://www.apple.com/certificateauthority/AppleWWDRCAG3.cer --output $HOME/AppleWWDRCAG3.cer --silent
+sudo security add-trusted-cert -d -r unspecified -k /Library/Keychains/System.keychain $HOME/AppleWWDRCAG3.cer
+rm $HOME/AppleWWDRCAG3.cer

images/macos/provision/core/commonutils.sh

@@ -42,11 +42,16 @@ done
 # brew cask install
 bcask_common_utils=(
     julia
-    virtualbox
-    vagrant
-    r
 )
 
+if is_Less_BigSur; then
+    bcask_common_utils+=(
+        virtualbox
+        vagrant
+        r
+    )
+fi
+
 for package in ${bcask_common_utils[@]}; do
     echo "Install $package"
     brew cask install $package

images/macos/software-report/SoftwareReport.Generator.ps1

@@ -55,6 +55,10 @@ if ( -not $os.IsHighSierra) {
     $markdown += New-MDList -Style Unordered -NoNewLine -Lines $lines
 }
 
+if ($os.IsLessThanBigSur) {
+    $markdown += New-MDList -Style Unordered -Lines @(Get-RVersion) -NoNewLine
+}
+
 $markdown += New-MDList -Style Unordered -Lines @(
     "Node.js ${nodejsVersion}"
     "NVM ${nvmVersion}"
@@ -63,7 +67,6 @@ $markdown += New-MDList -Style Unordered -Lines @(
     $python3Version,
     "Ruby ${rubyVersion}",
     (Get-DotnetVersionList),
-    (Get-RVersion),
     "Go ${goVersion}",
     "$phpVersion",
     "$juliaVersion"
@@ -139,8 +142,6 @@ $bazelVersion = Run-Command "bazel --version" | Take-Part -Part 0 -Delimiter "-"
 $bazeliskVersion = Run-Command "bazelisk version" | Select-String "Bazelisk version:" | Take-Part -Part 1 -Delimiter ":"
 $packerVersion = Run-Command "packer --version"
 $helmVersion = Run-Command "helm version --short"
-$vbox = Run-Command "vboxmanage -v"
-$vagrant = Run-Command "vagrant -v"
 $mongo = Run-Command "mongo --version" | Select-String "MongoDB shell version" | Take-Part -Part 3
 $mongod = Run-Command "mongod --version" | Select-String "db version " | Take-Part -Part 2
 $p7zip = Run-Command "7z i" | Select-String "7-Zip" | Take-Part -Part 0,2
@@ -166,10 +167,8 @@ $markdown += New-MDList -Style Unordered -NoNewLine -Lines @(
     $bazelVersion,
     "bazelisk $($bazeliskVersion.Trim())",
     "helm $helmVersion",
-    "virtualbox $vbox",
     "mongo $mongo",
     "mongod $mongod",
-    "$vagrant",
     $p7zip
 )
 if ($os.IsHigherThanMojave) {
@@ -177,8 +176,14 @@ if ($os.IsHigherThanMojave) {
     $markdown += New-MDList -Lines "Newman $newmanVersion" -Style Unordered -NoNewLine
 }
 if ($os.IsLessThanBigSur) {
+    $vagrant = Run-Command "vagrant -v"
+    $vbox = Run-Command "vboxmanage -v"
     $parallelVersion = Run-Command "parallel --version" | Select-String "GNU parallel" | Select-Object -First 1
-    $markdown += New-MDList -Lines $parallelVersion -Style Unordered
+    $markdown += New-MDList -Style Unordered -Lines @(
+        "virtualbox $vbox",
+        $vagrant,
+        $parallelVersion
+    )
 }
 $markdown += New-MDNewLine
 

images/macos/tests/Common.Tests.ps1

@@ -12,6 +12,14 @@ Describe "Disk free space" {
     }
 }
 
+Describe "Certificate" {
+    It "Apple Worldwide Developer Relations Certification Authority[expired: 2030-02] is installed" {
+        $sha1Hash = "06EC06599F4ED0027CC58956B4D3AC1255114F35"
+        $certs = security find-certificate -a -c Worldwide -p -Z | Out-String
+        $certs | Should -Match $sha1Hash
+    }
+}
+
 Describe "Git" {
     It "git is installed" {
         "git --version" | Should -ReturnZeroExitCode
@@ -213,11 +221,11 @@ Describe "Common utilities" {
         "helm version --short" | Should -ReturnZeroExitCode
     }
 
-    It "virtualbox" {
+    It "virtualbox" -Skip:($os.IsBigSur) {
         "vboxmanage -v" | Should -ReturnZeroExitCode
     }
 
-    It "vagrant" {
+    It "vagrant" -Skip:($os.IsBigSur) {
         "vagrant --version" | Should -ReturnZeroExitCode
     }
 

images/win/scripts/Installers/Install-Edge.ps1

@@ -24,7 +24,14 @@ $EdgeDriverVersionFile = Start-DownloadWithRetry -Url $EdgeDriverVersionUrl -Nam
 Write-Host "Download Microsoft Edge WebDriver..."
 $EdgeDriverLatestVersion = Get-Content -Path $EdgeDriverVersionFile
 $EdgeDriverArchName = "edgedriver_win64.zip"
-$EdgeDriverDownloadUrl="https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}"
+# A temporary workaround to install the previous driver version because 85.0.564.60 for win64 doesn't exist
+if ($EdgeDriverLatestVersion -eq "85.0.564.60")
+{
+    $EdgeDriverLatestVersion = "85.0.564.51"
+    Set-Content -Path $EdgeDriverVersionFile -Value $EdgeDriverLatestVersion
+}
+
+$EdgeDriverDownloadUrl = "https://msedgedriver.azureedge.net/${EdgeDriverLatestVersion}/${EdgeDriverArchName}"
 
 $EdgeDriverArchPath = Start-DownloadWithRetry -Url $EdgeDriverDownloadUrl -Name $EdgeDriverArchName