Update go-parser.psm1

Fix YAML dependency

azure-pipelines/get-tool-versions.yml

@@ -20,25 +20,38 @@ variables:
   VmImage: 'ubuntu-18.04'
 
 stages:
-- stage: Get_New_Versions
+- stage: Find_New_Versions
   dependsOn: []
   jobs:
-  - job: Get_Tool_Versions
+  - job: Find_New_Versions
     pool:
       name: $(PoolName)
       vmImage: $(VmImage)
     steps:
     - template: /azure-pipelines/templates/get-tool-versions-steps.yml
 
+- stage: Check_New_Versions
+  dependsOn: Find_New_Versions
+  jobs:
+  - job: Check_New_Versions
+    pool:
+      name: $(PoolName)
+      vmImage: $(VmImage)
+    variables:
+      ToolVersions: $[ stageDependencies.Find_New_Versions.Find_New_Versions.outputs['Get_versions.TOOL_VERSIONS'] ]
+    steps:
+    - template: /azure-pipelines/templates/check-versions.yml
+
 - stage: Trigger_Builds
-  dependsOn: Get_New_Versions
+  dependsOn: [Find_New_Versions, Check_New_Versions]
+  condition: and(succeeded(), ne(variables['WORKFLOW_FILE_NAME'], ''))
   jobs:
   - deployment: Run_Builds
     pool:
       name: $(PoolName)
       vmImage: $(VmImage)
     variables:
-      ToolVersions: $[ stageDependencies.Get_New_Versions.Get_Tool_Versions.outputs['Get_versions.TOOL_VERSIONS'] ]
+      ToolVersions: $[ stageDependencies.Find_New_Versions.Find_New_Versions.outputs['Get_versions.TOOL_VERSIONS'] ]
     timeoutInMinutes: 180
     environment: 'Get Available Tools Versions - Publishing Approval'
     strategy:

azure-pipelines/templates/check-versions.yml

@@ -0,0 +1,41 @@
+steps:
+- task: PowerShell@2
+  displayName: Check Versions
+  condition: and(succeeded(), eq(variables.ToolVersions, ''))
+  inputs:
+    TargetType: inline
+    script: |
+      throw "No new versions were found"
+
+- task: PowerShell@2
+  displayName: 'Set PIPELINE_URL variable'
+  inputs:
+    TargetType: inline
+    script: |
+      $ToolName = "$(TOOL_NAME)"
+      if ($ToolName -eq "Python") {
+        $PipelineUrl = " "
+      } else {
+        $PipelineUrl = "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
+      }
+      Write-Host "##vso[task.setvariable variable=PIPELINE_URL]$PipelineUrl"
+
+- task: PowerShell@2
+  displayName: 'Change build name'
+  inputs:
+    TargetType: inline
+    script: |
+      $newBuildName = "[FOUND] $(Build.BuildNumber)"
+      Write-Host "##vso[build.updatebuildnumber]$newBuildName"
+
+- task: PowerShell@2	
+  displayName: 'Send Slack notification'
+  inputs:
+    targetType: filePath
+    filePath: './get-new-tool-versions/send-slack-notification.ps1'
+    arguments: |
+      -Url "$(SLACK_CHANNEL_URL)" `
+      -ToolName "$(TOOL_NAME)" `
+      -ToolVersion "$(ToolVersions)" `
+      -PipelineUrl "$(PIPELINE_URL)" `
+      -ImageUrl "$(IMAGE_URL)"

azure-pipelines/templates/get-tool-versions-steps.yml

@@ -6,42 +6,4 @@ steps:
     targetType: filePath
     filePath: './get-new-tool-versions/get-new-tool-versions.ps1'
     arguments: |
-        -DistURL "$(DIST_URL)" `
-        -ManifestLink "$(MANIFEST_URL)" `
-        -VersionFilterToInclude $(INCLUDE_FILTER) `
-        -VersionFilterToExclude $(EXCLUDE_FILTER)
-
-- task: PowerShell@2
-  displayName: 'Cancel build'
-  condition: and(succeeded(), eq(variables['Get_versions.TOOL_VERSIONS'], ''))
-  inputs:
-    TargetType: inline
-    script: |
-      Import-Module "./azure-devops/azure-devops-api.ps1"
-      $azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) `
-                                           -ProjectName $(System.TeamProject) `
-                                           -AccessToken $(System.AccessToken)
-
-      $AzureDevOpsApi.UpdateBuildStatus($(Build.BuildId), 'Cancelling') | Out-Null
-
-- task: PowerShell@2
-  displayName: 'Set env variable'
-  condition: and(succeeded(), ne(variables['Get_versions.TOOL_VERSIONS'], ''))
-  inputs:
-    TargetType: inline
-    script: |
-      $PipelineUrl = "$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
-      Write-Output "##vso[task.setvariable variable=PIPELINE_URL]$PipelineUrl"
-
-- task: PowerShell@2
-  displayName: 'Send Slack notification'
-  condition: and(succeeded(), ne(variables['Get_versions.TOOL_VERSIONS'], ''))
-  inputs:
-    targetType: filePath
-    filePath: './get-new-tool-versions/send-slack-notification.ps1'
-    arguments: |
-      -Url "$(SLACK_CHANNEL_URL)" `
-      -ToolName "$(TOOL_NAME)" `
-      -ToolVersion "$(Get_versions.TOOL_VERSIONS)" `
-      -PipelineUrl "$(PIPELINE_URL)" `
-      -ImageUrl "$(IMAGE_URL)"
+        -ToolName "$(TOOL_NAME)"

azure-pipelines/templates/run-ci-builds-steps.yml

@@ -1,29 +1,15 @@
 steps:
 - checkout: self
 
-- task: PowerShell@2
-  displayName: 'Get source version'
-  inputs:
-    TargetType: inline
-    script: |
-      $url = "https://api.github.com/repos/$(REPOSITORY)/commits/$(BRANCH)"
-      $commit = Invoke-RestMethod -Uri $url -Method "GET"
-      Write-Output "##vso[task.setvariable variable=COMMIT_SHA]$($commit.sha)"
-
 - task: PowerShell@2
   displayName: 'Run builds'
   inputs:
     targetType: filePath
-    filePath: './azure-devops/run-ci-builds.ps1'
+    filePath: './github/run-ci-builds.ps1'
     arguments: |
-        -TeamFoundationCollectionUri $(System.TeamFoundationCollectionUri) `
-        -AzureDevOpsProjectName $(System.TeamProject) `
-        -AzureDevOpsAccessToken $(System.AccessToken) `
-        -SourceBranch $(BRANCH) `
-        -DefinitionId $(DEFINITION_ID) `
-        -SourceVersion $(COMMIT_SHA) `
-        -ManifestLink $(MANIFEST_URL) `
-        -WaitForBuilds $(WAIT_FOR_BUILDS) `
+        -RepositoryFullName $(REPOSITORY_FULL_NAME) `
+        -AccessToken $(GITHUB_TOKEN) `
+        -WorkflowFileName $(WORKFLOW_FILE_NAME) `
+        -WorkflowDispatchRef $(DISPATCH_REF) `
         -ToolVersions "$(ToolVersions)" `
-        -RetryIntervalSec $(INTERVAL_SEC) `
-        -RetryCount $(RETRY_COUNT)
+        -PublishReleases $(PUPLISH_RELEASES)

clean-toolcache.ps1

@@ -3,6 +3,11 @@ param (
 )
 
 $targetPath = $env:AGENT_TOOLSDIRECTORY
+if ([string]::IsNullOrEmpty($targetPath)) {
+    # GitHub Windows images don't have `AGENT_TOOLSDIRECTORY` variable
+    $targetPath = $env:RUNNER_TOOL_CACHE
+}
+
 if ($ToolName) {
     $targetPath = Join-Path $targetPath $ToolName
 }

common-helpers.psm1

@@ -83,3 +83,26 @@ function IsNixPlatform {
 
     return ($Platform -match "macos") -or ($Platform -match "darwin") -or ($Platform -match "ubuntu") -or ($Platform -match "linux")
 }
+
+<#
+.SYNOPSIS
+Get root directory of selected tool
+#>
+function GetToolDirectory {
+    param(
+        [Parameter(Mandatory=$true)]
+        [String]$ToolName,
+        [Parameter(Mandatory=$true)]
+        [String]$Version,
+        [Parameter(Mandatory=$true)]
+        [String]$Architecture
+    )
+    $targetPath = $env:AGENT_TOOLSDIRECTORY
+    if ([string]::IsNullOrEmpty($targetPath)) {
+        # GitHub Windows images don't have `AGENT_TOOLSDIRECTORY` variable
+        $targetPath = $env:RUNNER_TOOL_CACHE
+    }
+    $ToolcachePath = Join-Path -Path $targetPath -ChildPath $ToolName
+    $ToolcacheVersionPath = Join-Path -Path $ToolcachePath -ChildPath $Version
+    return Join-Path $ToolcacheVersionPath $Architecture
+}

get-new-tool-versions/get-new-tool-versions.Tests.ps1

@@ -1,93 +0,0 @@
-#Requires -Modules Pester
-
-Import-Module (Join-Path $PSScriptRoot "helpers.psm1") -Force
-  
-Describe "Validate-FiltersFormat" {
-    It "Filter with word" {
-        { Validate-FiltersFormat -Filters @("1two.2") } | Should -Throw "Invalid filter format"
-    }
-
-    It "Filter with non-word character" {
-        { Validate-FiltersFormat -Filters @("1,.2") } | Should -Throw "Invalid filter format"
-    }
-
-    It "Valid filters" {
-        { Validate-FiltersFormat -Filters @("*", "1", "1.*", "1.2", "1.2.*") } | Should -Not -Throw "Invalid filter format"
-    }
-}
-
-Describe "Format-Versions" {
-    It "Clean versions" {
-        $actualOutput = Format-Versions -Versions @("14.2.0", "1.14.0")
-        $expectedOutput = @("14.2.0", "1.14.0")
-        $actualOutput | Should -Be $expectedOutput
-    }
-
-    It "Versions with prefixes" {
-        $actualOutput = Format-Versions -Versions @("v14.2.0", "go1.14.0")
-        $expectedOutput = @("14.2.0", "1.14.0")
-        $actualOutput | Should -Be $expectedOutput
-    }
-
-    It "Skip beta and rc versions" {
-        $actualOutput = Format-Versions -Versions @("14.2.0-beta", "v1.14.0-rc-1")
-        $expectedOutput = @()
-        $actualOutput | Should -Be $expectedOutput
-    }
-    
-    It "Short version" {
-        $actualOutput = Format-Versions -Versions @("14.2", "v2.0")
-        $expectedOutput = @("14.2.0", "2.0.0")
-        $actualOutput | Should -Be $expectedOutput
-    }
-
-    It "Skip versions with 1 digit" {
-        $actualOutput = Format-Versions -Versions @("14", "v2")
-        $expectedOutput = @()
-        $actualOutput | Should -Be $expectedOutput
-    }
-}
-
-Describe "Select-VersionsByFilter" {
-    $inputVersions = @("8.2.1", "9.3.3", "10.0.2", "10.0.3", "10.5.6", "12.4.3", "12.5.1", "14.2.0")
-
-    It "Include filter only" {
-        $includeFilters = @("8.*", "14.*")
-        $excludeFilters = @()
-        $actualOutput = Select-VersionsByFilter -Versions $inputVersions -IncludeFilters $includeFilters -ExcludeFilters $excludeFilters
-        $expectedOutput = @("8.2.1", "14.2.0")
-        $actualOutput | Should -Be $expectedOutput
-    }
-
-    It "Include and exclude filters" {
-        $includeFilters = @("10.*", "12.*")
-        $excludeFilters = @("10.0.*", "12.4.3")
-        $actualOutput = Select-VersionsByFilter -Versions $inputVersions -IncludeFilters $includeFilters -ExcludeFilters $excludeFilters
-        $expectedOutput = @("10.5.6", "12.5.1")
-        $actualOutput | Should -Be $expectedOutput
-    }
-
-    It "Exclude filter only" {
-        $includeFilters = @()
-        $excludeFilters = @("10.*", "12.*")
-        $actualOutput = Select-VersionsByFilter -Versions $inputVersions -IncludeFilters $includeFilters -ExcludeFilters $excludeFilters
-        $expectedOutput = @("8.2.1", "9.3.3", "14.2.0")
-        $actualOutput | Should -Be $expectedOutput
-    }
-
-    It "Include and exclude filters are empty" {
-        $actualOutput = Select-VersionsByFilter -Versions $inputVersions
-        $expectedOutput = @("8.2.1", "9.3.3", "10.0.2", "10.0.3", "10.5.6", "12.4.3", "12.5.1", "14.2.0")
-        $actualOutput | Should -Be $expectedOutput
-    }
-}
-
-Describe "Skip-ExistingVersions" {
-    It "Substract versions correctly" {
-        $distInput = @("14.2.0", "14.3.0", "14.4.0", "14.4.1")
-        $manifestInput = @("12.0.0", "14.2.0", "14.4.0")
-        $actualOutput =  Skip-ExistingVersions -VersionsFromDist $distInput -VersionsFromManifest $manifestInput
-        $expectedOutput = @("14.3.0", "14.4.1")
-        $actualOutput | Should -Be $expectedOutput
-    }
-}

get-new-tool-versions/get-new-tool-versions.ps1

@@ -2,79 +2,26 @@
 .SYNOPSIS
 Check and return list of new available tool versions
 
-.PARAMETER DistURL
-Required parameter. Link to the json file included all available tool versions
-.PARAMETER ManifestLink
-Required parameter. Link to the the version-manifest.json file
-.PARAMETER VersionFilterToInclude
-Optional parameter. List of filters to include particular versions
-.PARAMETER VersionFilterToExclude
-Optional parameter. List of filters to exclude particular versions
-.PARAMETER RetryIntervalSec
-Optional parameter. Retry interval in seconds
-.PARAMETER RetryCount
-Optional parameter. Retry count
+.PARAMETER ToolName
+Required parameter. The name of tool for which parser is available (Node, Go, Python)
 #>
 
 param (
-    [Parameter(Mandatory)] [string] $DistURL,
-    [Parameter(Mandatory)] [string] $ManifestLink,
-    [string[]] $VersionFilterToInclude,
-    [string[]] $VersionFilterToExclude,
-    [UInt32] $RetryIntervalSec = 60,
-    [UInt32] $RetryCount = 3
+    [Parameter(Mandatory)] [string] $ToolName
 )
 
-Import-Module (Join-Path $PSScriptRoot "helpers.psm1")
+Import-Module "$PSScriptRoot/parsers/parsers-factory.psm1"
 
-function Get-VersionsByUrl {
-    param (
-        [Parameter(Mandatory)] [string] $ToolPackagesUrl,
-        [Parameter(Mandatory)] [UInt32] $RetryIntervalSec,
-        [Parameter(Mandatory)] [UInt32] $RetryCount
-    )
+$ToolVersionParser = Get-ToolVersionsParser -ToolName $ToolName
+$VersionsFromDist = $ToolVersionParser.GetAvailableVersions()
+$VersionsFromManifest = $ToolVersionParser.GetUploadedVersions()
 
-    $packages = Invoke-RestMethod $ToolPackagesUrl -MaximumRetryCount $RetryCount -RetryIntervalSec $RetryIntervalSec
-    return $packages.version
-}
+$VersionsToBuild = $VersionsFromDist | Where-Object { $VersionsFromManifest -notcontains $_ }
 
-if ($VersionFilterToInclude) {
-    Validate-FiltersFormat -Filters $VersionFilterToInclude
-}
-
-if ($VersionFilterToExclude) {
-    Validate-FiltersFormat -Filters $VersionFilterToExclude
-}
-
-Write-Host "Get the packages list from $DistURL"
-$versionsFromDist = Get-VersionsByUrl -ToolPackagesUrl $DistURL `
-                                      -RetryIntervalSec $RetryIntervalSec `
-                                      -RetryCount $RetryCount
-
-Write-Host "Get the packages list from $ManifestLink"
-[Version[]] $versionsFromManifest = Get-VersionsByUrl -ToolPackagesUrl $ManifestLink `
-                                                      -RetryIntervalSec $RetryIntervalSec `
-                                                      -RetryCount $RetryCount
-
-[Version[]] $formattedVersions = Format-Versions -Versions $versionsFromDist
-
-$formattedVersions = Select-VersionsByFilter -Versions $formattedVersions `
-                                             -IncludeFilters $VersionFilterToInclude `
-                                             -ExcludeFilters $VersionFilterToExclude
-
-if (-not $formattedVersions) {
-    Write-Host "Couldn't find available versions with current filters"
-    exit 1
-}
-
-$versionsToBuild = Skip-ExistingVersions -VersionsFromManifest $versionsFromManifest `
-                                         -VersionsFromDist $formattedVersions
-
-if ($versionsToBuild) {
-    $availableVersions = $versionsToBuild -join ","
-    $toolVersions = $availableVersions.Replace(",",", ")
-    Write-Host "The following versions are available to build:`n$toolVersions"
-    Write-Output "##vso[task.setvariable variable=TOOL_VERSIONS;isOutput=true]$toolVersions"
+if ($VersionsToBuild) {
+    $availableVersions = $VersionsToBuild -join ", "
+    Write-Host "The following versions are available to build:`n${availableVersions}"
+    Write-Host "##vso[task.setvariable variable=TOOL_VERSIONS;isOutput=true]${availableVersions}"
 } else {
     Write-Host "There aren't versions to build"
 }

get-new-tool-versions/helpers.psm1

@@ -1,17 +1,3 @@
-function Validate-FiltersFormat {
-    param (
-        [Parameter(Mandatory)] [string[]] $Filters
-    )
-
-    foreach($filter in $Filters) {
-        $filter.Split('.') | ForEach-Object {
-            if (($_ -notmatch '^\d+$') -and ($_ -ne '*')) {
-                throw "Invalid filter format - $filter"
-            }
-        }
-    }
-}
-
 function Format-Versions {
     param (
         [Parameter(Mandatory)] [string[]] $Versions

get-new-tool-versions/parsers/base-parser.psm1

@@ -0,0 +1,31 @@
+class BaseVersionsParser {
+    [Int32]$ApiRetryCount = 3
+    [Int32]$ApiRetryIntervalSeconds = 60
+
+    [SemVer[]] GetAvailableVersions() {
+        $allVersionsRaw = $this.ParseAllAvailableVersions()
+        $allVersions = $allVersionsRaw | ForEach-Object { $this.FormatVersion($_) }
+        $filteredVersions = $allVersions | Where-Object { $this.ShouldIncludeVersion($_) }
+        return $filteredVersions
+    }
+
+    [SemVer[]] GetUploadedVersions() {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [SemVer[]] ParseAllAvailableVersions() {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [bool] ShouldIncludeVersion([SemVer]$Version) {
+        throw "Method is not implemented in base class"
+    }
+
+    hidden [string] BuildGitHubFileUrl($OrganizationName, $RepositoryName, $BranchName, $FilePath) {
+        return "https://raw.githubusercontent.com/${OrganizationName}/${RepositoryName}/${BranchName}/${FilePath}"
+    }
+}

get-new-tool-versions/parsers/go-parser.psm1

@@ -0,0 +1,30 @@
+using module "./base-parser.psm1"
+
+class GoVersionsParser: BaseVersionsParser {
+    [SemVer[]] GetUploadedVersions() {
+        $url = $this.BuildGitHubFileUrl("actions", "go-versions", "main", "versions-manifest.json")
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [string[]] ParseAllAvailableVersions() {
+        $url = "https://golang.org/dl/?mode=json&include=all"
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        $cleanVersion = $VersionSpec -replace "^go", ""
+        $semanticVersion = $cleanVersion -replace "(\d+\.\d+\.?\d+?)((?:alpha|beta|rc))(\d*)",'$1-$2.$3'
+        return [SemVer]$semanticVersion
+    }
+
+    hidden [bool] ShouldIncludeVersion([SemVer]$Version) {
+        if ($Version.PreReleaseLabel) {
+            return $false
+        }
+
+        # For Go, we include all versions greater than 1.12
+        return $Version -gt [SemVer]"1.12.0"
+    }
+}

get-new-tool-versions/parsers/node-parser.psm1

@@ -0,0 +1,30 @@
+using module "./base-parser.psm1"
+
+class NodeVersionsParser: BaseVersionsParser {
+    [SemVer[]] GetUploadedVersions() {
+        $url = $this.BuildGitHubFileUrl("actions", "node-versions", "main", "versions-manifest.json")
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [string[]] ParseAllAvailableVersions() {
+        $url = "https://nodejs.org/dist/index.json"
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        $cleanVersion = $VersionSpec -replace "^v", ""
+        return [SemVer]$cleanVersion
+    }
+
+    hidden [bool] ShouldIncludeVersion([SemVer]$Version) {
+        if ($Version.Major -lt 8) {
+            return $false
+        }
+
+        # For Node.JS, we should include all LTS versions (all even-numbered releases)
+        # https://nodejs.org/en/about/releases/
+        return $Version.Major % 2 -eq 0
+    }
+}

get-new-tool-versions/parsers/parsers-factory.psm1

@@ -0,0 +1,19 @@
+using module "./node-parser.psm1"
+using module "./go-parser.psm1"
+using module "./python-parser.psm1"
+
+function Get-ToolVersionsParser {
+    param(
+        [Parameter(Mandatory)]
+        [string]$ToolName
+    )
+
+    switch ($ToolName) {
+        "Node" { return [NodeVersionsParser]::New() }
+        "Go" { return [GoVersionsParser]::New() }
+        "Python" { return [PythonVersionsParser]::New() }
+        Default {
+            throw "Unknown tool name"
+        }
+    }
+}

get-new-tool-versions/parsers/python-parser.psm1

@@ -0,0 +1,53 @@
+using module "./base-parser.psm1"
+
+class PythonVersionsParser: BaseVersionsParser {
+    [SemVer[]] GetUploadedVersions() {
+        $url = $this.BuildGitHubFileUrl("actions", "python-versions", "main", "versions-manifest.json")
+        $releases = Invoke-RestMethod $url -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        return $releases.version
+    }
+
+    hidden [string[]] ParseAllAvailableVersions() {
+        $stableVersionsUrl = "https://www.python.org/ftp/python"
+        $stableVersionsHtmlRaw = Invoke-WebRequest $stableVersionsUrl -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+        $stableVersionsList = $stableVersionsHtmlRaw.Links.href | Where-Object {
+            $parsed = $null
+            return $_.EndsWith("/") -and [SemVer]::TryParse($_.Replace("/", ""), [ref]$parsed)
+        }
+
+        return $stableVersionsList | ForEach-Object {
+            $subVersionsUrl = "${stableVersionsUrl}/${_}"
+            $subVersionsHtmlRaw = Invoke-WebRequest $subVersionsUrl -MaximumRetryCount $this.ApiRetryCount -RetryIntervalSec $this.ApiRetryIntervalSeconds
+            return $subVersionsHtmlRaw.Links.href | ForEach-Object {
+                if ($_ -match "^Python-(\d+\.\d+\.\d+[a-z]{0,2}\d*)\.tgz$") {
+                    return $Matches[1]
+                }
+            }
+        }
+    }
+
+    hidden [SemVer] FormatVersion([string]$VersionSpec) {
+        $VersionSpec -match "^(\d+)\.(\d+)\.(\d+)([a-z]{1,2})?(\d+)?$"
+        
+        if ($Matches.Count -gt 4) {
+            $VersionLabel = "{0}.{1}" -f $this.ConvertPythonLabel($Matches[4]), $Matches[5]
+            return [SemVer]::new($Matches[1], $Matches[2], $Matches[3], $VersionLabel)
+        }
+
+        return [SemVer]::new($Matches[1], $Matches[2], $Matches[3])
+    }
+
+    hidden [string] ConvertPythonLabel([string]$Label) {
+        switch ($Label) {
+            "a" { return "alpha" }
+            "b" { return "beta" }
+        }
+
+        return $Label
+    }
+
+    [bool] ShouldIncludeVersion([SemVer]$Version) {
+        # For Python, we include all versions greater than 3.9.0
+        return $Version -gt [SemVer]"3.9.0"
+    }
+}

get-new-tool-versions/send-slack-notification.ps1

@@ -27,10 +27,7 @@ param(
     [ValidateNotNullOrEmpty()]
     [System.String]$ToolVersion,
 
-    [Parameter(Mandatory)]
-    [ValidateNotNullOrEmpty()]
     [System.String]$PipelineUrl,
-
     [System.String]$ImageUrl = 'https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png'
 )
 
@@ -38,7 +35,10 @@ param(
 Import-Module $PSScriptRoot/helpers.psm1 -DisableNameChecking
 
 # Create JSON body
-$text = "The following versions of '$toolName' are available to upload: $toolVersion\nLink to the pipeline: $pipelineUrl"
+$text = "The following versions of '$toolName' are available to upload: $toolVersion"
+if (-not ([string]::IsNullOrWhiteSpace($PipelineUrl))) {
+    $text += "\nLink to the pipeline: $pipelineUrl"
+}
 $jsonBodyMessage = @"
 {
     "blocks": [

github/create-pull-request.ps1

@@ -2,10 +2,8 @@
 .SYNOPSIS
 Create commit with all unstaged changes in repository and create pull-request
 
-.PARAMETER RepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER RepositoryName
-Optional parameter. The name of tool repository
+.PARAMETER RepositoryFullName
+Required parameter. The owner and repository name. For example, 'actions/versions-package-tools'
 .PARAMETER AccessToken
 Required parameter. PAT Token to authorize
 .PARAMETER BranchName
@@ -18,8 +16,7 @@ Required parameter. The title of pull-request
 Required parameter. The description of pull-request
 #>
 param (
-    [Parameter(Mandatory)] [string] $RepositoryOwner,
-    [Parameter(Mandatory)] [string] $RepositoryName,
+    [Parameter(Mandatory)] [string] $RepositoryFullName,
     [Parameter(Mandatory)] [string] $AccessToken,
     [Parameter(Mandatory)] [string] $BranchName,
     [Parameter(Mandatory)] [string] $CommitMessage,
@@ -46,11 +43,11 @@ function Update-PullRequest {
 
     $updatedPullRequest = $GitHubApi.UpdatePullRequest($Title, $Body, $BranchName, $PullRequest.number)
 
-    if (($updatedPullRequest -eq $null) -or ($updatedPullRequest.html_url -eq $null)) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while updating pull request."
+    if (($null -eq $updatedPullRequest) -or ($null -eq $updatedPullRequest.html_url)) {
+        Write-Host "Unexpected error occurs while updating pull request."
         exit 1
     }
-    Write-host "##[section] Pull request updated: $($updatedPullRequest.html_url)"
+    Write-host "Pull request updated: $($updatedPullRequest.html_url)"
 }
 
 function Create-PullRequest {
@@ -67,12 +64,12 @@ function Create-PullRequest {
 
     $createdPullRequest = $GitHubApi.CreateNewPullRequest($Title, $Body, $BranchName)
 
-    if (($createdPullRequest -eq $null) -or ($createdPullRequest.html_url -eq $null)) {
-        Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while creating pull request."
+    if (($null -eq $createdPullRequest) -or ($null -eq $createdPullRequest.html_url)) {
+        Write-Host "Unexpected error occurs while creating pull request."
         exit 1
     }
 
-    Write-host "##[section] Pull request created: $($createdPullRequest.html_url)"
+    Write-host "Pull request created: $($createdPullRequest.html_url)"
 }
 
 Write-Host "Configure local git preferences"
@@ -87,8 +84,8 @@ Git-CommitAllChanges -Message $CommitMessage
 Write-Host "Push branch: $BranchName"
 Git-PushBranch -Name $BranchName -Force $true
 
-$gitHubApi = Get-GitHubApi -AccountName $RepositoryOwner -ProjectName $RepositoryName -AccessToken $AccessToken
-$pullRequest = $gitHubApi.GetPullRequest($BranchName, $RepositoryOwner)
+$gitHubApi = Get-GitHubApi -RepositoryFullName $RepositoryFullName -AccessToken $AccessToken
+$pullRequest = $gitHubApi.GetPullRequest($BranchName)
 
 if ($pullRequest.Count -gt 0) {
     Write-Host "Update pull request"

github/create-release.ps1

@@ -0,0 +1,86 @@
+<#
+.SYNOPSIS
+Trigger runs on the workflow_dispatch event to create tool release
+
+.PARAMETER RepositoryFullName
+Required parameter. The owner and repository name. For example, 'actions/versions-package-tools'
+.PARAMETER AccessToken
+Required parameter. PAT Token to authorize
+.PARAMETER ToolVersion
+Required parameter. Version of tool
+.PARAMETER TagName
+Required parameter. The name of the release tag
+.PARAMETER ReleaseBody
+Required parameter. Text describing the contents of the release
+.PARAMETER EventType
+Required parameter. The name of the repository dispatch event
+#>
+
+param (
+    [Parameter(Mandatory)] [string] $RepositoryFullName,
+    [Parameter(Mandatory)] [string] $AccessToken,
+    [Parameter(Mandatory)] [string] $ToolVersion,
+    [Parameter(Mandatory)] [string] $TagName,
+    [Parameter(Mandatory)] [string] $ReleaseBody,
+    [Parameter(Mandatory)] [string] $EventType,
+    [UInt32] $RetryIntervalSec = 10,
+    [UInt32] $RetryCount = 5
+)
+
+Import-Module (Join-Path $PSScriptRoot "github-api.psm1")
+
+function Create-Release {
+    param (
+        [Parameter(Mandatory)] [object] $GitHubApi,
+        [Parameter(Mandatory)] [string] $ToolVersion,
+        [Parameter(Mandatory)] [string] $TagName,
+        [Parameter(Mandatory)] [string] $ReleaseBody,
+        [Parameter(Mandatory)] [string] $EventType
+    )
+
+    $eventPayload = @{
+        ToolVersion = $ToolVersion
+        TagName = $TagName
+        ReleaseBody = $ReleaseBody
+    }
+
+    Write-Host "Create '$EventType' repository dispatch event"
+    $GitHubApi.CreateRepositoryDispatch($EventType, $eventPayload)
+}
+
+function Validate-ReleaseAvailability {
+    param (
+        [Parameter(Mandatory)] [object] $GitHubApi,
+        [Parameter(Mandatory)] [string] $TagName,
+        [Parameter(Mandatory)] [UInt32] $RetryIntervalSec,
+        [Parameter(Mandatory)] [UInt32] $RetryCount
+    )
+
+    do {
+        $createdRelease = $GitHubApi.GetReleases() | Where-Object { $_.tag_name -eq $TagName }
+        if ($createdRelease) {
+            Write-Host "Release was successfully created: $($createdRelease.html_url)"
+            return
+        }
+
+        $RetryCount--
+        Start-Sleep -Seconds $RetryIntervalSec
+    } while($RetryCount -gt 0)
+
+    Write-Host "Release was not created"
+    exit 1
+}
+
+$gitHubApi = Get-GitHubApi -RepositoryFullName $RepositoryFullName -AccessToken $AccessToken
+
+Create-Release -GitHubApi $gitHubApi `
+               -ToolVersion $ToolVersion `
+               -TagName $TagName `
+               -ReleaseBody $ReleaseBody `
+               -EventType $EventType
+
+Start-Sleep -s $RetryIntervalSec
+Validate-ReleaseAvailability -GitHubApi $gitHubApi `
+                             -TagName $TagName `
+                             -RetryIntervalSec $RetryIntervalSec `
+                             -RetryCount $RetryCount

github/github-api.psm1

@@ -5,8 +5,8 @@ The module that contains a bunch of methods to interact with GitHub API V3
 class GitHubApi
 {
     [string] $BaseUrl
-    [string] $RepoOwner
     [object] $AuthHeader
+    [string] $RepositoryOwner
 
     GitHubApi(
         [string] $AccountName,
@@ -15,6 +15,7 @@ class GitHubApi
     ) {
         $this.BaseUrl = $this.BuildBaseUrl($AccountName, $ProjectName)
         $this.AuthHeader = $this.BuildAuth($AccessToken)
+        $this.RepositoryOwner = $AccountName
     }
 
     [object] hidden BuildAuth([string]$AccessToken) {
@@ -43,9 +44,9 @@ class GitHubApi
         return $this.InvokeRestMethod($url, 'Post', $null, $requestBody)
     }
 
-    [object] GetPullRequest([string]$BranchName, [string]$RepositoryOwner){
+    [object] GetPullRequest([string]$BranchName){
         $url = "pulls"
-        return $this.InvokeRestMethod($url, 'GET', "head=${RepositoryOwner}:$BranchName&base=main", $null)
+        return $this.InvokeRestMethod($url, 'GET', "head=$($this.RepositoryOwner):${BranchName}&base=main", $null)
     }
 
     [object] UpdatePullRequest([string]$Title, [string]$Body, [string]$BranchName, [string]$PullRequestNumber){
@@ -82,6 +83,39 @@ class GitHubApi
         return $releases
     }
 
+    [void] CreateRepositoryDispatch([string]$EventType, [object]$EventPayload) {
+        $url = "dispatches"
+        $body = @{
+            event_type = $EventType
+            client_payload = $EventPayload
+        } | ConvertTo-Json
+
+        $this.InvokeRestMethod($url, 'POST', $null, $body)
+    }
+
+    [object] GetWorkflowRuns([string]$WorkflowFileName) {
+        $url = "actions/workflows/$WorkflowFileName/runs"
+        return $this.InvokeRestMethod($url, 'GET', $null, $null)
+    }
+
+    [object] GetWorkflowRunJobs([string]$WorkflowRunId) {
+        $url = "actions/runs/$WorkflowRunId/jobs"
+        return $this.InvokeRestMethod($url, 'GET', $null, $null)
+    }
+
+    [void] CreateWorkflowDispatch([string]$WorkflowFileName, [string]$Ref, [object]$Inputs) {
+        $url = "actions/workflows/${WorkflowFileName}/dispatches"
+        $body = @{ ref = $Ref }
+        
+        if ($Inputs) {
+            $body.inputs = $Inputs
+        }
+
+        $jsonBody = $body | ConvertTo-Json
+
+        $this.InvokeRestMethod($url, 'POST', $null, $jsonBody)
+    }
+
     [string] hidden BuildUrl([string]$Url, [string]$RequestParams) {
         if ([string]::IsNullOrEmpty($RequestParams)) {
             return "$($this.BaseUrl)/$($Url)"
@@ -117,10 +151,18 @@ class GitHubApi
 
 function Get-GitHubApi {
     param (
-        [string] $AccountName,
-        [string] $ProjectName,
+        [Parameter(ParameterSetName = 'RepositorySingle')]
+        [string] $RepositoryFullName,
+        [Parameter(ParameterSetName = 'RepositorySplitted')]
+        [string] $RepositoryOwner,
+        [Parameter(ParameterSetName = 'RepositorySplitted')]
+        [string] $RepositoryName,
         [string] $AccessToken
     )
 
-    return [GitHubApi]::New($AccountName, $ProjectName, $AccessToken)
+    if ($PSCmdlet.ParameterSetName -eq "RepositorySingle") {
+        $RepositoryOwner, $RepositoryName = $RepositoryFullName.Split('/', 2)
+    }
+
+    return [GitHubApi]::New($RepositoryOwner, $RepositoryName, $AccessToken)
 }

github/run-ci-builds.ps1

@@ -0,0 +1,91 @@
+<#
+.SYNOPSIS
+Trigger runs on the workflow_dispatch event to build and upload tool packages
+
+.PARAMETER RepositoryFullName
+Required parameter. The owner and repository name. For example, 'actions/versions-package-tools'
+.PARAMETER AccessToken
+Required parameter. PAT to authorize
+.PARAMETER WorkflowFileName
+Required parameter. The name of workflow file that will be triggered
+.PARAMETER WorkflowDispatchRef
+Required parameter. The reference of the workflow run. The reference can be a branch, tag, or a commit SHA.
+.PARAMETER ToolVersions
+Required parameter. List of tool versions to build and upload
+.PARAMETER PublishReleases
+Required parameter. Whether to publish releases, true or false
+#>
+
+param (
+    [Parameter(Mandatory)] [string] $RepositoryFullName,
+    [Parameter(Mandatory)] [string] $AccessToken,
+    [Parameter(Mandatory)] [string] $WorkflowFileName,
+    [Parameter(Mandatory)] [string] $WorkflowDispatchRef,
+    [Parameter(Mandatory)] [string] $ToolVersions,
+    [Parameter(Mandatory)] [string] $PublishReleases
+)
+
+Import-Module (Join-Path $PSScriptRoot "github-api.psm1")
+
+function Get-WorkflowRunLink {
+    param(
+        [Parameter(Mandatory)] [object] $GitHubApi,
+        [Parameter(Mandatory)] [string] $WorkflowFileName,
+        [Parameter(Mandatory)] [string] $ToolVersion
+    )
+
+    $listWorkflowRuns = $GitHubApi.GetWorkflowRuns($WorkflowFileName).workflow_runs | Sort-Object -Property 'run_number' -Descending
+
+    foreach ($workflowRun in $listWorkflowRuns) {
+        $workflowRunJob = $gitHubApi.GetWorkflowRunJobs($workflowRun.id).jobs | Select-Object -First 1
+
+        if ($workflowRunJob.name -match $ToolVersion) {
+            return $workflowRun.html_url
+        }
+    }
+
+    return $null
+}
+
+function Queue-Builds {
+    param (
+        [Parameter(Mandatory)] [object] $GitHubApi,
+        [Parameter(Mandatory)] [string] $ToolVersions,
+        [Parameter(Mandatory)] [string] $WorkflowFileName,
+        [Parameter(Mandatory)] [string] $WorkflowDispatchRef,
+        [Parameter(Mandatory)] [string] $PublishReleases
+    )
+
+    $inputs = @{
+        PUBLISH_RELEASES = $PublishReleases
+    }
+    
+    $ToolVersions.Split(',') | ForEach-Object { 
+        $version = $_.Trim()
+        $inputs.VERSION = $version
+
+        Write-Host "Queue build for $version..."
+        $GitHubApi.CreateWorkflowDispatch($WorkflowFileName, $WorkflowDispatchRef, $inputs)
+
+        Start-Sleep -s 10
+        $workflowRunLink = Get-WorkflowRunLink -GitHubApi $GitHubApi `
+                                               -WorkflowFileName $WorkflowFileName `
+                                               -ToolVersion $version
+
+        if (-not $workflowRunLink) {
+            Write-Host "Could not find build for $version..."
+            exit 1
+        }
+
+        Write-Host "Link to the build: $workflowRunLink"
+    }
+}
+
+$gitHubApi = Get-GitHubApi -RepositoryFullName $RepositoryFullName -AccessToken $AccessToken
+
+Write-Host "Versions to build: $ToolVersions"
+Queue-Builds -GitHubApi $gitHubApi `
+             -ToolVersions $ToolVersions `
+             -WorkflowFileName $WorkflowFileName `
+             -WorkflowDispatchRef $WorkflowDispatchRef `
+             -PublishReleases $PublishReleases 

packages-generation/manifest-generator.ps1

@@ -1,13 +1,10 @@
 <#
 .SYNOPSIS
 Generate versions manifest based on repository releases
-
 .DESCRIPTION
 Versions manifest is needed to find the latest assets for particular version of tool
-.PARAMETER GitHubRepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER GitHubRepositoryName
-Required parameter. The name of tool repository
+.PARAMETER RepositoryFullName
+Required parameter. The owner and repository name. For example, 'actions/versions-package-tools'
 .PARAMETER GitHubAccessToken
 Required parameter. PAT Token to overcome GitHub API Rate limit
 .PARAMETER OutputFile
@@ -17,8 +14,7 @@ Path to the json file with parsing configuration
 #>
 
 param (
-    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
-    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
+    [Parameter(Mandatory)] [string] $RepositoryFullName,
     [Parameter(Mandatory)] [string] $GitHubAccessToken,
     [Parameter(Mandatory)] [string] $OutputFile,
     [Parameter(Mandatory)] [string] $ConfigurationFile
@@ -29,7 +25,7 @@ Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
 
 $configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
 
-$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
+$gitHubApi = Get-GitHubApi -RepositoryFullName $RepositoryFullName -AccessToken $GitHubAccessToken
 $releases = $gitHubApi.GetReleases()
 $versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration
 $versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force

packages-generation/manifest-utils.Tests.ps1

@@ -141,4 +141,24 @@ Describe "Build-VersionsManifest" {
         [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
         Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
     }
+
+    It "build correct manifest if release includes one asset" {
+        $asset = @(
+            @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+        )
+        $expectedManifestFile = @(
+            [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" }
+        )
+        
+        $releases = @(
+            @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $asset },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFile },
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
 }

packages-generation/manifest-utils.psm1

@@ -65,11 +65,13 @@ function Build-VersionsManifest {
         }
         
         $stable = $version.PreReleaseLabel ? $false : $true
+        [array]$releaseAssets = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration }
+
         $versionsHash.Add($versionKey, [PSCustomObject]@{
             version = $versionKey
             stable = $stable
             release_url = $release.html_url
-            files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration }
+            files = $releaseAssets
         })
     }
 

pester-extensions.psm1

@@ -4,30 +4,52 @@ Pester extension that allows to run command and validate exit code
 .EXAMPLE
 "python file.py" | Should -ReturnZeroExitCode
 #>
+
+function Get-CommandResult {
+    Param (
+        [Parameter(Mandatory=$true)]
+        [string] $Command,
+        [switch] $Multiline
+    )
+    # CMD and bash trick to suppress and show error output because some commands write to stderr (for example, "python --version")
+    If ($IsWindows) {
+        $stdout = & $env:comspec /c "$Command 2>&1"
+    } else {
+        $stdout = & bash -c "$Command 2>&1"
+    }
+    $exitCode = $LASTEXITCODE
+
+    return @{
+        Output = If ($Multiline -eq $true) { $stdout } else { [string]$stdout }
+        ExitCode = $exitCode
+    }
+}
+
 function ShouldReturnZeroExitCode {
     Param(
-        [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-        [String]$ActualValue,
-        [switch]$Negate
+        [String] $ActualValue,
+        [switch] $Negate,
+        [string] $Because # This parameter is unused by we need it to match Pester asserts signature
     )
 
-    Write-Host "Run command '${ActualValue}'"
-    Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ }
-    $actualExitCode = $LASTEXITCODE
+    $result = Get-CommandResult $ActualValue
 
-    [bool]$succeeded = $actualExitCode -eq 0
+    [bool]$succeeded = $result.ExitCode -eq 0
     if ($Negate) { $succeeded = -not $succeeded }
 
     if (-not $succeeded)
     {
-        $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}"
+        $commandOutputIndent = " " * 4
+        $commandOutput = ($result.Output | ForEach-Object { "${commandOutputIndent}${_}" }) -join "`n"
+        $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}`n${commandOutput}"
     }
 
-    return New-Object PSObject -Property @{
+    return [PSCustomObject] @{
         Succeeded      = $succeeded
         FailureMessage = $failureMessage
     }
 }
 
-Add-AssertionOperator -Name ReturnZeroExitCode `
-                    -Test  $function:ShouldReturnZeroExitCode
+if (Get-Command -Name Add-AssertionOperator -ErrorAction SilentlyContinue) {
+    Add-AssertionOperator -Name ReturnZeroExitCode -InternalName ShouldReturnZeroExitCode -Test ${function:ShouldReturnZeroExitCode}
+}