Merge pull request #118 from dmitry-shibanov/v-dmshib/fix-python-3.11.0-alpha.1

Fix python generation for 3.11

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @actions/virtual-environments-owners

.github/workflows/create-pr.yml

@@ -0,0 +1,33 @@
+name: Create Pull Request
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  create_pr:
+    name: Create Pull Request
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+
+    - name: Create versions-manifest.json
+      run: |
+        ./helpers/packages-generation/manifest-generator.ps1 -RepositoryFullName "$env:GITHUB_REPOSITORY" `
+                                                             -GitHubAccessToken "${{secrets.GITHUB_TOKEN}}" `
+                                                             -OutputFile "./versions-manifest.json" `
+                                                             -ConfigurationFile "./config/python-manifest-config.json"
+    - name: Create GitHub PR
+      run: |
+        $formattedDate = Get-Date -Format "MM/dd/yyyy"
+        ./helpers/github/create-pull-request.ps1 `
+                            -RepositoryFullName "$env:GITHUB_REPOSITORY" `
+                            -AccessToken "${{secrets.GITHUB_TOKEN}}" `
+                            -BranchName "update-versions-manifest-file" `
+                            -CommitMessage "Update versions-manifest" `
+                            -PullRequestTitle "[versions-manifest] Update for release from ${formattedDate}" `
+                            -PullRequestBody "Update versions-manifest.json for release from ${formattedDate}"

.github/workflows/create-release.yml

@@ -0,0 +1,18 @@
+name: Create release
+on:
+  repository_dispatch:
+    types: [create-release]
+
+jobs:
+  create_release:
+    name: Create release ${{ github.event.client_payload.ToolVersion }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Create release for Python ${{ github.event.client_payload.ToolVersion }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.event.client_payload.TagName }}
+        release_name: ${{ github.event.client_payload.ToolVersion }}
+        body: ${{ github.event.client_payload.ReleaseBody }}

.github/workflows/releases-validation.yml

@@ -1,5 +1,6 @@
 name: Validate 'versions-manifest.json' file
-on: 
+on:
+  workflow_dispatch:
   push:
     branches:
       - main
@@ -32,4 +33,4 @@ jobs:
       shell: pwsh
 
     - name: Run simple code
-      run: python -c 'import math; print(math.factorial(5))'
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/test-python-version.yml

@@ -0,0 +1,34 @@
+name: Test Python version
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Python version to build'
+        required: true
+      architecture:
+        description: 'The target architecture (x86, x64) of the Python'
+        required: false
+        default: 'x64'
+
+jobs:
+  test-python:
+    name: Test Python ${{ github.event.inputs.version }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04] 
+    steps:
+    - name: Setup Python ${{ github.event.inputs.version }}
+      uses: actions/setup-python@main
+      with:
+        python-version: ${{ github.event.inputs.version }}
+        architecture: ${{ github.event.inputs.architecture }}
+
+    - name: Validate version
+      run: |
+        python --version
+      shell: pwsh
+
+    - name: Run simple code
+      run: python -c 'import math; print(math.factorial(5))'

.github/workflows/validate-manifest.yml

@@ -0,0 +1,47 @@
+name: Validate manifest
+
+on:
+   schedule:
+    - cron:  '0 8,20 * * *'
+    
+   pull_request:
+     branches:
+      - main
+     paths:
+      - 'versions-manifest.json'
+env:
+  TOOL_NAME: "Python"
+defaults:
+  run:
+    shell: pwsh
+    
+jobs:
+  validation:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v2
+      with:
+          submodules: true
+          
+    - name: Validate python-versions manifest
+      run: .\helpers\packages-generation\manifest-validator.ps1 -ManifestUrl https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json -AccessToken ${{ secrets.GITHUB_TOKEN }}
+
+  check_build:
+    name: Check validation for failures 
+    runs-on: ubuntu-latest
+    needs: [validation]
+    if: failure()
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Send Slack notification if validation fails
+        run: |
+          $pipelineUrl = "$env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/actions/runs/$env:GITHUB_RUN_ID"
+          $message = "The validation of python-versions manifest failed. \nLink to the pipeline: $pipelineUrl"
+          .\helpers\get-new-tool-versions\send-slack-notification.ps1 -Url "${{ secrets.SLACK_CHANNEL_URL }}" `
+                                                                      -ToolName "${{ env.TOOL_NAME }}" `
+                                                                      -Text "$message" `
+                                                                      -ImageUrl "https://www.python.org/static/community_logos/python-powered-h-100x130.png"

README.md

@@ -10,7 +10,7 @@ Some versions are pre-installed on [virtual-environments](https://github.com/act
 More versions will (soon!) be available to install on-the-fly through the [`setup-python`](https://github.com/actions/setup-python) action.
 
 ## Adding new versions
-We are trying to build and release new versions of Python as soon as they are released. Please open an issue if any versions are missing.
+We are trying to build and release new versions of Python as soon as they are released. Please open an issue in [actions/virtual-environments](https://github.com/actions/virtual-environments) if any versions are missing.
 
 ## Contribution
 Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

azure-pipelines/build-python-packages.yml

@@ -116,3 +116,17 @@ stages:
     Architecture: x86
   jobs:
   - template: /azure-pipelines/templates/test-job.yml
+
+- stage: Publish_Release
+  dependsOn: [Test_Python_MacOS, Test_Python_Ubuntu_1604, Test_Python_Ubuntu_1804, Test_Python_Ubuntu_2004, Test_Python_x64_Windows, Test_Python_x86_Windows]
+  jobs:
+  - deployment: Publish_Release
+    pool:
+      name: Azure Pipelines
+      vmImage: ubuntu-18.04
+    environment: 'Get Available Tools Versions - Publishing Approval'
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - template: /azure-pipelines/templates/publish-release-steps.yml

azure-pipelines/templates/build-job.yml

@@ -7,7 +7,7 @@ jobs:
   steps:
   - checkout: self
     submodules: true
-    
+
   - task: PowerShell@2
     displayName: 'Build Python $(VERSION)'
     inputs:

azure-pipelines/templates/publish-release-steps.yml

@@ -0,0 +1,47 @@
+steps:
+- download: none
+
+- checkout: self
+  submodules: true
+
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: 'current'
+    path: $(Build.BinariesDirectory)
+
+- task: PowerShell@2
+  displayName: 'Create release Python $(VERSION)'
+  inputs:
+    TargetType: inline
+    script: |
+      $tagName = "$(VERSION)-$(Build.BuildId)"
+      $releaseBody = "Python $(VERSION)"
+      ./helpers/github/create-release.ps1 -RepositoryFullName "$(Build.Repository.Name)" `
+                                          -AccessToken "$(GITHUB_TOKEN)" `
+                                          -ToolVersion "$(VERSION)" `
+                                          -TagName "$tagName" `
+                                          -ReleaseBody "$releaseBody" `
+                                          -EventType "$(EVENT_TYPE)"
+- task: GitHubRelease@1
+  displayName: 'Upload release assets'
+  inputs:
+    gitHubConnection: 'Github Connection'
+    action: edit
+    tag: '$(VERSION)-$(Build.BuildId)'
+    title: '$(VERSION)'
+    releaseNotesSource: inline
+    releaseNotesInline: '$(RELEASE_NOTES_CONTENT)'
+    assets: '$(Build.BinariesDirectory)/*/*'
+    assetUploadMode: replace
+    addChangeLog: false
+
+- task: PowerShell@2
+  displayName: 'Trigger "Create Pull Request" workflow'
+  inputs:
+    TargetType: inline
+    script: |
+      Import-Module (Join-Path (Get-Location).Path "github-api.psm1")
+      $gitHubApi = Get-GitHubApi -RepositoryFullName "$(Build.Repository.Name)" -AccessToken "$(GITHUB_TOKEN)"
+      $gitHubApi.CreateWorkflowDispatch("$(WORKFLOW_FILE_NAME)", "$(WORKFLOW_DISPATCH_REF)", "$(INPUTS)")
+      Write-Host "Please find created Pull request here: $(Build.Repository.Uri)/pulls"
+    workingDirectory: '$(Build.SourcesDirectory)/helpers/github'

azure-pipelines/templates/test-job.yml

@@ -27,6 +27,7 @@ jobs:
       archiveFilePatterns: '$(Build.BinariesDirectory)/python-$(VERSION)-$(Platform)-$(Architecture).*'
       destinationFolder: $(Build.BinariesDirectory)
       cleanDestinationFolder: false
+      overwriteExistingFiles: true
 
   - task: PowerShell@2
     displayName: 'Apply build artifact to the local machines'
@@ -76,6 +77,7 @@ jobs:
           }
         }
         Invoke-Pester -Script $pesterParams -OutputFile "test_results.xml" -OutputFormat NUnitXml
+      pwsh: true
       workingDirectory: '$(Build.SourcesDirectory)/tests'
 
   - task: PublishTestResults@2

builders/build-python.ps1

@@ -1,6 +1,6 @@
-using module "./builders/win-python-builder.psm1"
+using module "./win-python-builder.psm1"
-using module "./builders/ubuntu-python-builder.psm1"
+using module "./ubuntu-python-builder.psm1"
-using module "./builders/macos-python-builder.psm1"
+using module "./macos-python-builder.psm1"
 
 <#
 .SYNOPSIS

builders/macos-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class macOSPythonBuilder : NixPythonBuilder {
     <#
@@ -40,10 +40,10 @@ class macOSPythonBuilder : NixPythonBuilder {
         ### and then add the appropriate paths for the header and library files to configure command.
         ### Link to documentation (https://cpython-devguide.readthedocs.io/setup/#build-dependencies)
         if ($this.Version -lt "3.7.0") {
-            $env:LDFLAGS = "-L$(brew --prefix openssl)/lib"
+            $env:LDFLAGS = "-L/usr/local/opt/[email protected]/lib"
-            $env:CFLAGS = "-I$(brew --prefix openssl)/include"
+            $env:CFLAGS = "-I/usr/local/opt/[email protected]/include"
         } else {
-            $configureString += " --with-openssl=/usr/local/opt/openssl"
+            $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
         }
 
         ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
@@ -52,6 +52,7 @@ class macOSPythonBuilder : NixPythonBuilder {
             $configureString += " --enable-loadable-sqlite-extensions"
             $env:LDFLAGS += " -L$(brew --prefix sqlite3)/lib"
             $env:CFLAGS += " -I$(brew --prefix sqlite3)/include"
+            $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
         }
 
         Execute-Command -Command $configureString

builders/nix-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class NixPythonBuilder : PythonBuilder {
     <#

builders/ubuntu-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/nix-python-builder.psm1"
+using module "./nix-python-builder.psm1"
 
 class UbuntuPythonBuilder : NixPythonBuilder {
     <#

builders/win-python-builder.psm1

@@ -1,4 +1,4 @@
-using module "./builders/python-builder.psm1"
+using module "./python-builder.psm1"
 
 class WinPythonBuilder : PythonBuilder {
     <#

installers/nix-setup-template.sh

@@ -50,7 +50,7 @@ chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJORMINOR pyth
 
 echo "Upgrading PIP..."
 ./python -m ensurepip
-./python -m pip install --ignore-installed pip
+./python -m pip install --ignore-installed pip --no-warn-script-location
 
 echo "Create complete file"
 touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

installers/win-setup-template.ps1

@@ -28,16 +28,20 @@ function Remove-RegistryEntries {
     $versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
 
     $regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
-    $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
+    if (Test-Path -Path Registry::$regPath) {
-    foreach ($key in $regKeys) {
+        $regKeys = Get-ChildItem -Path Registry::$regPath -Recurse | Where-Object Property -Ccontains DisplayName
-        if ($key.getValue("DisplayName") -match $versionFilter) {
+        foreach ($key in $regKeys) {
-            Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            if ($key.getValue("DisplayName") -match $versionFilter) {
+                Remove-Item -Path $key.PSParentPath -Recurse -Force -Verbose
+            }
         }
     }
 
     $regPath = "HKEY_CLASSES_ROOT\Installer\Products"
-    Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+    if (Test-Path -Path Registry::$regPath) {
-        Remove-Item Registry::$_ -Recurse -Force -Verbose
+        Get-ChildItem -Path Registry::$regPath | Where-Object { $_.GetValue("ProductName") -match $versionFilter } | ForEach-Object {
+            Remove-Item Registry::$_ -Recurse -Force -Verbose
+        }
     }
 
     $uninstallRegistrySections = @(
@@ -121,9 +125,14 @@ if ($LASTEXITCODE -ne 0) {
     Throw "Error happened during Python installation"
 }
 
+Write-Host "Create `python3` symlink"
+if ($MajorVersion -ne "2") {
+    New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"
+}
+
 Write-Host "Install and upgrade Pip"
 $PythonExePath = Join-Path -Path $PythonArchPath -ChildPath "python.exe"
-cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip"
+cmd.exe /c "$PythonExePath -m ensurepip && $PythonExePath -m pip install --upgrade pip --no-warn-script-location"
 
 Write-Host "Create complete file"
 New-Item -ItemType File -Path $PythonVersionPath -Name "$Architecture.complete" | Out-Null

tests/python-tests.ps1

@@ -1,5 +1,5 @@
 param (
-    [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
+    [semver] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Version,
     [string] [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
     $Platform
@@ -7,6 +7,7 @@ param (
 
 Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
+Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
 
 function Analyze-MissingModules([string] $buildOutputLocation) {
     $searchStringStart = "Failed to build these modules:"
@@ -20,9 +21,14 @@ function Analyze-MissingModules([string] $buildOutputLocation) {
     $regexMatch = [regex]::match($SplitBuiltOutput, $Pattern)
     if ($regexMatch.Success)
     {
+        $module = $regexMatch.Groups[1].Value.Trim()
         Write-Host "Failed missing modules:"
-        Write-Host $regexMatch.Groups[1].Value
+        Write-Host $module
-        return 1
+        if ( ($module -eq "_tkinter") -and ( [semver]"$($Version.Major).$($Version.Minor)" -ge [semver]"3.10" -and $Version.PreReleaseLabel ) ) {
+          Write-Host "$module $Version ignored"
+        } else {
+          return 1
+        }
     }
 
     return 0
@@ -47,6 +53,11 @@ Describe "Tests" {
         }
     }
 
+    It "Run pip" {
+        "pip install requests" | Should -ReturnZeroExitCode
+        "pip uninstall requests -y" | Should -ReturnZeroExitCode
+    }
+
     if (IsNixPlatform $Platform) {
 
         It "Check for failed modules in build_output" {
@@ -59,7 +70,8 @@ Describe "Tests" {
         }
 
         It "Check if python configuration is correct" {
-            "python ./sources/python-config-test.py $Version" | Should -ReturnZeroExitCode
+            $nativeVersion = Convert-Version -version $Version
+            "python ./sources/python-config-test.py $Version $nativeVersion" | Should -ReturnZeroExitCode
         }
 
         It "Check if shared libraries are linked correctly" {
@@ -68,11 +80,11 @@ Describe "Tests" {
     }
 
     # Pyinstaller 3.5 does not support Python 3.8.0. Check issue https://github.com/pyinstaller/pyinstaller/issues/4311
-    if ($Version -lt "3.8.0") {
+    if ($Version -lt "3.8.0" -and $Version.Major -ne "2") {
         It "Validate Pyinstaller" {
             "pip install pyinstaller" | Should -ReturnZeroExitCode
             "pyinstaller --onefile ./sources/simple-test.py" | Should -ReturnZeroExitCode
             "./dist/simple-test" | Should -ReturnZeroExitCode
         }
     }
-}
+}

tests/sources/python-config-test.py

@@ -1,4 +1,5 @@
 import distutils.sysconfig
+from distutils.version import LooseVersion
 import sysconfig
 import sys
 import platform
@@ -7,6 +8,7 @@ import os
 # Define variables
 os_type = platform.system()
 version = sys.argv[1]
+nativeVersion = sys.argv[2]
 
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
@@ -41,7 +43,7 @@ else:
 ### Validate macOS
 if os_type == 'Darwin':
     ### Validate openssl links
-    if version < "3.7.0":
+    if LooseVersion(nativeVersion) < LooseVersion("3.7.0"):
         expected_ldflags = '-L/usr/local/opt/[email protected]/lib'
         ldflags = sysconfig.get_config_var('LDFLAGS')
 
@@ -49,8 +51,8 @@ if os_type == 'Darwin':
             print('Invalid ldflags: %s; Expected: %s' % (ldflags, expected_ldflags))
             exit(1)
     else:
-        expected_openssl_includes = '-I/usr/local/opt/openssl/include'
+        expected_openssl_includes = '-I/usr/local/opt/openssl@1.1/include'
-        expected_openssl_ldflags ='-L/usr/local/opt/openssl/lib'
+        expected_openssl_ldflags ='-L/usr/local/opt/openssl@1.1/lib'
         
         openssl_includes = sysconfig.get_config_var('OPENSSL_INCLUDES')
         openssl_ldflags = sysconfig.get_config_var('OPENSSL_LDFLAGS')

tests/sources/python-modules.py

@@ -139,6 +139,7 @@ standard_library = [
     'sre_constants',
     'sre_parse',
     'ssl',
+    '_ssl',
     'stat',
     'string',
     'stringprep',
@@ -255,6 +256,15 @@ if sys.version_info > (3, 7):
 if sys.version_info > (3, 8):
     standard_library.remove('dummy_threading')
 
+# 'symbol' and 'formatter' modules have been removed from Python 3.10
+if sys.version_info >= (3, 10):
+    standard_library.remove('symbol')
+    standard_library.remove('formatter')
+
+# 'binhex' module has been removed from Python 3.11
+if sys.version_info >= (3, 11):
+    standard_library.remove('binhex')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',
@@ -265,7 +275,6 @@ excluded_modules = [
 def check_missing_modules(expected_modules):
     missing = []
     for module in expected_modules:
-        print('Try to import module ', module)
         try:
             importlib.import_module(module)
         except: